Skip to main content

subc_daemon/
control.rs

1#[path = "control/list_diagnostics.rs"]
2pub(crate) mod list_diagnostics;
3
4use std::{
5    collections::{BTreeMap, BTreeSet, HashMap, HashSet},
6    fmt,
7    net::SocketAddr,
8    path::{Path, PathBuf},
9    sync::{Arc, Mutex, RwLock},
10    time::{Duration, Instant as StdInstant},
11};
12
13use crate::scopes::admission::denied_capability;
14use serde::{Deserialize, Serialize};
15use subc_control::{
16    ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
17    ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
18    ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
19    ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
20    StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
21    SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
22    SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
23};
24use subc_protocol::{
25    call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
26    error_codes,
27    manifest::{
28        validate_hello_capability_grammar, validate_hello_confirm_operations,
29        validate_hello_display_name, validate_hello_event_declarations,
30        validate_hello_flow_operations, validate_hello_operation_deferral,
31        validate_hello_rich_ui_descriptor, validate_hello_role_declarations,
32        validate_hello_self_signal_declarations, validate_role_declarations_value,
33        CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
34        ProviderRole,
35    },
36    peer::{ObservedPeer, RouteBindPeer, OBSERVED_PEER_CAPABILITY, OPERATOR_APP_CAPABILITY},
37    provider_declarations::ProviderDeclarations,
38    scope::{
39        ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
40        CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
41        SCOPE_SYNC_OP,
42    },
43    session::{
44        validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
45        ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
46        OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
47        MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS, ROLE_VERSIONS_FIELD,
48    },
49    BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
50    Priority, RouteTarget, PROTOCOL_VERSION,
51};
52use tokio::time::{timeout_at, Instant};
53use tracing::{debug, info, instrument::WithSubscriber, warn};
54
55use crate::{
56    caller_identity::CallerIdentity,
57    capability_requirements::{
58        log_duplicate_claim_events, log_manifest_change, log_requirement_events,
59        CapabilityRequirementEvaluator, CapabilityVerdict, DuplicateClaimSource, RegisteredModule,
60        RequirementStatus, RuntimeModule,
61    },
62    child_roster::DaemonShutdownFlag,
63    daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
64    forwarding::{
65        CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
66        ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
67        PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
68    },
69    observability::{
70        ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
71    },
72    privileged_prompt::{prompt_text, PrivilegedOperation},
73    provenance::{
74        process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
75    },
76    registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
77    router::{daemon_error_flags, RouteCtx, RouterError},
78    scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
79    server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
80    stderr_tail::{CaptureState, TailEntry},
81    supervise::{
82        validate_spec, ModuleProcessLiveness, ReservedHelloRejection, ReservedHelloVerification,
83        SpawnSubscribeRefusal, SupervisorHandle, SwapHelloAdmission,
84    },
85    ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
86};
87
88/// Lowest envelope version this subc build will negotiate.
89///
90/// Module HELLO negotiation is exact: peers must use the daemon's locked
91/// protocol version. Older and newer peers receive `version_unsupported` and
92/// are not registered.
93pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
94
95const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
96const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
97const CAP_PING_PONG: &str = "ping_pong_v1";
98const CAP_SESSION_ATTACH: &str = "session_attach_v1";
99const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
100
101const SUBC_CONTROL_OPS: &[&str] = &[
102    "claim.mint",
103    "claim.present",
104    ops::DAEMON_SHUTDOWN_INTENT,
105    ops::ACTIVITY_RECORD,
106    ops::ACTIVITY_QUERY,
107    ops::ACTIVITY_SUBSCRIBE,
108    ops::SERVER_DESCRIBE,
109    ops::CATALOG_LIST,
110    ops::CATALOG_WATCH,
111    ops::ROUTE_OPEN,
112    ops::ROUTE_POLL,
113    ops::ROUTE_CLOSING,
114    ops::ROUTE_CLOSED,
115    ops::SUPERVISOR_LIST,
116    ops::SUPERVISOR_RESTART,
117    ops::SUPERVISOR_SWAP,
118    ops::SUPERVISOR_RELOAD,
119    ops::SUPERVISOR_RESCAN,
120    ops::SUPERVISOR_RELEASE_RESERVED,
121    ops::SUPERVISOR_SET_ENABLED,
122    ops::SUPERVISOR_HEALTH_PROBE,
123    ops::SUPERVISOR_HEALTH,
124    ops::SUPERVISOR_STDERR_TAIL,
125    ops::SUPERVISOR_TERMINALS,
126    ops::SUPERVISOR_ROUTES,
127    ops::SUPERVISOR_PROVENANCE,
128    ops::SUPERVISOR_SPAWN_SNAPSHOT,
129    ops::SUPERVISOR_SPAWN_SUBSCRIBE,
130];
131
132/// Client ops the daemon serves but leaves out of `subc_ops`. Each is
133/// announced by a capability in `server.describe` instead, and clients check
134/// that capability before sending it. `ck daemon --json` prints `subc_ops`
135/// verbatim and scripts read that output, so a new name there would change it.
136/// A malformed body for one of these is still refused `invalid_control_body`
137/// rather than as an unknown op.
138const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[
139    ops::SUPERVISOR_RESCAN_CHECKED,
140    "config.status",
141    "config.plan",
142    "config.apply",
143    "config.stamp",
144    "config.unstamp",
145    "config.history",
146    "config.rollback",
147];
148
149mod caller_attestation;
150mod channel_operations;
151#[cfg(all(test, unix, feature = "test-support"))]
152#[path = "control/channel_operations_tests.rs"]
153mod channel_operations_tests;
154pub(crate) mod claims;
155#[cfg(test)]
156#[path = "control/config_advertisement_tests.rs"]
157mod config_advertisement_tests;
158mod config_gate;
159use caller_attestation::{caller_for_target, scope_acts_for_another};
160#[cfg(all(test, unix, feature = "test-support"))]
161#[path = "control/caller_attestation_tests.rs"]
162mod caller_attestation_tests;
163mod scope_expiry;
164
165mod agent_commands;
166pub(crate) mod catalog;
167mod rescan_checked;
168use rescan_checked::{RescanMode, RescanPlan};
169
170const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
171    "claim.mint",
172    "claim.present",
173    MODULE_TO_SUBC_OP_CATALOG_UPDATE,
174    MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS,
175    "supervisor.live_roots",
176    "supervisor.agent_commands",
177    SCOPE_SYNC_OP,
178    SCOPE_APPLY_OP,
179    SCOPE_DESCRIBE_OP,
180    subc_protocol::scope::ROUTE_AUTHORIZE_OP,
181    "operator.confirm",
182];
183
184/// Module-originated ops the daemon answers but does not advertise in
185/// `HELLO_ACK`. Empty today; an op is served from here while the feature it
186/// belongs to is incomplete, so no module is told it works before it does.
187const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
188
189const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
190
191/// How long subc waits for a module to ack a relayed route.bind before returning
192/// `module_timeout`. The ack waits on the module's own configure, which for AFT
193/// includes a synchronous bounded project walk (up to ~20k files) plus gitignore
194/// and DB-open work — on a cold page cache or a large repo that legitimately
195/// exceeds a couple of seconds. The default is generous because rejecting a VALID
196/// bind is far worse than waiting on a slow one; a consumer that wants a tighter
197/// bound retries the bind itself (the sanctioned warm-bind-retry pattern).
198pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
199
200/// How many CONSECUTIVE full-budget relay timeouts against one target module
201/// open that module's bind-relay breaker.
202///
203/// Three, so that the breaker is NOT REACHABLE INSIDE ONE CLIENT CALL. Both
204/// SDKs default to a 30s request deadline and the relay budget defaults to 12s,
205/// so three consecutive full-budget timeouts take ~36s to observe: every client
206/// whose open contributed to opening the breaker had already given up on its
207/// own. That is what makes opening the breaker unable to turn a call that would
208/// have succeeded into a refusal — it can only make an already-failing module
209/// fail faster.
210///
211/// Two would be reachable inside one default deadline. One would convict a
212/// module on a single cold-cache bind, which is exactly the valid-but-slow case
213/// `DEFAULT_ROUTE_BIND_RELAY_TIMEOUT`'s own doc comment exists to protect.
214pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
215
216/// How long a module's bind-relay breaker stays open before exactly one
217/// `route.open` is let through as a probe.
218///
219/// Bounded BELOW by the relay budget: a cooldown at or under the 12s budget
220/// re-pays a full-budget stall almost continuously, and the breaker stops being
221/// a saving worth its own state. Bounded ABOVE by the SDKs' 30s default request
222/// deadline: a client that starts retrying after the module recovers has to get
223/// a probe opportunity inside its own deadline, or the breaker converts a
224/// recovered module into a failed call — the failure it exists to prevent,
225/// pointed the other way.
226///
227/// 20s sits between those with room on both sides, and it caps what a wedged
228/// module can cost at one full-budget wait per 20s ACROSS THE WHOLE DAEMON
229/// rather than one per `route.open` per connection. The stall that motivated
230/// this, with its measurements, is written up in
231/// `docs/designs/route-open-head-of-line.md`: 268 opens against one module each
232/// waited the whole budget out.
233pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
234
235const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
236const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
237
238fn reload_verdict(
239    configured: &Path,
240    spawned_from: Option<&Path>,
241    image: subc_control::RunningImageAgreement,
242) -> PendingReloadVerdict {
243    let path = match spawned_from {
244        Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
245            ReloadPathAgreement::Match
246        }
247        Some(spawned_from) => ReloadPathAgreement::Mismatch {
248            configured: configured.to_path_buf(),
249            spawned_from: spawned_from.to_path_buf(),
250        },
251        None => ReloadPathAgreement::Unavailable {
252            reason: if matches!(
253                image,
254                subc_control::RunningImageAgreement::Unavailable {
255                    reason: subc_control::RunningImageUnavailableReason::NotRunning
256                }
257            ) {
258                ReloadPathUnavailableReason::NotRunning
259            } else {
260                ReloadPathUnavailableReason::SpawnedPathUnavailable
261            },
262        },
263    };
264    PendingReloadVerdict::new(path, image)
265}
266
267fn launch_spec_agreement(
268    configured: &crate::supervise::ModuleSpec,
269    running: bool,
270    spawned: Option<&crate::supervise::LaunchSpecDigest>,
271) -> subc_control::LaunchSpecAgreement {
272    use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
273    if !running {
274        return Agreement::Unavailable {
275            reason: Reason::NotRunning,
276        };
277    }
278    match spawned {
279        None => Agreement::Unavailable {
280            reason: Reason::SpawnedSpecUnavailable,
281        },
282        Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
283            Agreement::Match
284        }
285        Some(_) => Agreement::Mismatch,
286    }
287}
288
289fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
290    if configured == spawned_from {
291        return true;
292    }
293    #[cfg(windows)]
294    {
295        // Windows spawn records the resolved verbatim path (\\?\...), while
296        // configuration keeps its original spelling. Resolve both with the OS
297        // instead of stripping prefixes or folding case, which could conflate
298        // distinct names in case-sensitive directories. Different resolved
299        // paths still differ even if their files have identical content.
300        match (
301            std::fs::canonicalize(configured),
302            std::fs::canonicalize(spawned_from),
303        ) {
304            (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
305            _ => false,
306        }
307    }
308    #[cfg(not(windows))]
309    {
310        false
311    }
312}
313
314#[derive(Clone)]
315struct DaemonProvenanceFacts {
316    build: DaemonBuildProvenance,
317    pid: Option<u32>,
318    started_at_ms: Option<u64>,
319    start_clock: Option<crate::clock::StartClock>,
320    executable_path: Option<PathBuf>,
321    executable_identity: Option<SpawnedFileIdentity>,
322    process_start_time: Option<u64>,
323    probe: ExecutableIdentityProbe,
324}
325
326impl Default for DaemonProvenanceFacts {
327    fn default() -> Self {
328        Self {
329            build: DaemonBuildProvenance {
330                build_git_sha: None,
331                build_lock_digest: None,
332            },
333            pid: None,
334            started_at_ms: None,
335            start_clock: None,
336            executable_path: None,
337            executable_identity: None,
338            process_start_time: None,
339            probe: ExecutableIdentityProbe::default(),
340        }
341    }
342}
343
344#[derive(Debug, Clone)]
345struct SupervisorRescanContext {
346    supervisor: Supervisor,
347    config_path: PathBuf,
348    configured_port: Option<u16>,
349    storage_config: Option<crate::daemon_config::StorageConfig>,
350    admission_facts_carrier_module_id: Option<String>,
351    admission_facts_targets: Option<Vec<String>>,
352    scope_authority_owners: Vec<String>,
353    federated_scope_owner: String,
354    privileged_operator_prompt: PrivilegedOperatorPrompt,
355    /// Counts executed rescans that set out to change the supervised modules'
356    /// staged or applied configuration, counted before the first change so a
357    /// rescan that fails partway still counts. With the daemon incarnation it
358    /// names the state a `supervisor.rescan_checked` preview was computed
359    /// against.
360    generation: Arc<std::sync::atomic::AtomicU64>,
361    module_entries: Arc<Mutex<rescan_checked::ModuleEntries>>,
362}
363
364/// Refusal labels passed to `observe_route_open_refusal` that mean the target
365/// module is not serving right now, and so open or extend an outage in the
366/// route outage tracker. Every one of them is only reachable after the target
367/// was found in the registry, which is what keeps an arbitrary client-chosen
368/// id from ever creating tracker state.
369///
370/// Deliberately absent: `not_registered` and `removed` (the id may be
371/// anything a client sent, and a removed module is gone on purpose),
372/// `protocol_none` (such a module never serves routes, so nothing is out),
373/// `role_not_provided`, `op_not_allowed`, `bad_consumer_identity`, the
374/// capability and admission-facts refusals (they refuse the caller, not a
375/// module outage), and `relay_reservation_failed` (its code ranges over
376/// capacity limits as well as a vanished connection). Capacity, breaker,
377/// relay-timeout and module-rejection refusals do not pass through that
378/// function at all; the breaker logs its own transitions.
379///
380/// The two not-serving refusals that bypass that function record themselves
381/// at their own sites: `supervised_not_registered` and `declared_not_ready`.
382/// `required_capability_unprovided` is not tracked: the module itself is up,
383/// and the outage belongs to the missing provider.
384const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
385    "reloading",
386    "supervisor_not_live",
387    "registration_not_active",
388    "no_forwarding_connection",
389    "relay_send_failed",
390];
391
392/// Observe test request cancellation relative to the connection's cleanup.
393#[cfg(test)]
394struct ActivityRequestPause {
395    connection: ConnectionId,
396    started: tokio::sync::Notify,
397    cancelled: tokio::sync::Notify,
398    cleanup_started: std::sync::atomic::AtomicBool,
399    cancelled_after_cleanup: std::sync::atomic::AtomicBool,
400}
401
402#[cfg(test)]
403struct ActivityRequestCancellation(Arc<ActivityRequestPause>);
404
405#[cfg(test)]
406impl Drop for ActivityRequestCancellation {
407    fn drop(&mut self) {
408        self.0.cancelled_after_cleanup.store(
409            self.0
410                .cleanup_started
411                .load(std::sync::atomic::Ordering::Acquire),
412            std::sync::atomic::Ordering::Release,
413        );
414        self.0.cancelled.notify_one();
415    }
416}
417
418/// A test hold: signal arrival, then wait for release on the holding thread.
419#[cfg(test)]
420type TestPause = (
421    tokio::sync::oneshot::Sender<()>,
422    std::sync::mpsc::Receiver<()>,
423);
424
425/// A test hook run once at a fixed point in a handler.
426#[cfg(test)]
427type TestHook = Box<dyn FnOnce() + Send>;
428
429/// A test hold for async code: signal arrival, then await release.
430#[cfg(test)]
431type AsyncTestPause = (
432    tokio::sync::oneshot::Sender<()>,
433    tokio::sync::oneshot::Receiver<()>,
434);
435
436/// Real channel-0 control handler for subc itself.
437#[derive(Clone)]
438pub struct ControlHandler {
439    #[cfg(test)]
440    activity_request_pause: Arc<Mutex<Option<Arc<ActivityRequestPause>>>>,
441    /// Holds connection cleanup at a numbered point: 0 is just after the agent
442    /// declaration queue closes, 1 just after the registration channel is
443    /// forgotten.
444    #[cfg(test)]
445    cleanup_pause: Arc<Mutex<Option<(u8, TestPause)>>>,
446    /// Runs once in `route.open`'s launch-nonce attestation, right after the
447    /// nonce is resolved, to interleave a nonce withdrawal there.
448    #[cfg(test)]
449    after_nonce_attestation: Arc<Mutex<Option<TestHook>>>,
450    /// Holds a registration channel's HELLO after its reader resolved the
451    /// channel and before the blocking HELLO handler starts.
452    #[cfg(test)]
453    hello_ingress_pause: Arc<Mutex<Option<AsyncTestPause>>>,
454    /// Holds an agent command declaration at a numbered point in its handler.
455    #[cfg(test)]
456    agent_commands_pause: Arc<Mutex<Option<(u8, TestPause)>>>,
457    claims: Arc<claims::Claims>,
458    registry: Arc<Registry>,
459    config_runtime: Option<Arc<crate::config_gate::runtime::RuntimeOwner>>,
460    config_review_admission: Arc<dyn crate::config_gate::ReviewAdmission>,
461    config_review_admission_is_default: bool,
462    activity: Option<crate::activity::ActivityLedger>,
463    activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
464    catalog_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
465    forwarding: Arc<ForwardingTable>,
466    process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
467    supervisor: SupervisorHandle,
468    pub(crate) resource_tracker: Arc<crate::resource_totals::Tracker>,
469    agent_declarations: agent_commands::Queues,
470    list_admission: Arc<tokio::sync::Semaphore>,
471    subc_capabilities: Arc<[String]>,
472    /// Daemon-wide route.bind relay budget. Used as the fallback when the
473    /// target module has no per-module override in
474    /// `route_bind_relay_timeouts`.
475    route_bind_relay_timeout: Duration,
476    /// Per-module route.bind relay budget overrides, keyed by module id. When
477    /// `handle_route_open` resolves the deadline for a target module, a
478    /// per-module entry wins over the daemon-wide value above.
479    route_bind_relay_timeouts: BTreeMap<String, Duration>,
480    /// Per-target-module bind-relay breaker state. Shared with the forwarding
481    /// table, which is where a new module connection resets it.
482    route_bind_breakers: RouteBindBreakers,
483    /// Live relay admissions keyed by target module. Shared through the
484    /// forwarding table so cloned or separately built handlers enforce one cap.
485    route_bind_concurrency: RouteBindConcurrency,
486    /// Start and end of each module's not-serving period as seen by
487    /// `route.open`, so an outage gets one line at each edge instead of only
488    /// the per-refusal INFO lines. Taken from the forwarding table, so every
489    /// handler built over one table shares it.
490    route_outages: Arc<crate::route_outage::RouteOutageTracker>,
491    /// Consecutive relay timeouts that open a module's breaker.
492    route_bind_breaker_threshold: u32,
493    /// How long a breaker stays open before one probe is admitted.
494    route_bind_breaker_cooldown: Duration,
495    health_probe_timeout: Duration,
496    /// Central storage policy. When set, each registering module receives its
497    /// resolved storage descriptor in HELLO_ACK; `None` leaves the field absent.
498    storage_config: Option<crate::daemon_config::StorageConfig>,
499    /// The machine id established at boot, served on every HELLO_ACK and on
500    /// `server.describe`. Fixed for the daemon's lifetime: `ck machine adopt`
501    /// changes the file, never this value. `None` serves no id.
502    machine_id: Option<crate::machine_id::MachineId>,
503    admission_facts_carrier_module_id: Option<String>,
504    admission_facts_targets: Option<Vec<String>>,
505    /// Scope records with their sync authorities and tombstones; see
506    /// `crate::scopes`. Shared by clones of this handler, so every connection
507    /// reads and writes one table.
508    scopes: Arc<RwLock<ScopeTable>>,
509    /// Wakes the expiry loop when an accepted scope change may have added a
510    /// deadline, so the loop can wait without ticking while no live scope has
511    /// one. A permit is stored when nothing waits, so no wake is lost.
512    scope_deadline_added: Arc<tokio::sync::Notify>,
513    /// The configured `scope_authority_owners`, kept so a rescan can report a
514    /// changed value as needing a daemon restart; rescan never applies it.
515    scope_authority_owners: Vec<String>,
516    federated_scope_owner: String,
517    /// Fixed at startup so rescan cannot change the policy that guards rescan itself.
518    privileged_operator_prompt: PrivilegedOperatorPrompt,
519    /// The local and peer address of each accepted TCP client connection, read
520    /// from the daemon's own socket, so a privileged request can name its caller.
521    caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
522    build_class_warned: Arc<Mutex<HashSet<ConnectionId>>>,
523    observed_peers: Arc<Mutex<HashMap<ConnectionId, ObservedPeer>>>,
524    captured_peers: Arc<Mutex<HashMap<ConnectionId, subc_os::local_peer::CapturedPeer>>>,
525    config_desktop_stamps:
526        Arc<Mutex<HashMap<ConnectionId, crate::operator_apps::OperatorAppStamp>>>,
527    local_connections: Arc<Mutex<HashSet<ConnectionId>>>,
528    pub(crate) operator_app_policy: crate::operator_apps::OperatorAppPolicy,
529    pub(crate) operator_apps:
530        Arc<Mutex<HashMap<ConnectionId, crate::operator_apps::OperatorAppStamp>>>,
531    /// One launch entry per app, behind a leaf lock: no other lock is taken
532    /// while it is held. Entries are memory-only, so a daemon or
533    /// installed-service restart drops them and a relaunched daemon never
534    /// mistakes an older process for its own launch.
535    pub(crate) launch_table: std::sync::Arc<crate::app_open::LaunchTable>,
536    /// Serializes `app.open` launches: the liveness check in the decision and
537    /// the later publish must not interleave across concurrent calls, or two
538    /// calls could both launch and the second publish would orphan the first
539    /// process. Refusals and `already_running` answers share the same lock;
540    /// `app.open` is rare, so the cost is negligible.
541    pub(crate) app_open_launch_lock: std::sync::Arc<tokio::sync::Mutex<()>>,
542    // Portable wire samples exercise dispatch without starting an OS process.
543    #[cfg(test)]
544    pub(crate) app_open_test_platform: Option<crate::operator_apps::AppOpenPlatform>,
545    #[cfg(test)]
546    pub(crate) app_open_test_launch: Option<fn() -> Result<u32, crate::app_open::AppOpenRefusal>>,
547    #[cfg(test)]
548    recovery_exit_test_after_step: Option<fn(usize) -> Result<(), crate::config_gate::GateError>>,
549    #[cfg(test)]
550    recovery_exit_test_completed: Option<fn(u64)>,
551    #[cfg(test)]
552    apply_reconcile_pause: Arc<Mutex<Option<AsyncTestPause>>>,
553    #[cfg(test)]
554    recovery_exit_test_quiet_retry: bool,
555    #[cfg(test)]
556    recovery_exit_test_retry_pause: Option<fn(u64) -> Option<PathBuf>>,
557    #[cfg(test)]
558    config_delivery_test_wakes:
559        Option<tokio::sync::mpsc::UnboundedSender<(&'static str, tokio::time::Instant)>>,
560    #[cfg(all(test, unix))]
561    config_delivery_test_idle:
562        Option<tokio::sync::mpsc::UnboundedSender<tokio::sync::oneshot::Sender<()>>>,
563    /// Test-only Windows signature and flash hooks. Production builds carry
564    /// neither: with the publisher unset, the production table never reaches
565    /// verification, and the default attention hook only logs.
566    #[allow(dead_code)]
567    #[cfg(windows)]
568    pub(crate) windows_verify: Option<crate::app_open::WindowsVerifyFn>,
569    #[allow(dead_code)]
570    #[cfg(windows)]
571    pub(crate) windows_flash: Option<crate::app_open::WindowsFlashFn>,
572    /// Test-only tracking fault hook: makes opening the tracking handle on
573    /// the launched pid fail, so a test can prove an untrackable launch is
574    /// stopped rather than left running.
575    #[allow(dead_code)]
576    #[cfg(windows)]
577    pub(crate) windows_track: Option<crate::app_open::WindowsTrackFn>,
578    /// Test-only count of Windows spawn attempts, so a test can prove a
579    /// refused launch started no process. Unit tests only.
580    #[cfg(all(test, windows))]
581    pub(crate) windows_spawn_attempts: Arc<std::sync::atomic::AtomicU64>,
582    #[cfg(test)]
583    caller_lookup: Option<TestCallerLookup>,
584    /// The launch nonce each module connection presented at HELLO, which is how
585    /// a `scope.sync` is matched to the owner's current launch.
586    hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
587    rescan: Option<SupervisorRescanContext>,
588    connected_clients: ConnectedClients,
589    counters: DaemonCounters,
590    /// Why this daemon serves no local endpoint, when binding one failed. `None`
591    /// when the endpoint is served, and also on a handler that was never told.
592    local_endpoint_unavailable: Option<String>,
593    capability_evaluator: Arc<CapabilityRequirementEvaluator>,
594    /// The supervisor's daemon-shutdown flag. Once it is set, requirement
595    /// changes are logged at INFO with `cause=daemon_shutdown` instead of at
596    /// their usual severity (see [`log_requirement_events`]). A handler built
597    /// without one has a flag that is never set.
598    daemon_shutdown: DaemonShutdownFlag,
599    daemon_provenance: DaemonProvenanceFacts,
600    #[cfg(test)]
601    control_dispatch_delay: Option<Duration>,
602    #[cfg(test)]
603    provenance_probe_override: Option<subc_control::RunningImageAgreement>,
604    /// Holds `route.open` after the route.bind is queued to the module and
605    /// before it is marked enqueued, so a test can make the module's reply (or
606    /// a client teardown) land exactly inside that window.
607    #[cfg(test)]
608    route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
609    /// Holds a HELLO after its registry entry is reserved and before its
610    /// forwarding endpoint is installed, so a test can read the catalog and
611    /// open routes while the module is registered but not yet routable. A
612    /// blocking barrier because the HELLO handler is synchronous; the test
613    /// runs the HELLO on its own thread.
614    #[cfg(test)]
615    hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
616    #[cfg(test)]
617    hello_ack_to_publication_hold: Option<Arc<std::sync::Barrier>>,
618    #[cfg(test)]
619    hello_before_publication_hold: Option<Arc<std::sync::Barrier>>,
620    #[cfg(test)]
621    hello_after_launch_check_hold: Option<Arc<std::sync::Barrier>>,
622    #[cfg(test)]
623    hello_inside_publication_hold: Option<Arc<std::sync::Barrier>>,
624}
625
626impl fmt::Debug for ControlHandler {
627    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
628        f.debug_struct("ControlHandler")
629            .field("registry", &self.registry)
630            .field("forwarding", &self.forwarding)
631            .field("process_liveness", &self.process_liveness.is_some())
632            .field("supervisor", &self.supervisor)
633            .field("subc_capabilities", &self.subc_capabilities)
634            .finish()
635    }
636}
637
638struct RouteOpenRequest {
639    target: RouteTarget,
640    identity: BindIdentity,
641    consumer_identity: Option<ConsumerIdentity>,
642    consumer_capabilities: Option<Vec<String>>,
643    role_versions: Option<BTreeMap<String, String>>,
644    busy_refusals: Option<bool>,
645    admission_facts: Option<serde_json::Value>,
646    scope: Option<ScopeSelector>,
647    accepts_close_reasons: Option<Vec<String>>,
648}
649
650#[derive(Clone, Copy)]
651enum RouteOpenPhase {
652    Admission,
653    ProjectRoot,
654    EgressReservation,
655    BindSend,
656    BindAckWait,
657}
658
659/// Store elapsed time for each route-open phase, including attempts that end
660/// in an early refusal or backend error. Admission also includes rechecks and
661/// commit bookkeeping performed between the other timed phases.
662struct RouteOpenTimings {
663    module_id: String,
664    principal: String,
665    admission: Duration,
666    project_root: Duration,
667    egress_reservation: Duration,
668    bind_send: Duration,
669    bind_ack_wait: Duration,
670    phase: Option<RouteOpenPhase>,
671    phase_started_at: Instant,
672}
673
674impl RouteOpenTimings {
675    fn new() -> Self {
676        Self {
677            module_id: String::new(),
678            // Refusals before attestation must not claim a verified principal.
679            principal: "unresolved".to_string(),
680            admission: Duration::ZERO,
681            project_root: Duration::ZERO,
682            egress_reservation: Duration::ZERO,
683            bind_send: Duration::ZERO,
684            bind_ack_wait: Duration::ZERO,
685            phase: Some(RouteOpenPhase::Admission),
686            phase_started_at: Instant::now(),
687        }
688    }
689
690    fn finish(&mut self) {
691        let elapsed = self.phase_started_at.elapsed();
692        match self.phase.take() {
693            Some(RouteOpenPhase::Admission) => self.admission += elapsed,
694            Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
695            Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
696            Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
697            Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
698            None => {}
699        }
700    }
701
702    fn enter(&mut self, phase: RouteOpenPhase) {
703        self.finish();
704        self.phase = Some(phase);
705        self.phase_started_at = Instant::now();
706    }
707}
708
709struct RouteBindReservationGuard {
710    forwarding: Arc<ForwardingTable>,
711    endpoint: ModuleEndpointId,
712    relay_corr: u64,
713    armed: bool,
714}
715
716struct ModuleControlRpcGuard {
717    forwarding: Arc<ForwardingTable>,
718    endpoint: ModuleEndpointId,
719    corr: u64,
720    armed: bool,
721}
722
723impl ModuleControlRpcGuard {
724    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
725        Self {
726            forwarding,
727            endpoint,
728            corr,
729            armed: true,
730        }
731    }
732
733    fn disarm(&mut self) {
734        self.armed = false;
735    }
736}
737
738impl Drop for ModuleControlRpcGuard {
739    fn drop(&mut self) {
740        if self.armed {
741            let _ = self
742                .forwarding
743                .cancel_module_control_rpc(self.endpoint, self.corr);
744        }
745    }
746}
747
748impl RouteBindReservationGuard {
749    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
750        Self {
751            forwarding,
752            endpoint,
753            relay_corr,
754            armed: true,
755        }
756    }
757
758    fn release_and_disarm(&mut self) {
759        if !self.armed {
760            return;
761        }
762        if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
763            self.endpoint,
764            self.relay_corr,
765            RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
766        ) {
767            send_goodbye_target_best_effort(
768                &self.forwarding.counters(),
769                &target,
770                "canceled route.bind",
771            );
772        }
773        self.armed = false;
774    }
775
776    fn disarm(&mut self) {
777        self.armed = false;
778    }
779}
780
781impl Drop for RouteBindReservationGuard {
782    fn drop(&mut self) {
783        self.release_and_disarm();
784    }
785}
786
787/// Per-target-module circuit breaker around the `route.bind` relay.
788///
789/// The connection reader is serial per connection, so a module whose `on_bind`
790/// sits on the ack blocks every LATER frame on the connections that call it,
791/// including calls to unrelated modules. This does not make any module's bind
792/// fast; it stops the daemon paying the full budget again and again for a
793/// condition it has already observed.
794///
795/// State is keyed by TARGET MODULE and shared by every connection: a wedged
796/// module wedges everyone, so what one connection learned should protect the
797/// rest.
798///
799/// THE MAP IS EMPTY WHILE THE FLEET IS HEALTHY. An entry appears only when a
800/// relay to that module has actually timed out, and is removed again when a
801/// relay is accepted or the module reconnects, so it cannot grow with traffic
802/// or with modules that behave.
803///
804/// # Why a `std` mutex here is not the head-of-line defect again
805///
806/// Acquisition never awaits. The critical section is a hash lookup plus a few
807/// integer updates, with no I/O and no `.await` inside it, so a reader task
808/// cannot be descheduled behind it the way it can behind
809/// `tokio::sync::Mutex::lock().await` or a semaphore permit. It is the same
810/// primitive, held for the same kind of work, as the refusal counter this very
811/// path already increments.
812///
813/// It is also NOT on the data-plane splice path: only `route.open` and module
814/// registration touch it, so bound-route frames gain no state check and no
815/// contention.
816#[derive(Debug, Clone, Default)]
817pub(crate) struct RouteBindBreakers {
818    modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
819}
820
821#[derive(Debug, Clone, Default)]
822pub(crate) struct RouteBindConcurrency {
823    modules: Arc<Mutex<HashMap<String, usize>>>,
824}
825
826struct RouteBindConcurrencyGuard {
827    concurrency: RouteBindConcurrency,
828    module_id: String,
829}
830
831impl RouteBindConcurrency {
832    /// Admit without waiting. Waiting here would move the bind stall from the
833    /// module reply to a semaphore and restore reader head-of-line blocking.
834    fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
835        let mut modules = self
836            .modules
837            .lock()
838            .expect("route.bind concurrency mutex poisoned");
839        let in_flight = modules.entry(module_id.to_string()).or_default();
840        if *in_flight >= limit {
841            return Err(*in_flight);
842        }
843        *in_flight += 1;
844        Ok(RouteBindConcurrencyGuard {
845            concurrency: self.clone(),
846            module_id: module_id.to_string(),
847        })
848    }
849}
850
851impl Drop for RouteBindConcurrencyGuard {
852    fn drop(&mut self) {
853        let mut modules = self
854            .concurrency
855            .modules
856            .lock()
857            .expect("route.bind concurrency mutex poisoned");
858        let remove = {
859            let in_flight = modules
860                .get_mut(&self.module_id)
861                .expect("admitted route.bind has a concurrency entry");
862            *in_flight -= 1;
863            *in_flight == 0
864        };
865        if remove {
866            modules.remove(&self.module_id);
867        }
868    }
869}
870
871#[derive(Debug, Default)]
872struct ModuleBreakerState {
873    /// Relay timeouts observed with no accepted relay in between.
874    consecutive_timeouts: u32,
875    /// `Some` while the breaker is open: the instant the cooldown expires and
876    /// the next arrival may probe. `None` means closed.
877    cooldown_until: Option<Instant>,
878    /// A half-open probe has been admitted and has not settled yet. This is
879    /// what makes the probe EXACTLY ONE: the flag is set under the same lock
880    /// that read the cooldown, so concurrent opens arriving at the moment the
881    /// cooldown expires cannot all decide that they are the probe.
882    probe_in_flight: Option<Arc<()>>,
883}
884
885/// What the breaker decided for one `route.open`, before any relay work.
886enum RouteBindAdmission<'a> {
887    Admitted {
888        guard: RouteBindBreakerGuard<'a>,
889        /// This open is the single half-open probe, so the transition is worth
890        /// one log line.
891        probe: bool,
892    },
893    Refused {
894        consecutive_timeouts: u32,
895        /// What is left of the cooldown. Zero when the refusal is because the
896        /// one probe is already in flight rather than because the cooldown has
897        /// not elapsed.
898        retry_in: Duration,
899        probe_in_flight: bool,
900    },
901}
902
903/// An outstanding admission, which must be told how its relay settled.
904///
905/// `Drop` settles it as inconclusive, so an early return between admission and
906/// the relay -- or the whole handler being cancelled when the client
907/// disconnects -- releases a half-open probe slot instead of leaving the
908/// breaker wedged half-open with no further probes.
909struct RouteBindBreakerGuard<'a> {
910    breakers: RouteBindBreakers,
911    module_id: &'a str,
912    probe_token: Option<Arc<()>>,
913    settled: bool,
914}
915
916impl RouteBindBreakerGuard<'_> {
917    /// The module answered within the budget and took the bind. THE ONLY
918    /// OUTCOME THAT CLEARS THE COUNT. Returns true when this closed an open
919    /// breaker, which is a transition worth logging.
920    fn record_accepted(&mut self) -> bool {
921        self.settled = true;
922        self.breakers.record_accepted(self.module_id)
923    }
924
925    /// The relay burned the whole budget with no answer. THE ONLY ARM THAT
926    /// COUNTS TOWARD OPENING.
927    fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
928        self.settled = true;
929        self.breakers.record_timeout(
930            self.module_id,
931            self.probe_token.as_ref(),
932            threshold,
933            cooldown,
934        )
935    }
936
937    /// Everything else: the module REJECTED the bind, its connection went away
938    /// mid-relay, or the waiter was cancelled.
939    ///
940    /// None of these is evidence that a module is slow, and each already has
941    /// its own refusal with its own code. A module that rejects a bind in
942    /// microseconds is healthy and must never be convicted for it; a module
943    /// that died has said nothing about the module that replaces it. So these
944    /// neither increment nor reset the count -- they only release a probe slot.
945    fn record_inconclusive(&mut self) {
946        self.settled = true;
947        self.breakers
948            .record_inconclusive(self.module_id, self.probe_token.as_ref());
949    }
950}
951
952impl Drop for RouteBindBreakerGuard<'_> {
953    fn drop(&mut self) {
954        if !self.settled {
955            self.breakers
956                .record_inconclusive(self.module_id, self.probe_token.as_ref());
957        }
958    }
959}
960
961/// The breaker moved to open, reported so the caller can log it outside the
962/// lock. Opening is rare and load-bearing; the refusals that follow are
963/// frequent and are counted rather than logged.
964struct BreakerOpened {
965    consecutive_timeouts: u32,
966    /// True when a failed probe re-opened an already-open breaker, which reads
967    /// very differently in a log from a first opening.
968    reopened_after_probe: bool,
969}
970
971impl RouteBindBreakers {
972    fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
973        self.modules
974            .lock()
975            .expect("route.bind breaker mutex poisoned")
976    }
977
978    /// Decide whether this `route.open` may attempt its relay. Takes the map
979    /// lock and nothing else, and never awaits.
980    fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
981        let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
982            probe: probe_token.is_some(),
983            guard: RouteBindBreakerGuard {
984                breakers: self.clone(),
985                module_id,
986                probe_token,
987                settled: false,
988            },
989        };
990
991        let mut modules = self.lock();
992        let Some(state) = modules.get_mut(module_id) else {
993            return admitted(None);
994        };
995        let Some(cooldown_until) = state.cooldown_until else {
996            return admitted(None);
997        };
998        if state.probe_in_flight.is_some() {
999            return RouteBindAdmission::Refused {
1000                consecutive_timeouts: state.consecutive_timeouts,
1001                retry_in: Duration::ZERO,
1002                probe_in_flight: true,
1003            };
1004        }
1005        let now = Instant::now();
1006        if now < cooldown_until {
1007            return RouteBindAdmission::Refused {
1008                consecutive_timeouts: state.consecutive_timeouts,
1009                retry_in: cooldown_until - now,
1010                probe_in_flight: false,
1011            };
1012        }
1013        let token = Arc::new(());
1014        state.probe_in_flight = Some(Arc::clone(&token));
1015        admitted(Some(token))
1016    }
1017
1018    fn record_accepted(&self, module_id: &str) -> bool {
1019        self.lock()
1020            .remove(module_id)
1021            .is_some_and(|state| state.cooldown_until.is_some())
1022    }
1023
1024    fn record_timeout(
1025        &self,
1026        module_id: &str,
1027        probe_token: Option<&Arc<()>>,
1028        threshold: u32,
1029        cooldown: Duration,
1030    ) -> Option<BreakerOpened> {
1031        let mut modules = self.lock();
1032        let state = modules.entry(module_id.to_string()).or_default();
1033        let was_open = state.cooldown_until.is_some();
1034        let was_probe = Self::owns_probe(state, probe_token);
1035        if was_probe {
1036            state.probe_in_flight = None;
1037        }
1038        state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
1039        if state.consecutive_timeouts < threshold {
1040            return None;
1041        }
1042        state.cooldown_until = Some(Instant::now() + cooldown);
1043        Some(BreakerOpened {
1044            consecutive_timeouts: state.consecutive_timeouts,
1045            reopened_after_probe: was_open && was_probe,
1046        })
1047    }
1048
1049    fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
1050        // A relay can finish after the breaker was reset or after it opened
1051        // again and started a new probe. Only the guard whose token matches the
1052        // active probe may release it, so a late relay never frees a newer probe.
1053        state
1054            .probe_in_flight
1055            .as_ref()
1056            .zip(token)
1057            .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
1058    }
1059
1060    fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
1061        if let Some(state) = self.lock().get_mut(module_id) {
1062            if Self::owns_probe(state, probe_token) {
1063                state.probe_in_flight = None;
1064            }
1065        }
1066    }
1067
1068    /// Discard what was learned about a module, because the process it was
1069    /// learned about is gone. Returns the discarded count when it was non-zero.
1070    ///
1071    /// A BREAKER IS A CACHED VERDICT ABOUT A PROCESS, NOT ABOUT A NAME. A
1072    /// `module_id` is a configuration identity that outlives any particular
1073    /// child; what the breaker observed was the process behind the module
1074    /// connection of the moment. When a new connection registers under that id
1075    /// the verdict's subject no longer exists, so the verdict is stale by
1076    /// construction rather than merely likely to be wrong. Keeping it would
1077    /// apply a dead process's record to a live one, which is the same defect
1078    /// class this breaker exists to stop the daemon committing.
1079    ///
1080    /// A half-open probe in flight is discarded with the rest: it was a
1081    /// question about the old process.
1082    pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
1083        self.lock()
1084            .remove(module_id)
1085            .map(|state| state.consecutive_timeouts)
1086            .filter(|discarded| *discarded > 0)
1087    }
1088
1089    /// Open breakers, for the `server.describe` counters object. `None` when
1090    /// none is open, so the key stays absent rather than present-and-empty.
1091    ///
1092    /// This is the operator's answer to "is this module refusing instantly or
1093    /// is it fine?", which look identical from a client that retries and then
1094    /// succeeds.
1095    fn open_snapshot(&self) -> Option<serde_json::Value> {
1096        let now = Instant::now();
1097        let modules = self.lock();
1098        let open = modules
1099            .iter()
1100            .filter_map(|(module_id, state)| {
1101                let cooldown_until = state.cooldown_until?;
1102                Some((
1103                    module_id.clone(),
1104                    serde_json::json!({
1105                        "consecutive_timeouts": state.consecutive_timeouts,
1106                        "cooldown_remaining_ms":
1107                            cooldown_until.saturating_duration_since(now).as_millis() as u64,
1108                        "probe_in_flight": state.probe_in_flight.is_some(),
1109                    }),
1110                ))
1111            })
1112            .collect::<serde_json::Map<String, serde_json::Value>>();
1113        (!open.is_empty()).then_some(serde_json::Value::Object(open))
1114    }
1115}
1116
1117impl ControlHandler {
1118    pub(crate) fn install_registration_service(
1119        &self,
1120        router: std::sync::Weak<crate::Router>,
1121        auth: crate::server::ServerAuth,
1122    ) {
1123        self.supervisor.install_registration_service(router, auth);
1124    }
1125    pub fn new(registry: Arc<Registry>) -> Self {
1126        Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
1127    }
1128
1129    pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
1130        let counters = forwarding.counters();
1131        // Taken from the forwarding table rather than created here, so that the
1132        // breaker a `route.open` consults is the same one a module's
1133        // registration resets, however many handlers are built over one table.
1134        let route_bind_breakers = forwarding.route_bind_breakers();
1135        let route_bind_concurrency = forwarding.route_bind_concurrency();
1136        let route_outages = forwarding.route_outages();
1137        Self {
1138            registry,
1139            #[cfg(test)]
1140            activity_request_pause: Arc::new(Mutex::new(None)),
1141            #[cfg(test)]
1142            cleanup_pause: Arc::new(Mutex::new(None)),
1143            #[cfg(test)]
1144            after_nonce_attestation: Arc::new(Mutex::new(None)),
1145            #[cfg(test)]
1146            hello_ingress_pause: Arc::new(Mutex::new(None)),
1147            #[cfg(test)]
1148            agent_commands_pause: Arc::new(Mutex::new(None)),
1149            claims: Arc::new(claims::Claims::default()),
1150            config_runtime: None,
1151            config_review_admission: Arc::new(crate::config_gate::MissingReviewAdmission),
1152            config_review_admission_is_default: true,
1153            activity: None,
1154            activity_streams: Arc::new(Mutex::new(HashMap::new())),
1155            catalog_streams: Arc::new(Mutex::new(HashMap::new())),
1156            forwarding,
1157            process_liveness: None,
1158            supervisor: SupervisorHandle::new(),
1159            resource_tracker: Arc::new(crate::resource_totals::Tracker::new(
1160                env!("CARGO_PKG_VERSION").to_owned(),
1161            )),
1162            agent_declarations: agent_commands::Queues::default(),
1163            list_admission: Arc::new(tokio::sync::Semaphore::new(4)),
1164            subc_capabilities: Arc::from([
1165                CAP_MANIFEST_REGISTRATION.to_string(),
1166                CAP_CHANNEL_LIFECYCLE.to_string(),
1167                CAP_PING_PONG.to_string(),
1168                CAP_SESSION_ATTACH.to_string(),
1169                CAP_ADMISSION_FACTS_RELAY.to_string(),
1170                CAP_SCOPES_V1.to_string(),
1171                subc_protocol::scope::SCOPE_FEDERATED_CAPABILITY.to_string(),
1172                subc_protocol::peer::CALLER_ATTESTATION_CAPABILITY.to_string(),
1173                CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
1174                subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
1175                subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
1176                subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
1177                subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
1178                subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
1179                subc_protocol::scope::WORKER_RUNS_CAPABILITY.to_string(),
1180                subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
1181                subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY.to_string(),
1182                subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
1183                subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
1184                subc_protocol::ROUTE_CLOSE_REASONS_CAPABILITY.to_string(),
1185            ]),
1186            route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
1187            route_bind_relay_timeouts: BTreeMap::new(),
1188            route_bind_breakers,
1189            route_bind_concurrency,
1190            route_outages,
1191            route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
1192            route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
1193            health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
1194            storage_config: None,
1195            machine_id: None,
1196            admission_facts_carrier_module_id: None,
1197            admission_facts_targets: None,
1198            scopes: Arc::new(RwLock::new(ScopeTable::new(
1199                crate::daemon_config::default_scope_authority_owners(),
1200            ))),
1201            scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
1202            scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
1203            federated_scope_owner: crate::daemon_config::default_federated_scope_owner(),
1204            privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
1205            caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
1206            build_class_warned: Arc::new(Mutex::new(HashSet::new())),
1207            observed_peers: Arc::new(Mutex::new(HashMap::new())),
1208            captured_peers: Arc::new(Mutex::new(HashMap::new())),
1209            config_desktop_stamps: Arc::new(Mutex::new(HashMap::new())),
1210            local_connections: Arc::new(Mutex::new(HashSet::new())),
1211            operator_app_policy: Default::default(),
1212            operator_apps: Arc::new(Mutex::new(HashMap::new())),
1213            launch_table: std::sync::Arc::new(crate::app_open::LaunchTable::new()),
1214            app_open_launch_lock: std::sync::Arc::new(tokio::sync::Mutex::new(())),
1215            #[cfg(test)]
1216            app_open_test_platform: None,
1217            #[cfg(test)]
1218            app_open_test_launch: None,
1219            #[cfg(test)]
1220            recovery_exit_test_after_step: None,
1221            #[cfg(test)]
1222            recovery_exit_test_completed: None,
1223            #[cfg(test)]
1224            apply_reconcile_pause: Arc::new(Mutex::new(None)),
1225            #[cfg(test)]
1226            recovery_exit_test_quiet_retry: false,
1227            #[cfg(test)]
1228            recovery_exit_test_retry_pause: None,
1229            #[cfg(test)]
1230            config_delivery_test_wakes: None,
1231            #[cfg(all(test, unix))]
1232            config_delivery_test_idle: None,
1233            #[cfg(windows)]
1234            windows_verify: None,
1235            #[cfg(windows)]
1236            windows_flash: None,
1237            #[cfg(windows)]
1238            windows_track: None,
1239            #[cfg(all(test, windows))]
1240            windows_spawn_attempts: Arc::new(std::sync::atomic::AtomicU64::new(0)),
1241            #[cfg(test)]
1242            caller_lookup: None,
1243            hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
1244            rescan: None,
1245            connected_clients: ConnectedClients::new(),
1246            local_endpoint_unavailable: None,
1247            counters,
1248            capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
1249            daemon_shutdown: DaemonShutdownFlag::default(),
1250            daemon_provenance: DaemonProvenanceFacts::default(),
1251            #[cfg(test)]
1252            control_dispatch_delay: None,
1253            #[cfg(test)]
1254            provenance_probe_override: None,
1255            #[cfg(test)]
1256            route_bind_relay_sent_hold: None,
1257            #[cfg(test)]
1258            hello_registry_to_forwarding_hold: None,
1259            #[cfg(test)]
1260            hello_ack_to_publication_hold: None,
1261            #[cfg(test)]
1262            hello_before_publication_hold: None,
1263            #[cfg(test)]
1264            hello_after_launch_check_hold: None,
1265            #[cfg(test)]
1266            hello_inside_publication_hold: None,
1267        }
1268    }
1269
1270    /// Set the central storage policy: registering modules then receive their
1271    /// resolved storage descriptor in HELLO_ACK.
1272    pub fn with_storage_config(
1273        mut self,
1274        storage_config: Option<crate::daemon_config::StorageConfig>,
1275    ) -> Self {
1276        self.storage_config = storage_config;
1277        self
1278    }
1279
1280    /// Open the ledger on its own writer thread; an unavailable sink does not
1281    /// prevent the daemon from serving routing or supervisor requests.
1282    pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1283        self.activity = Some(crate::activity::ActivityLedger::open(
1284            run_dir.join("activity.jsonl"),
1285            incarnation,
1286        ));
1287        self.supervisor.set_activity(self.activity.clone());
1288        self
1289    }
1290
1291    /// Set the machine id served to every registering module (HELLO_ACK) and on
1292    /// `server.describe`.
1293    pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1294        self.machine_id = machine_id;
1295        self
1296    }
1297
1298    /// Configure the exact reserved module and target ids permitted to relay
1299    /// opaque admission facts. Config-file loading validates this authority;
1300    /// this builder keeps the same policy available to embedded test daemons.
1301    pub fn with_admission_facts_config(
1302        mut self,
1303        carrier_module_id: Option<String>,
1304        targets: Option<Vec<String>>,
1305    ) -> Self {
1306        self.admission_facts_carrier_module_id = carrier_module_id;
1307        self.admission_facts_targets = targets;
1308        self
1309    }
1310
1311    /// Set the module ids whose scopes may carry `agent_id` and `delegates`.
1312    /// Replaces the scope table with an empty one under the new list, so call it
1313    /// while building the handler, before any module can sync.
1314    pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1315        self.scopes = Arc::new(RwLock::new(
1316            ScopeTable::new(owners.iter().cloned())
1317                .with_federated_owner(self.federated_scope_owner.clone()),
1318        ));
1319        self.scope_authority_owners = owners;
1320        self
1321    }
1322
1323    /// Set the exclusive federated owner while building the handler, before syncs.
1324    pub fn with_federated_scope_owner(mut self, owner: String) -> Self {
1325        self.scopes = Arc::new(RwLock::new(
1326            ScopeTable::new(self.scope_authority_owners.iter().cloned())
1327                .with_federated_owner(owner.clone()),
1328        ));
1329        self.federated_scope_owner = owner;
1330        self
1331    }
1332
1333    /// Set the startup policy before installing the rescan context or serving requests.
1334    pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1335        self.privileged_operator_prompt = mode;
1336        self
1337    }
1338
1339    /// Override the route.bind relay timeout. Used by tests that assert the
1340    /// timeout path so they don't block on the production-safe default.
1341    pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1342        self.route_bind_relay_timeout = timeout;
1343        self
1344    }
1345
1346    /// Install per-module route.bind relay budget overrides. A module id
1347    /// listed here wins over the daemon-wide default set via
1348    /// `with_route_bind_relay_timeout`. Values are pre-resolved at parse time
1349    /// from `subc.jsonc` (per-module > daemon-wide > absent), so callers pass
1350    /// the same `Duration` the bind path will use.
1351    pub fn with_route_bind_relay_timeouts(
1352        mut self,
1353        timeouts: impl IntoIterator<Item = (String, Duration)>,
1354    ) -> Self {
1355        self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1356        self
1357    }
1358
1359    /// Resolve the route.bind relay budget for a specific target module id.
1360    /// Per-module overrides win; the daemon-wide value (set via
1361    /// `with_route_bind_relay_timeout` or the built-in default) is the
1362    /// fallback. Exposed so config-aware callers (bootstrap, tests) can audit
1363    /// the same resolution `handle_route_open` will use.
1364    pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1365        self.route_bind_relay_timeouts
1366            .get(module_id)
1367            .copied()
1368            .unwrap_or(self.route_bind_relay_timeout)
1369    }
1370
1371    /// Override the per-module bind-relay breaker policy.
1372    ///
1373    /// Used by tests, which cannot spend three production budgets opening a
1374    /// breaker or twenty seconds waiting for its cooldown. The production
1375    /// values are `DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD` and
1376    /// `DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN`, whose doc comments carry the
1377    /// reasoning for the numbers.
1378    pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1379        self.route_bind_breaker_threshold = threshold.max(1);
1380        self.route_bind_breaker_cooldown = cooldown;
1381        self
1382    }
1383
1384    #[cfg(test)]
1385    pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1386        self.health_probe_timeout = timeout;
1387        self
1388    }
1389
1390    #[cfg(test)]
1391    pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1392        self.control_dispatch_delay = Some(delay);
1393        self
1394    }
1395
1396    /// `route.open` waits on `hold` twice after queueing a route.bind: once to
1397    /// announce it is parked, once to be released. Build the barrier for two
1398    /// parties, the test being the other one.
1399    #[cfg(test)]
1400    pub(crate) fn with_route_bind_relay_sent_hold(
1401        mut self,
1402        hold: Arc<tokio::sync::Barrier>,
1403    ) -> Self {
1404        self.route_bind_relay_sent_hold = Some(hold);
1405        self
1406    }
1407
1408    /// A HELLO waits on `hold` twice between reserving its registry entry and
1409    /// installing its forwarding endpoint: once to announce it is parked, once
1410    /// to be released. Build the barrier for two parties, the test being the
1411    /// other one, and run the HELLO on a thread of its own.
1412    #[cfg(test)]
1413    pub(crate) fn with_hello_registry_to_forwarding_hold(
1414        mut self,
1415        hold: Arc<std::sync::Barrier>,
1416    ) -> Self {
1417        self.hello_registry_to_forwarding_hold = Some(hold);
1418        self
1419    }
1420
1421    #[cfg(test)]
1422    fn hold_between_registry_and_forwarding(&self) {
1423        if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1424            hold.wait();
1425            hold.wait();
1426        }
1427    }
1428
1429    /// Park after the ACK is queued and forwarding is installed, until a test
1430    /// has read the ACK and issued a registry lookup from another connection.
1431    #[cfg(test)]
1432    fn hold_between_ack_and_publication(&self) {
1433        if let Some(hold) = &self.hello_ack_to_publication_hold {
1434            hold.wait();
1435            hold.wait();
1436        }
1437    }
1438
1439    #[cfg(all(test, unix))]
1440    pub(crate) fn with_hello_before_publication_hold(
1441        mut self,
1442        hold: Arc<std::sync::Barrier>,
1443    ) -> Self {
1444        self.hello_before_publication_hold = Some(hold);
1445        self
1446    }
1447
1448    #[cfg(all(test, unix))]
1449    pub(crate) fn with_hello_after_launch_check_hold(
1450        mut self,
1451        hold: Arc<std::sync::Barrier>,
1452    ) -> Self {
1453        self.hello_after_launch_check_hold = Some(hold);
1454        self
1455    }
1456
1457    #[cfg(all(test, unix))]
1458    pub(crate) fn with_hello_inside_publication_hold(
1459        mut self,
1460        hold: Arc<std::sync::Barrier>,
1461    ) -> Self {
1462        self.hello_inside_publication_hold = Some(hold);
1463        self
1464    }
1465
1466    pub fn with_process_liveness(
1467        mut self,
1468        process_liveness: Arc<dyn ModuleProcessLiveness>,
1469    ) -> Self {
1470        self.process_liveness = Some(process_liveness);
1471        self
1472    }
1473
1474    pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1475        self.supervisor = supervisor;
1476        if let Some(owner) = &self.config_runtime {
1477            self.supervisor.install_binding_retirement(owner);
1478        }
1479        self.supervisor.set_activity(self.activity.clone());
1480        self
1481    }
1482
1483    pub fn with_daemon_provenance(
1484        mut self,
1485        pid: u32,
1486        started_at_ms: u64,
1487        executable_path: Option<PathBuf>,
1488        build_git_sha: Option<String>,
1489        build_lock_digest: Option<String>,
1490    ) -> Self {
1491        let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1492        let process_start_time = process_start_time(pid);
1493        self.daemon_provenance = DaemonProvenanceFacts {
1494            build: DaemonBuildProvenance {
1495                build_git_sha,
1496                build_lock_digest,
1497            },
1498            pid: Some(pid),
1499            started_at_ms: Some(started_at_ms),
1500            start_clock: None,
1501            executable_path,
1502            executable_identity,
1503            process_start_time,
1504            probe: ExecutableIdentityProbe::default(),
1505        };
1506        self
1507    }
1508
1509    pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1510        self.daemon_provenance.start_clock = Some(clock);
1511        self
1512    }
1513
1514    /// Inject the Unix wall-clock millisecond source used by every scope expiry check.
1515    pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1516        self.scopes
1517            .write()
1518            .unwrap_or_else(|poisoned| poisoned.into_inner())
1519            .set_wall_clock(Arc::new(clock));
1520        self
1521    }
1522
1523    #[cfg(test)]
1524    fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1525        self.provenance_probe_override = Some(result);
1526        self
1527    }
1528
1529    /// Install the configured module set and its reserved capability bindings.
1530    /// Bindings are configuration-scoped and may point at a provider that has not
1531    /// been installed yet, so this does not require the bound module to exist.
1532    pub fn with_capability_config(
1533        self,
1534        modules: impl IntoIterator<Item = (String, bool)>,
1535        reserved_capabilities: BTreeMap<String, String>,
1536    ) -> Self {
1537        self.capability_evaluator
1538            .configure(modules, reserved_capabilities);
1539        self
1540    }
1541
1542    /// Share the supervisor's daemon-shutdown flag with this handler, so
1543    /// requirement changes after the daemon has begun its announced shutdown
1544    /// are not reported as faults.
1545    pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1546        self.daemon_shutdown = flag;
1547        self
1548    }
1549
1550    pub fn with_supervisor_rescan(
1551        mut self,
1552        supervisor: Supervisor,
1553        config_path: impl Into<PathBuf>,
1554        configured_port: Option<u16>,
1555    ) -> Self {
1556        self.rescan = Some(SupervisorRescanContext {
1557            supervisor,
1558            config_path: config_path.into(),
1559            configured_port,
1560            storage_config: self.storage_config.clone(),
1561            admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1562            admission_facts_targets: self.admission_facts_targets.clone(),
1563            scope_authority_owners: self.scope_authority_owners.clone(),
1564            federated_scope_owner: self.federated_scope_owner.clone(),
1565            privileged_operator_prompt: self.privileged_operator_prompt,
1566            generation: Arc::default(),
1567            module_entries: Arc::default(),
1568        });
1569        self
1570    }
1571
1572    pub(crate) fn with_supervisor_rescan_entries(
1573        self,
1574        modules: &[crate::daemon_config::ConfiguredModule],
1575    ) -> Self {
1576        if let Some(context) = &self.rescan {
1577            let mut entries = context
1578                .module_entries
1579                .lock()
1580                .unwrap_or_else(|e| e.into_inner());
1581            entries.seed(modules);
1582        }
1583        self
1584    }
1585
1586    pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1587        self.connected_clients = connected_clients;
1588        self
1589    }
1590
1591    /// Report why this daemon serves no local endpoint, in `server.describe`, so
1592    /// an operator can see a bind failure without reading the daemon log.
1593    pub fn with_local_endpoint_unavailable(mut self, reason: Option<String>) -> Self {
1594        self.local_endpoint_unavailable = reason;
1595        self
1596    }
1597
1598    pub fn forwarding(&self) -> Arc<ForwardingTable> {
1599        Arc::clone(&self.forwarding)
1600    }
1601
1602    pub(crate) fn counters(&self) -> DaemonCounters {
1603        self.counters.clone()
1604    }
1605
1606    /// Wake at each candidate's own deadline so a stalled fresh exec emits its
1607    /// requirement event without depending on an operator polling a status command.
1608    pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1609        let handler = Arc::downgrade(&self);
1610        let evaluator = self.capability_evaluator.clone();
1611        tokio::spawn(async move {
1612            loop {
1613                evaluator.wait_for_change_or_deadline().await;
1614                let Some(handler) = handler.upgrade() else {
1615                    break;
1616                };
1617                handler.refresh_capability_requirements();
1618            }
1619        });
1620    }
1621
1622    /// The timer only prompts sweeps. Each sweep reads wall time again, so a
1623    /// deadline passed during sleep is enforced on the first tick after wake.
1624    /// While no live scope carries a deadline the loop waits for an accepted
1625    /// scope change instead of ticking, so a daemon without deadlines never
1626    /// wakes for expiry.
1627    pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1628        let context = scope_expiry::ScopeExpiryContext::new(&self);
1629        let deadline_added = self.scope_deadline_added.clone();
1630        tokio::spawn(async move {
1631            loop {
1632                if !context.has_deadlines() {
1633                    deadline_added.notified().await;
1634                    continue;
1635                }
1636                tokio::time::sleep(Duration::from_secs(1)).await;
1637                if let Err(error) = context.sweep() {
1638                    warn!(%error, "scope expiry sweep failed");
1639                }
1640            }
1641        });
1642    }
1643
1644    #[cfg(test)]
1645    pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1646        scope_expiry::ScopeExpiryContext::new(self).sweep()
1647    }
1648
1649    fn publish_scope_changes(
1650        &self,
1651        changes: &[crate::scopes::ScopeTagChange],
1652        expired: &[crate::scopes::ScopeExpired],
1653    ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1654        scope_expiry::ScopeExpiryContext::new(self).publish_scope_changes(changes, expired)
1655    }
1656
1657    fn runtime_capability_snapshot(
1658        &self,
1659    ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1660        let runtime = self
1661            .supervisor
1662            .list()
1663            .into_iter()
1664            .map(|module| {
1665                let status = module.status().map_err(|err| {
1666                    RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1667                })?;
1668                Ok(RuntimeModule {
1669                    module_id: status.module_id,
1670                    state: status.state,
1671                    enabled: status.enabled,
1672                })
1673            })
1674            .collect::<Result<Vec<_>, RouterError>>()?;
1675        // Include every registration, published or not. The capability
1676        // evaluator counts a module as providing its capabilities as soon as it
1677        // has registered, even before `catalog.list` shows it, so a HELLO in
1678        // progress does not change other modules' required-capability
1679        // verdicts. See `Registry::list_registered_modules`.
1680        let registrations = self.registry.list_registered_modules().map_err(|err| {
1681            RouterError::backend(
1682                0,
1683                0,
1684                format!("failed to list capability registrations: {err}"),
1685            )
1686        })?;
1687        let registrations = registrations
1688            .into_iter()
1689            .map(|registration| RegisteredModule {
1690                module_id: registration.manifest.module_id,
1691                module_version: registration.manifest.module_version,
1692                capabilities: registration.manifest.capabilities,
1693            })
1694            .collect();
1695        Ok((runtime, registrations))
1696    }
1697
1698    /// The capability side effects of a module becoming the active registration
1699    /// for its id: cache its manifest (logging how it differs from the cached
1700    /// one), run the deny census when its declarations call for one, and recompute the
1701    /// requirement statuses. An ordinary HELLO does this as it registers; a swap
1702    /// candidate's does not, and the supervisor does it at promotion instead,
1703    /// through [`crate::supervise::SwapPromotionObserver`].
1704    fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1705        let cached_registration = RegisteredModule {
1706            module_id: registration.manifest.module_id.clone(),
1707            module_version: registration.manifest.module_version.clone(),
1708            capabilities: registration.manifest.capabilities.clone(),
1709        };
1710        let change = self.capability_evaluator.record_hello(&cached_registration);
1711        log_manifest_change(&cached_registration.module_id, &change);
1712        if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1713            self.enforce_capability_denies();
1714        }
1715        self.refresh_capability_requirements();
1716    }
1717
1718    /// Point the shared supervisor handle at this handler for swap promotions.
1719    /// Called wherever a handler is put behind the `Arc` the router serves, so
1720    /// it can be held weakly.
1721    pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1722        let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1723            Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1724        self.supervisor.set_swap_promotion_observer(observer);
1725    }
1726
1727    pub fn refresh_capability_requirements(&self) {
1728        match self.runtime_capability_snapshot() {
1729            Ok((runtime, registrations)) => {
1730                log_requirement_events(
1731                    self.capability_evaluator
1732                        .evaluate_now(&runtime, &registrations),
1733                    &self.daemon_shutdown,
1734                );
1735            }
1736            Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1737        }
1738    }
1739
1740    /// Reconcile only live, attested route bindings after a capability deny edge
1741    /// or target claim was added. This is deliberately a control-plane census:
1742    /// the opaque forwarding hot path must not grow a per-frame capability check.
1743    fn enforce_capability_denies(&self) {
1744        let registrations = match self.registry.list_registered_modules() {
1745            Ok(snapshot) => snapshot,
1746            Err(err) => {
1747                warn!(error = %err, "failed to read registrations for capability deny census");
1748                return;
1749            }
1750        };
1751        let manifests = registrations
1752            .into_iter()
1753            .map(|registration| {
1754                (
1755                    registration.manifest.module_id.clone(),
1756                    registration.manifest,
1757                )
1758            })
1759            .collect::<BTreeMap<_, _>>();
1760        let census = match self.forwarding.route_census(None) {
1761            Ok(census) => census,
1762            Err(err) => {
1763                warn!(error = %err, "failed to read route census for capability deny enforcement");
1764                return;
1765            }
1766        };
1767
1768        for (target_module_id, routes) in census {
1769            let Some(target_manifest) = manifests.get(&target_module_id) else {
1770                continue;
1771            };
1772            let mut closed_routes = Vec::new();
1773            let mut goodbyes = Vec::new();
1774            for route in routes {
1775                let Principal::Reserved {
1776                    module_id: opening_module_id,
1777                } = &route.principal
1778                else {
1779                    continue;
1780                };
1781                let Some(opening_manifest) = manifests.get(opening_module_id) else {
1782                    continue;
1783                };
1784                let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1785                    continue;
1786                };
1787
1788                match self.forwarding.release_client_route(
1789                    route.goodbye_target.connection_id,
1790                    route.goodbye_target.channel,
1791                    route.goodbye_target.epoch,
1792                ) {
1793                    Ok(RouteRelease::Removed(module_goodbye)) => {
1794                        warn!(
1795                            opening_module_id,
1796                            target_module_id,
1797                            capability,
1798                            "force-closing route because an attested capability deny edge now matches"
1799                        );
1800                        goodbyes.push(module_goodbye);
1801                        goodbyes.push(route.goodbye_target.clone());
1802                        closed_routes.push(route);
1803                    }
1804                    Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1805                    Err(err) => warn!(
1806                        opening_module_id,
1807                        target_module_id,
1808                        capability,
1809                        error = %err,
1810                        "failed to force-close capability-denied route"
1811                    ),
1812                }
1813            }
1814
1815            if closed_routes.is_empty() {
1816                continue;
1817            }
1818            send_route_control_pushes(
1819                &self.forwarding,
1820                closed_routes,
1821                ClientControlPush::RouteClosed {
1822                    module_id: target_module_id,
1823                    channels: Vec::new(),
1824                    reason: RouteCloseReason::CapabilityDenied,
1825                    drained: false,
1826                    abandoned: 0,
1827                    excluded_subscriptions: 0,
1828                    terminal: Some(false),
1829                },
1830            );
1831            // route.closed records the reason; the route GOODBYE ends the
1832            // client's handle and settles any calls still pending on it.
1833            self.emit_route_goodbyes(goodbyes);
1834        }
1835    }
1836
1837    /// Why a registered module is not accepting new route binds, or `None` when
1838    /// it is. This is the module's effective readiness: its declared readiness
1839    /// first, then every `need: required` capability it declares evaluating to
1840    /// `provided`. `route.open` and `catalog.list` both read it here so the
1841    /// catalog never reports a module routable that `route.open` would refuse.
1842    fn not_ready_reason(
1843        &self,
1844        registration: &crate::registry::ModuleRegistration,
1845    ) -> Option<NotReadyReason> {
1846        if !registration.ready {
1847            return Some(NotReadyReason {
1848                reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1849                capability: None,
1850            });
1851        }
1852        self.first_unprovided_required_capability(registration)
1853            .map(|capability| NotReadyReason {
1854                reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1855                capability: Some(capability),
1856            })
1857    }
1858
1859    /// The lexicographically first capability this registration declares
1860    /// `need: required` whose evaluator verdict is not `provided`.
1861    ///
1862    /// The verdicts are the capability evaluator's own; nothing here decides
1863    /// what "provided" means. The evaluator counts a capability provided as
1864    /// soon as a module claiming it has REGISTERED, not once that module is
1865    /// ready. That distinction is what keeps two modules that require each
1866    /// other's capabilities from deadlocking: if "provided" meant "the claimant
1867    /// is ready", each would wait for the other to become ready first and
1868    /// neither ever would. Do not tighten it to readiness.
1869    ///
1870    /// A required capability with no verdict at all means this registration's
1871    /// HELLO or catalog.update landed after the last recompute; recompute once
1872    /// rather than let a missing verdict read as either answer. If it is still
1873    /// missing (the recompute itself failed) the capability counts as
1874    /// unprovided: the refusal is retryable, and routing a module whose
1875    /// required provider is unknown is the outcome this check exists to stop.
1876    fn first_unprovided_required_capability(
1877        &self,
1878        registration: &crate::registry::ModuleRegistration,
1879    ) -> Option<String> {
1880        let required = registration
1881            .manifest
1882            .capabilities
1883            .iter()
1884            .flat_map(|declarations| declarations.requires.iter())
1885            .filter(|requirement| requirement.need == CapabilityNeed::Required)
1886            .map(|requirement| requirement.capability.as_str())
1887            .collect::<BTreeSet<_>>();
1888        if required.is_empty() {
1889            return None;
1890        }
1891        let module_id = registration.manifest.module_id.as_str();
1892        let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1893        if required
1894            .iter()
1895            .any(|capability| verdict(capability).is_none())
1896        {
1897            self.refresh_capability_requirements();
1898        }
1899        required
1900            .into_iter()
1901            .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1902            .map(str::to_string)
1903    }
1904
1905    fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1906        self.capability_evaluator
1907            .statuses()
1908            .into_iter()
1909            .map(capability_requirement_status)
1910            .collect()
1911    }
1912
1913    /// Remove a connection's registry entries WITHOUT signalling the supervisor's
1914    /// registration-release watch. The signal is what the supervisor waits on
1915    /// before spawning a replacement, so it must only fire once forwarding
1916    /// teardown is also done (see [`Self::cleanup_connection`] /
1917    /// [`Self::handle_goodbye`]). Unpublished HELLO rollback uses the separate
1918    /// registry-only path below because publication already owns the spawn lock.
1919    fn deregister_connection(
1920        &self,
1921        connection_id: ConnectionId,
1922        reason: RegistrationEndReason,
1923    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1924        if let Some(spawn) = self.supervisor.registration_spawn(connection_id) {
1925            spawn.end_primary_registration();
1926        }
1927        self.registry
1928            .deregister_connection_with_reason(connection_id, reason)
1929    }
1930
1931    // HELLO publication owns the spawn transition guard. claim.mint is allowed
1932    // only after HELLO acceptance, so a failed first HELLO has no bearer tokens
1933    // or claim-bound consumers to revoke. Remove only its unpublished registry
1934    // entry without re-entering primary-registration teardown.
1935    fn rollback_unpublished_hello(
1936        &self,
1937        connection: ConnectionId,
1938    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1939        self.registry.deregister_connection_with_reason(
1940            connection,
1941            RegistrationEndReason::RegistrationFailed,
1942        )
1943    }
1944
1945    fn request_refusal_close(
1946        &self,
1947        connection: ConnectionId,
1948        sink: Option<&crate::FrameSink>,
1949        reason: CloseReason,
1950    ) {
1951        if let Some(sink) = sink {
1952            sink.terminate_ingress();
1953        }
1954        if self.claims.close_notifications_enabled() {
1955            self.forwarding.request_connection_close(connection, reason);
1956        }
1957    }
1958
1959    pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1960        if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1961            return None;
1962        }
1963        let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1964            parse_client_control_request(&frame.body)
1965        else {
1966            return None;
1967        };
1968        Some(target_module_id(&target).to_string())
1969    }
1970
1971    pub(crate) fn route_open_capacity_refusal(
1972        &self,
1973        ctx: &RouteCtx,
1974        frame: &Frame,
1975        target_module_id: &str,
1976        in_flight: usize,
1977        limit: usize,
1978    ) -> Result<Frame, RouterError> {
1979        // Sent before the request reaches `handle_route_open`, so the caller
1980        // fields come from the raw frame here.
1981        let caller = self.route_open_caller_from_frame(ctx, frame);
1982        self.route_open_admission_refusal_frame(
1983            ctx,
1984            &caller,
1985            frame,
1986            target_module_id,
1987            "open_admission_full",
1988            (in_flight, limit),
1989            format!(
1990                "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1991            ),
1992        )
1993    }
1994
1995    fn route_open_target_capacity_refusal(
1996        &self,
1997        ctx: &RouteCtx,
1998        caller: &RouteOpenCaller,
1999        frame: &Frame,
2000        target_module_id: &str,
2001        in_flight: usize,
2002    ) -> Result<Frame, RouterError> {
2003        self.route_open_admission_refusal_frame(
2004            ctx,
2005            caller,
2006            frame,
2007            target_module_id,
2008            "target_binds_full",
2009            (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
2010            format!(
2011                "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
2012            ),
2013        )
2014    }
2015
2016    /// Admission pressure clears as existing binds settle, so its refusal must
2017    /// remain in the deployed SDKs' closed retryable set: `unknown_module`,
2018    /// `module_reloading`, `module_warming`, `target_unavailable`, or
2019    /// `module_timeout`. `target_unavailable` is honest for an attempt that
2020    /// cannot currently reach its target; `module_timeout` would falsely claim
2021    /// that a wait expired. A new, cleaner code would be terminal to deployed
2022    /// clients, so it requires a client-tolerance rollout before daemon emission.
2023    #[allow(clippy::too_many_arguments)]
2024    fn route_open_admission_refusal_frame(
2025        &self,
2026        ctx: &RouteCtx,
2027        caller: &RouteOpenCaller,
2028        frame: &Frame,
2029        target_module_id: &str,
2030        reason: &'static str,
2031        (in_flight, limit): (usize, usize),
2032        message: impl Into<String>,
2033    ) -> Result<Frame, RouterError> {
2034        let code = error_codes::TARGET_UNAVAILABLE;
2035        self.counters.increment_route_open_refused(code);
2036        info!(
2037            target: "control",
2038            code,
2039            reason,
2040            module_id = ?target_module_id,
2041            connection_id = ctx.connection_id.get(),
2042            principal = %caller.principal,
2043            scope = %caller.scope,
2044            harness = %caller.harness,
2045            in_flight,
2046            limit,
2047            "route.open refused"
2048        );
2049        control_error_frame(frame, code, message.into())
2050    }
2051
2052    /// Test-only compatibility entry point for unit control handling that does not have a socket sink.
2053    ///
2054    /// The real server path uses [`Self::handle_control_frame`] so module HELLO registration can
2055    /// record the module connection's [`crate::FrameSink`] and session attach can await the module
2056    /// relay response. This seam stays cfg(test) so production has only one channel-0 path.
2057    #[cfg(test)]
2058    pub fn handle_control(
2059        &self,
2060        connection_id: ConnectionId,
2061        frame: Frame,
2062    ) -> Result<Vec<Frame>, RouterError> {
2063        match frame.header.ty {
2064            FrameType::Ping => Ok(vec![pong(&frame)?]),
2065            FrameType::Hello => self.handle_hello(connection_id, None, frame),
2066            FrameType::Goodbye => self.handle_goodbye(connection_id),
2067            ty => Ok(vec![control_error_frame(
2068                &frame,
2069                "unsupported_control_frame",
2070                format!("unsupported channel-0 frame {ty:?}"),
2071            )?]),
2072        }
2073    }
2074
2075    pub async fn handle_control_frame(
2076        &self,
2077        ctx: &RouteCtx,
2078        frame: Frame,
2079    ) -> Result<Vec<Frame>, RouterError> {
2080        self.handle_control_frame_timed(ctx, frame, None).await
2081    }
2082
2083    #[cfg(any(test, feature = "test-support"))]
2084    pub fn with_test_operator_app_policy(
2085        mut self,
2086        policy: crate::operator_apps::TestOperatorAppPolicy,
2087    ) -> Self {
2088        self.operator_app_policy = policy.into();
2089        self
2090    }
2091
2092    /// Inject the Windows signature check for tests. The injected check
2093    /// answers the verified version string or the rejection reason; the
2094    /// daemon enforces the floor itself on the returned version.
2095    #[cfg(all(windows, any(test, feature = "test-support")))]
2096    pub fn with_test_windows_verifier(mut self, verify: crate::app_open::WindowsVerifyFn) -> Self {
2097        self.windows_verify = Some(verify);
2098        self
2099    }
2100
2101    /// Record which pids the daemon would flash for `already_running`.
2102    #[cfg(all(windows, any(test, feature = "test-support")))]
2103    pub fn with_test_windows_flash(mut self, flash: crate::app_open::WindowsFlashFn) -> Self {
2104        self.windows_flash = Some(flash);
2105        self
2106    }
2107
2108    /// Inject a tracking failure for tests: makes `track_launched_process`
2109    /// fail so a test can prove a launch that cannot be tracked is stopped,
2110    /// never left running.
2111    #[cfg(all(windows, any(test, feature = "test-support")))]
2112    pub fn with_test_windows_track(mut self, track: crate::app_open::WindowsTrackFn) -> Self {
2113        self.windows_track = Some(track);
2114        self
2115    }
2116
2117    pub(crate) fn record_captured_peer(
2118        &self,
2119        connection: ConnectionId,
2120        peer: subc_os::local_peer::CapturedPeer,
2121    ) {
2122        self.record_observed_peer(connection, peer.observation().clone());
2123        let identity = self.operator_app_policy.config_identity(peer.clone());
2124        #[cfg(windows)]
2125        let identity = identity.or_else(|| self.config_windows_launch_identity(&peer));
2126        if let Some(identity) = identity {
2127            // Keep the connect-time launch incarnation. Looking it up afresh
2128            // from a numeric pipe-client PID could match a later reused PID.
2129            self.config_desktop_stamps
2130                .lock()
2131                .unwrap_or_else(|p| p.into_inner())
2132                .insert(connection, identity);
2133        }
2134        self.captured_peers
2135            .lock()
2136            .unwrap_or_else(|p| p.into_inner())
2137            .insert(connection, peer.clone());
2138        #[cfg(windows)]
2139        if let Some(stamp) = self.stamp_windows_launch(&peer) {
2140            self.operator_apps
2141                .lock()
2142                .unwrap_or_else(|p| p.into_inner())
2143                .insert(connection, stamp);
2144        }
2145        if let Some(stamp) = self.operator_app_policy.stamp(peer) {
2146            self.operator_apps
2147                .lock()
2148                .unwrap_or_else(|p| p.into_inner())
2149                .insert(connection, stamp);
2150        }
2151    }
2152
2153    pub(crate) fn record_local_connection(&self, connection: ConnectionId) {
2154        self.local_connections
2155            .lock()
2156            .unwrap_or_else(|p| p.into_inner())
2157            .insert(connection);
2158    }
2159
2160    // Keep the connection-level delivery decision shared with route.bind. The
2161    // target's capability opt-in is separate from recognition of the caller.
2162    fn connection_operator_app(
2163        &self,
2164        stamp: Option<&crate::operator_apps::OperatorAppStamp>,
2165    ) -> Option<subc_protocol::peer::OperatorApp> {
2166        stamp
2167            .filter(|stamp| stamp.delivery_enabled(&self.operator_app_policy))
2168            .map(|stamp| stamp.app.clone())
2169    }
2170
2171    fn handle_connection_describe(
2172        &self,
2173        ctx: &RouteCtx,
2174        frame: Frame,
2175    ) -> Result<Vec<Frame>, RouterError> {
2176        if serde_json::from_slice::<subc_control::ConnectionDescribeRequest>(&frame.body).is_err() {
2177            return Ok(vec![control_error_frame(
2178                &frame,
2179                "invalid_control_body",
2180                "malformed connection.describe body",
2181            )?]);
2182        }
2183        let local = self
2184            .local_connections
2185            .lock()
2186            .unwrap_or_else(|p| p.into_inner())
2187            .contains(&ctx.connection_id);
2188        let stamps = self.operator_apps.lock().unwrap_or_else(|p| p.into_inner());
2189        let operator_app = self.connection_operator_app(stamps.get(&ctx.connection_id));
2190        Ok(vec![control_response_body_frame(
2191            &frame,
2192            &subc_control::ConnectionDescribeReply::new(local, operator_app),
2193            ops::CONNECTION_DESCRIBE,
2194        )?])
2195    }
2196
2197    pub(crate) fn record_observed_peer(
2198        &self,
2199        connection: ConnectionId,
2200        peer: subc_os::local_peer::ObservedPeer,
2201    ) {
2202        use subc_os::local_peer::{ObservedPeer as OsPeer, PeerUser};
2203        let peer = match peer {
2204            OsPeer::Unobserved => ObservedPeer::unobserved(),
2205            OsPeer::Observed(peer) => {
2206                let observed = match peer.user {
2207                    PeerUser::Uid(uid) => ObservedPeer::observed_uid(uid),
2208                    PeerUser::Sid(sid) => ObservedPeer::observed_sid(sid),
2209                };
2210                observed
2211                    .with_exe(
2212                        peer.exe
2213                            .and_then(|path| path.into_os_string().into_string().ok()),
2214                    )
2215                    .with_signing_id(peer.signing_id)
2216            }
2217        };
2218        self.observed_peers
2219            .lock()
2220            .unwrap_or_else(|p| p.into_inner())
2221            .insert(connection, peer);
2222    }
2223
2224    pub(crate) fn record_caller_endpoints(
2225        &self,
2226        connection: ConnectionId,
2227        endpoints: Option<(SocketAddr, SocketAddr)>,
2228    ) {
2229        let mut callers = self
2230            .caller_endpoints
2231            .lock()
2232            .unwrap_or_else(|p| p.into_inner());
2233        callers.remove(&connection);
2234        if let Some(endpoints) = endpoints {
2235            callers.insert(connection, endpoints);
2236        }
2237    }
2238
2239    #[cfg(test)]
2240    pub(crate) fn with_caller_lookup_for_test(
2241        mut self,
2242        lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
2243    ) -> Self {
2244        self.caller_lookup = Some(Arc::new(lookup));
2245        self
2246    }
2247
2248    async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
2249        let endpoints = self
2250            .caller_endpoints
2251            .lock()
2252            .unwrap_or_else(|p| p.into_inner())
2253            .get(&connection)
2254            .copied();
2255        #[cfg(test)]
2256        let lookup = self.caller_lookup.clone();
2257        tokio::task::spawn_blocking(move || {
2258            #[cfg(test)]
2259            if let Some(lookup) = lookup {
2260                return lookup(endpoints);
2261            }
2262            endpoints
2263                .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
2264                .unwrap_or_default()
2265        })
2266        .await
2267        .unwrap_or_default()
2268    }
2269
2270    pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
2271        frame.header.channel == 0
2272            && frame.header.ty == FrameType::Request
2273            && self
2274                .forwarding
2275                .module_endpoint_for_connection(ctx.connection_id)
2276                .is_ok_and(|endpoint| endpoint.is_none())
2277            && parse_client_control_request(&frame.body)
2278                .ok()
2279                .is_some_and(|request| privileged_operation(&request).is_some())
2280    }
2281
2282    pub(crate) async fn handle_control_frame_timed(
2283        &self,
2284        ctx: &RouteCtx,
2285        frame: Frame,
2286        dispatch_started_at: Option<StdInstant>,
2287    ) -> Result<Vec<Frame>, RouterError> {
2288        let first = ctx.egress.first_ingress_frame();
2289        self.handle_control_frame_ingress(ctx, frame, dispatch_started_at, first)
2290            .await
2291    }
2292
2293    pub(crate) async fn handle_control_frame_ingress(
2294        &self,
2295        ctx: &RouteCtx,
2296        frame: Frame,
2297        dispatch_started_at: Option<StdInstant>,
2298        first: bool,
2299    ) -> Result<Vec<Frame>, RouterError> {
2300        if ctx.egress.ingress_terminal() {
2301            return Ok(Vec::new());
2302        }
2303        #[cfg(test)]
2304        self.claims.record_dispatch(frame.header.corr);
2305        let claim_binding = self.claims.binding(ctx.connection_id);
2306        let mut prefix_attempt = (frame.header.ty == FrameType::Hello
2307            && claim_binding
2308                .as_ref()
2309                .is_some_and(|binding| binding.prefix_module_id().is_some()))
2310        .then(|| self.prefix_hello_attempt(ctx));
2311        if claim_binding
2312            .as_ref()
2313            .is_some_and(|binding| binding.prefix_module_id().is_some())
2314        {
2315            let registered = self
2316                .forwarding
2317                .module_endpoint_for_connection(ctx.connection_id)
2318                .map_err(RouterError::Forwarding)?
2319                .is_some();
2320            if !registered && frame.header.ty != FrameType::Hello {
2321                return self
2322                    .refuse_claim_connection(ctx, &frame, "claim_hello_required")
2323                    .await;
2324            }
2325            if registered && frame.header.ty == FrameType::Goodbye {
2326                ctx.egress.terminate_ingress();
2327                let result = self.handle_goodbye(ctx.connection_id);
2328                self.claims.connection_closed(ctx.connection_id);
2329                self.request_refusal_close(
2330                    ctx.connection_id,
2331                    Some(&ctx.egress),
2332                    CloseReason::new("claim_registration_ended", "prefix registration ended"),
2333                );
2334                return result;
2335            }
2336            if registered && frame.header.ty == FrameType::Request {
2337                let allowed =
2338                    serde_json::from_slice::<ControlOpProbe>(&frame.body).is_ok_and(|probe| {
2339                        matches!(
2340                            probe.op.as_str(),
2341                            "catalog.update"
2342                                | "catalog.update_role_declarations"
2343                                | "scope.describe"
2344                                | "supervisor.live_roots"
2345                                | "activity.record"
2346                        )
2347                    });
2348                if !allowed {
2349                    return Ok(vec![control_error_frame(
2350                        &frame,
2351                        channel_operations::CLAIM_OPERATION_NOT_ALLOWED,
2352                        "prefix registrations have only brokered module operations",
2353                    )?]);
2354                }
2355            }
2356            if registered
2357                && frame.header.ty == FrameType::Push
2358                && !serde_json::from_slice::<ControlOpProbe>(&frame.body)
2359                    .is_ok_and(|probe| probe.op == "route.status")
2360            {
2361                return Ok(vec![control_error_frame(
2362                    &frame,
2363                    channel_operations::CLAIM_OPERATION_NOT_ALLOWED,
2364                    "prefix registrations may push only route.status for routes they serve",
2365                )?]);
2366            }
2367        }
2368        match frame.header.ty {
2369            FrameType::Ping => Ok(vec![pong(&frame)?]),
2370            FrameType::Hello => {
2371                if claim_binding
2372                    .as_ref()
2373                    .is_some_and(|binding| binding.prefix_module_id().is_none())
2374                {
2375                    return self
2376                        .refuse_claim_connection(ctx, &frame, "claim_hello_not_allowed")
2377                        .await;
2378                }
2379                // Resolved once, here: the blocking HELLO below may outlive this
2380                // reader and run after cleanup has forgotten the channel, and it
2381                // must still be judged as this channel's HELLO.
2382                let channel_spawn = self.supervisor.registration_spawn(ctx.connection_id);
2383                if let Some(spawn) = &channel_spawn {
2384                    spawn.wait_launch().await;
2385                    // Refused launches close the channel. Leave their pending
2386                    // HELLO unanswered instead of racing a reply against shutdown.
2387                    if !spawn.launch_ready() {
2388                        ctx.egress.terminate_ingress();
2389                        return Ok(Vec::new());
2390                    }
2391                    #[cfg(test)]
2392                    self.pause_hello_ingress_for_test().await;
2393                } else if claim_binding.is_none() {
2394                    if let Ok(hello) = serde_json::from_slice::<serde_json::Value>(&frame.body) {
2395                        if let (Some(module), Some(nonce)) = (
2396                            hello
2397                                .get("manifest")
2398                                .and_then(|value| value.get("module_id"))
2399                                .and_then(|value| value.as_str()),
2400                            hello.get("launch_nonce").and_then(|value| value.as_str()),
2401                        ) {
2402                            if !self
2403                                .config_runtime
2404                                .as_ref()
2405                                .is_some_and(|owner| owner.module_is_gated(module))
2406                                && !self.supervisor.wait_program_binding(module, nonce).await
2407                            {
2408                                self.request_refusal_close(
2409                                    ctx.connection_id,
2410                                    Some(&ctx.egress),
2411                                    CloseReason::new(
2412                                        "program_refused",
2413                                        "the spawned program binding failed",
2414                                    ),
2415                                );
2416                                return Ok(vec![control_error_frame(
2417                                    &frame,
2418                                    if self
2419                                        .config_runtime
2420                                        .as_ref()
2421                                        .is_some_and(|owner| owner.is_recovering())
2422                                    {
2423                                        "config_gated"
2424                                    } else {
2425                                        "program_refused"
2426                                    },
2427                                    "the spawned program binding failed",
2428                                )?]);
2429                            }
2430                        }
2431                    }
2432                }
2433                let handler = self.clone();
2434                let connection = ctx.connection_id;
2435                let egress = ctx.egress.clone();
2436                let correlation = frame.header.corr;
2437                let dispatcher = tracing::dispatcher::get_default(|dispatcher| dispatcher.clone());
2438                let span = tracing::Span::current();
2439                let hello_binding = claim_binding.clone();
2440                let replies = tokio::task::spawn_blocking(move || {
2441                    tracing::dispatcher::with_default(&dispatcher, || {
2442                        span.in_scope(|| {
2443                            handler.handle_hello_checked(
2444                                connection,
2445                                Some(egress),
2446                                frame,
2447                                true,
2448                                hello_binding,
2449                                channel_spawn,
2450                            )
2451                        })
2452                    })
2453                })
2454                .await
2455                .map_err(|error| RouterError::backend(0, correlation, error.to_string()))??;
2456                #[cfg(test)]
2457                if prefix_attempt.is_some() {
2458                    self.claims.pause_prefix_commit_for_test(false).await;
2459                }
2460                if let Some(attempt) = &mut prefix_attempt {
2461                    let accepted = !replies
2462                        .iter()
2463                        .any(|reply| reply.header.ty == FrameType::Error)
2464                        && self
2465                            .registry
2466                            .get_module_by_connection(ctx.connection_id)
2467                            .map_err(|err| RouterError::backend(0, correlation, err.to_string()))?
2468                            .is_some();
2469                    if accepted {
2470                        attempt.accepted = true;
2471                    } else {
2472                        ctx.egress.terminate_ingress();
2473                        for reply in replies {
2474                            ctx.egress.send_flushed(reply).await?;
2475                        }
2476                        return Ok(Vec::new());
2477                    }
2478                }
2479                Ok(replies)
2480            }
2481            FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
2482            FrameType::Cancel => {
2483                // Channel-0 cancellation withdraws approvals and subscriptions
2484                // and fences pending agent declarations;
2485                // successful cancellation has no ordinary reply.
2486                // A config Apply may also own an OS prompt. Cancel that prompt
2487                // before config cancellation can report the request handled.
2488                if self
2489                    .agent_declarations
2490                    .cancel(ctx.connection_id, frame.header.corr)
2491                    || self
2492                        .forwarding
2493                        .operator_confirms()
2494                        .cancel(ctx.connection_id, frame.header.corr)
2495                    || self.cancel_config_approval(ctx.connection_id, frame.header.corr)
2496                    || self
2497                        .supervisor
2498                        .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
2499                    || self.cancel_activity_subscription(ctx, &frame)?
2500                    || self.cancel_catalog_subscription(ctx.connection_id, frame.header.corr)
2501                {
2502                    Ok(Vec::new())
2503                } else {
2504                    Ok(vec![control_error_frame(
2505                        &frame,
2506                        "unknown_subscription",
2507                        "no subscription has this correlation id",
2508                    )?])
2509                }
2510            }
2511            FrameType::Request => {
2512                // This additive operation is not part of the existing exhaustive
2513                // module-control enum. Probe the op before decoding that enum.
2514                let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
2515                let claim_present = op.as_ref().is_some_and(|probe| probe.op == "claim.present");
2516                if let Some(refusal) = self
2517                    .recovery_request_refusal(
2518                        ctx,
2519                        &frame,
2520                        op.as_ref().map(|probe| probe.op.as_str()),
2521                    )
2522                    .await?
2523                {
2524                    if claim_present {
2525                        self.reclaim_presented_expiry(&frame);
2526                        return self
2527                            .refuse_claim_connection(ctx, &frame, "config_gated")
2528                            .await;
2529                    }
2530                    return Ok(vec![refusal]);
2531                }
2532                if claim_present {
2533                    return self.handle_claim_present(ctx, &frame, first).await;
2534                }
2535                if op.as_ref().is_some_and(|probe| probe.op == "claim.mint") {
2536                    return self.handle_claim_mint(ctx, &frame).await;
2537                }
2538                if let Some(refusal) = self.registration_operation_refusal(
2539                    ctx,
2540                    &frame,
2541                    op.as_ref().map(|probe| probe.op.as_str()),
2542                )? {
2543                    return Ok(vec![refusal]);
2544                }
2545                if op
2546                    .as_ref()
2547                    .is_some_and(|probe| probe.op == ops::CATALOG_WATCH)
2548                {
2549                    return self.handle_catalog_watch(ctx, frame);
2550                }
2551                if op
2552                    .as_ref()
2553                    .is_some_and(|probe| probe.op == MODULE_TO_SUBC_OP_CATALOG_UPDATE)
2554                    && subc_protocol::provider_declarations::catalog_update_has_provider_declarations(&frame.body)
2555                        .unwrap_or(false)
2556                {
2557                    return Ok(vec![control_error_frame(
2558                        &frame,
2559                        error_codes::CATALOG_UPDATE_FROZEN_DECLARATIONS,
2560                        "provider_declarations are fixed at HELLO; restart the module to change them",
2561                    )?]);
2562                }
2563                if let Some(probe) = op
2564                    .as_ref()
2565                    .filter(|probe| config_gate::is_module_config_op(&probe.op))
2566                {
2567                    return self
2568                        .handle_module_config_request(ctx, frame, &probe.op)
2569                        .await;
2570                }
2571                if op
2572                    .as_ref()
2573                    .is_some_and(|probe| config_gate::is_config_op(&probe.op))
2574                {
2575                    return self.handle_config_gate_request(ctx, frame).await;
2576                }
2577                if op
2578                    .as_ref()
2579                    .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
2580                {
2581                    return self.handle_activity_record(ctx, frame).await;
2582                }
2583                if op.as_ref().is_some_and(|probe| {
2584                    probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
2585                }) {
2586                    return self.handle_activity_read(ctx, frame).await;
2587                }
2588                if op
2589                    .as_ref()
2590                    .is_some_and(|probe| probe.op == "operator.confirm")
2591                {
2592                    return self.handle_operator_confirm(ctx, frame);
2593                }
2594                if op.as_ref().is_some_and(|probe| probe.op == ops::APP_OPEN) {
2595                    return self
2596                        .observe_control_wait(
2597                            ctx,
2598                            None,
2599                            ops::APP_OPEN,
2600                            self.handle_app_open(ctx, frame),
2601                        )
2602                        .await;
2603                }
2604                if op
2605                    .as_ref()
2606                    .is_some_and(|probe| probe.op == ops::CONNECTION_DESCRIBE)
2607                {
2608                    return self.handle_connection_describe(ctx, frame);
2609                }
2610                if op
2611                    .as_ref()
2612                    .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
2613                {
2614                    return self.handle_scope_served_by(ctx.connection_id, frame);
2615                }
2616                if self
2617                    .forwarding
2618                    .module_endpoint_for_connection(ctx.connection_id)
2619                    .map_err(RouterError::Forwarding)?
2620                    .is_some()
2621                {
2622                    if !is_known_module_request_op(&frame.body) {
2623                        return Ok(vec![control_error_frame(
2624                            &frame,
2625                            "unsupported_control_frame",
2626                            "module-originated channel-0 REQUEST is not supported",
2627                        )?]);
2628                    }
2629                    let request = match parse_module_control_request_from_module(&frame.body) {
2630                        Ok(request) => request,
2631                        Err((err, ControlRequestBodyError::UnknownOp)) => {
2632                            return Ok(vec![control_error_frame(
2633                                &frame,
2634                                "unsupported_control_frame",
2635                                format!("unsupported module-originated channel-0 REQUEST: {err}"),
2636                            )?])
2637                        }
2638                        Err((err, ControlRequestBodyError::InvalidBody)) => {
2639                            return Ok(vec![control_error_frame(
2640                                &frame,
2641                                "invalid_control_body",
2642                                format!("malformed module control body: {err}"),
2643                            )?])
2644                        }
2645                    };
2646                    let op = module_control_request_op(&request);
2647                    let corr = frame.header.corr;
2648                    log_control_dispatch_arrival(op, ctx.connection_id, corr);
2649                    if let ModuleControlRequestFromModule::AgentCommands {
2650                        started,
2651                        finished,
2652                        known,
2653                    } = request
2654                    {
2655                        return self
2656                            .agent_declarations
2657                            .enqueue(
2658                                self,
2659                                ctx,
2660                                frame,
2661                                crate::resource_totals::Declaration {
2662                                    started,
2663                                    finished,
2664                                    known,
2665                                },
2666                            )
2667                            .await;
2668                    }
2669                    let result =
2670                        self.handle_module_control_request(ctx.connection_id, frame, request);
2671                    log_slow_control_dispatch(
2672                        dispatch_started_at,
2673                        op,
2674                        ctx.connection_id,
2675                        corr,
2676                        None,
2677                    );
2678                    return result;
2679                }
2680
2681                if is_known_module_request_op(&frame.body) {
2682                    let is_agents = serde_json::from_slice::<ControlOpProbe>(&frame.body)
2683                        .is_ok_and(|probe| probe.op == "supervisor.agent_commands");
2684                    let code = if is_agents {
2685                        "not_supervised"
2686                    } else if op
2687                        .as_ref()
2688                        .is_some_and(|probe| probe.op == subc_protocol::scope::ROUTE_AUTHORIZE_OP)
2689                    {
2690                        error_codes::ROUTE_AUTHORIZE_NOT_PERMITTED
2691                    } else {
2692                        "not_registered"
2693                    };
2694                    return Ok(vec![control_error_frame(
2695                        &frame,
2696                        code,
2697                        "module-originated control requests require the module's own registered connection",
2698                    )?]);
2699                }
2700
2701                let request = match parse_client_control_request(&frame.body) {
2702                    Ok(request) => request,
2703                    Err((err, ControlRequestBodyError::UnknownOp)) => {
2704                        return Ok(vec![control_error_frame(
2705                            &frame,
2706                            "unknown_control_op",
2707                            format!("unknown client control op: {err}"),
2708                        )?])
2709                    }
2710                    Err((err, ControlRequestBodyError::InvalidBody)) => {
2711                        return Ok(vec![control_error_frame(
2712                            &frame,
2713                            "invalid_control_body",
2714                            format!("malformed client control body: {err}"),
2715                        )?])
2716                    }
2717                };
2718                let op = client_control_request_op(&request);
2719                let corr = frame.header.corr;
2720                log_control_dispatch_arrival(op, ctx.connection_id, corr);
2721                let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
2722                #[cfg(test)]
2723                if let Some(delay) = self.control_dispatch_delay {
2724                    tokio::time::sleep(delay).await;
2725                }
2726                let result = self
2727                    .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
2728                    .await;
2729                if let Some(timings) = route_open_timings.as_mut() {
2730                    timings.finish();
2731                }
2732                log_slow_control_dispatch(
2733                    dispatch_started_at,
2734                    op,
2735                    ctx.connection_id,
2736                    corr,
2737                    route_open_timings.as_ref(),
2738                );
2739                result
2740            }
2741            FrameType::Push => {
2742                let Some(endpoint) = self
2743                    .forwarding
2744                    .module_endpoint_for_connection(ctx.connection_id)
2745                    .map_err(RouterError::Forwarding)?
2746                else {
2747                    return Ok(vec![control_error_frame(
2748                        &frame,
2749                        "unsupported_control_frame",
2750                        "client-originated channel-0 PUSH is not supported",
2751                    )?]);
2752                };
2753                self.handle_status_update(endpoint, frame)
2754            }
2755            FrameType::Response | FrameType::Error
2756                if self
2757                    .forwarding
2758                    .module_endpoint_for_connection(ctx.connection_id)
2759                    .map_err(RouterError::Forwarding)?
2760                    .is_some() =>
2761            {
2762                self.handle_module_relay_response(ctx.connection_id, frame)
2763            }
2764            ty => Ok(vec![control_error_frame(
2765                &frame,
2766                "unsupported_control_frame",
2767                format!("unsupported channel-0 frame {ty:?}"),
2768            )?]),
2769        }
2770    }
2771
2772    #[cfg(test)]
2773    fn pause_cleanup_for_test(&self, stage: u8) {
2774        let mut slot = self.cleanup_pause.lock().unwrap_or_else(|p| p.into_inner());
2775        if slot.as_ref().is_some_and(|(at, _)| *at == stage) {
2776            let (_, (reached, release)) = slot.take().unwrap();
2777            drop(slot);
2778            let _ = reached.send(());
2779            let _ = release.recv_timeout(Duration::from_secs(5));
2780        }
2781    }
2782
2783    /// Run `hook` once, in the next `route.open` that attests a launch nonce,
2784    /// right after the nonce is resolved.
2785    #[cfg(all(test, unix, feature = "test-support"))]
2786    pub(crate) fn after_nonce_attestation_for_test(&self, hook: impl FnOnce() + Send + 'static) {
2787        *self
2788            .after_nonce_attestation
2789            .lock()
2790            .unwrap_or_else(|p| p.into_inner()) = Some(Box::new(hook));
2791    }
2792
2793    #[cfg(test)]
2794    pub(super) fn pause_after_nonce_attestation_for_test(&self) {
2795        let hook = self
2796            .after_nonce_attestation
2797            .lock()
2798            .unwrap_or_else(|p| p.into_inner())
2799            .take();
2800        if let Some(hook) = hook {
2801            hook();
2802        }
2803    }
2804
2805    #[cfg(test)]
2806    async fn pause_hello_ingress_for_test(&self) {
2807        let pause = self
2808            .hello_ingress_pause
2809            .lock()
2810            .unwrap_or_else(|p| p.into_inner())
2811            .take();
2812        if let Some((reached, release)) = pause {
2813            let _ = reached.send(());
2814            let _ = release.await;
2815        }
2816    }
2817
2818    pub fn cleanup_connection(
2819        &self,
2820        connection_id: ConnectionId,
2821    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2822        // The server loop joins its offloaded request tasks (route opens,
2823        // privileged requests and activity reads) before this cleanup runs, and
2824        // inline dispatch has ended. Two kinds of work are not joined. Agent
2825        // command declarations run on their queue's detached worker, so the
2826        // queue is closed first: every declaration still queued or validating
2827        // has its commit fence cancelled before anything below can change how
2828        // the connection resolves. A HELLO whose reader was dropped keeps
2829        // running on its blocking thread. It carries the channel spawn its
2830        // reader resolved at ingress instead of re-reading the index forgotten
2831        // below, so it is still judged as that channel's HELLO: the retired
2832        // spawn refuses it before it records anything, and a channel HELLO
2833        // never records a launch nonce, so it cannot leave this connection any
2834        // nonce authority.
2835        self.agent_declarations.close(connection_id);
2836        self.close_catalog_subscriptions(connection_id);
2837        #[cfg(test)]
2838        self.pause_cleanup_for_test(0);
2839        if let Some(spawn) = self.supervisor.registration_spawn(connection_id) {
2840            spawn.retire_registration_channel();
2841        }
2842        self.supervisor
2843            .forget_registration_connection(connection_id);
2844        #[cfg(test)]
2845        self.pause_cleanup_for_test(1);
2846        self.local_connections
2847            .lock()
2848            .unwrap_or_else(|p| p.into_inner())
2849            .remove(&connection_id);
2850        self.caller_endpoints
2851            .lock()
2852            .unwrap_or_else(|p| p.into_inner())
2853            .remove(&connection_id);
2854        self.build_class_warned
2855            .lock()
2856            .unwrap_or_else(|p| p.into_inner())
2857            .remove(&connection_id);
2858        self.observed_peers
2859            .lock()
2860            .unwrap_or_else(|p| p.into_inner())
2861            .remove(&connection_id);
2862        self.captured_peers
2863            .lock()
2864            .unwrap_or_else(|p| p.into_inner())
2865            .remove(&connection_id);
2866        self.config_desktop_stamps
2867            .lock()
2868            .unwrap_or_else(|p| p.into_inner())
2869            .remove(&connection_id);
2870        self.operator_apps
2871            .lock()
2872            .unwrap_or_else(|p| p.into_inner())
2873            .remove(&connection_id);
2874        self.forwarding
2875            .operator_confirms()
2876            .connection_closed(connection_id);
2877        let cleanup = self.cleanup_connection_with_end_reason(
2878            connection_id,
2879            RegistrationEndReason::ConnectionClosed,
2880        );
2881        // A draining writer can still be reached by a route until forwarding
2882        // cleanup finishes. Keep its claim sink visible to launch retirement.
2883        self.claims.connection_closed(connection_id);
2884        cleanup
2885    }
2886
2887    fn cleanup_connection_with_end_reason(
2888        &self,
2889        connection_id: ConnectionId,
2890        requested_reason: RegistrationEndReason,
2891    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2892        #[cfg(test)]
2893        if let Some(pause) = self
2894            .activity_request_pause
2895            .lock()
2896            .unwrap()
2897            .as_ref()
2898            .filter(|pause| pause.connection == connection_id)
2899        {
2900            pause
2901                .cleanup_started
2902                .store(true, std::sync::atomic::Ordering::Release);
2903        }
2904        if let Some(runtime) = &self.config_runtime {
2905            runtime.disconnected(connection_id);
2906        }
2907        self.agent_declarations.close(connection_id);
2908        let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2909            self.registry
2910                .get_module_by_connection(connection_id)?
2911                .and_then(|registration| self.supervisor.get(&registration.manifest.module_id))
2912                .and_then(|module| module.registration_end_reason().ok().flatten())
2913                .unwrap_or(requested_reason)
2914        } else {
2915            requested_reason
2916        };
2917        let crash_closed = self
2918            .registry
2919            .get_module_by_connection(connection_id)?
2920            .and_then(|registration| {
2921                self.forwarding
2922                    .module_endpoint_for_connection(connection_id)
2923                    .ok()
2924                    .flatten()
2925                    .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2926                    .map(|routes| (registration.manifest.module_id, routes))
2927            });
2928        let crash_closed = crash_closed.map(|(module_id, routes)| {
2929            let terminal = match self.supervisor.get(&module_id) {
2930                None => false,
2931                Some(module) => match module.will_recover_after_connection_loss() {
2932                    Ok(will_recover) => !will_recover,
2933                    Err(err) => {
2934                        warn!(
2935                            %module_id,
2936                            error = %err,
2937                            "failed to read crash recovery verdict; reporting non-terminal conservatively"
2938                        );
2939                        false
2940                    }
2941                },
2942            };
2943            // The forwarding table gates all providers at the start of daemon
2944            // shutdown, before their connections are closed. An ordinary
2945            // module disconnect still reports crash if that gate is not set.
2946            let (reason, drained) = match self
2947                .forwarding
2948                .daemon_drain_result_for_connection(connection_id)
2949            {
2950                Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2951                Ok(None) => (RouteCloseReason::Crash, false),
2952                Err(err) => {
2953                    warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2954                    (RouteCloseReason::Crash, false)
2955                }
2956            };
2957            (module_id, routes, reason, drained, terminal)
2958        });
2959        let registrations = self.deregister_connection(connection_id, end_reason);
2960        // Read before teardown: the spawn whose registration this connection
2961        // was. Its registration ending is an authority change for it.
2962        let ended_authority = self
2963            .forwarding
2964            .target_authority_for_connection(connection_id)
2965            .ok()
2966            .flatten();
2967        let cleanup = if crash_closed.is_some() {
2968            self.forwarding.cleanup_connection_counted(connection_id)
2969        } else {
2970            // A module connection's teardown needs the count of abandoned
2971            // route.bind relays for its route.closed notice below. Any other
2972            // connection, such as a client's, sends no such notice and needs
2973            // only its routes released, so it uses the route-only wrapper and
2974            // reports zero.
2975            self.forwarding
2976                .cleanup_connection(connection_id)
2977                .map(|released| crate::forwarding::ConnectionCleanup {
2978                    released,
2979                    abandoned_relays: 0,
2980                })
2981        };
2982        // The route.closed push waits for forwarding teardown because only
2983        // teardown knows how many pending route.bind relays it aborted. It still
2984        // goes out before the GOODBYEs for the released routes, and its targets
2985        // were captured above, before teardown removed those routes.
2986        if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2987            let abandoned = cleanup
2988                .as_ref()
2989                .map_or(0, |cleanup| cleanup.abandoned_relays);
2990            send_route_control_pushes(
2991                &self.forwarding,
2992                routes,
2993                ClientControlPush::RouteClosed {
2994                    module_id,
2995                    channels: Vec::new(),
2996                    reason,
2997                    drained,
2998                    abandoned,
2999                    excluded_subscriptions: 0,
3000                    terminal: Some(terminal),
3001                },
3002            );
3003        }
3004        if let Ok(cleanup) = cleanup {
3005            self.emit_route_goodbyes(cleanup.released);
3006        }
3007        if let Some(record) = ended_authority {
3008            self.change_spawn_authority(&record);
3009        }
3010        // Signal the registration-release watch only now that BOTH registry and
3011        // forwarding teardown are done, so a supervisor waiting to spawn a
3012        // replacement never observes release while old routes still exist.
3013        if matches!(&registrations, Ok(r) if !r.is_empty()) {
3014            crate::supervise::notify_registration_release();
3015            self.capability_evaluator.wake_deadline_loop();
3016            self.refresh_capability_requirements();
3017        }
3018        self.supervisor.remove_spawn_subscribers(connection_id);
3019        self.activity_streams
3020            .lock()
3021            .unwrap_or_else(|p| p.into_inner())
3022            .retain(|(id, _), task| {
3023                if *id == connection_id {
3024                    task.abort();
3025                    false
3026                } else {
3027                    true
3028                }
3029            });
3030        // Sync authority dies with its connection, so the owner's next
3031        // connection can take it; the owner's scopes stay as they are.
3032        self.hello_launch_nonces
3033            .lock()
3034            .unwrap_or_else(|poisoned| poisoned.into_inner())
3035            .forget(connection_id);
3036        self.scopes
3037            .write()
3038            .unwrap_or_else(|poisoned| poisoned.into_inner())
3039            .release_connection(connection_id);
3040        registrations
3041    }
3042
3043    pub(crate) fn handle_route_goodbye(
3044        &self,
3045        connection_id: ConnectionId,
3046        route_channel: u16,
3047        route_epoch: u32,
3048    ) -> Result<bool, RouterError> {
3049        debug!(
3050            connection_id = connection_id.get(),
3051            route_channel, route_epoch, "handling route GOODBYE"
3052        );
3053        let RouteRelease::Removed(released_route) = self
3054            .forwarding
3055            .release_client_route(connection_id, route_channel, route_epoch)
3056            .map_err(RouterError::Forwarding)?
3057        else {
3058            return Ok(false);
3059        };
3060        self.emit_route_goodbyes(vec![released_route]);
3061        Ok(true)
3062    }
3063
3064    fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
3065        scope_expiry::ScopeExpiryContext::new(self).emit_route_goodbyes(released_routes);
3066    }
3067
3068    /// Best-effort GOODBYE to a module for a route channel subc reserved but then
3069    /// abandoned (route.bind relay timed out, its waiter was cancelled, or subc's
3070    /// own commit failed after the module had already accepted). Without this, a
3071    /// module that accepts late keeps a binding subc has torn down, so a later
3072    /// frame on that module channel could misdeliver if the channel is reused.
3073    ///
3074    /// Never closes the shared module connection on failure: a dropped notification
3075    /// only wastes a bounded amount of warm module-side state, which the module's
3076    /// own idle reaper reclaims. Only call this once the route.bind relay was
3077    /// actually enqueued to the module — if the relay send itself failed, the
3078    /// module never created a binding and there is nothing to tear down.
3079    fn send_abandoned_route_bind_goodbye(
3080        &self,
3081        module_sink: &crate::FrameSink,
3082        negotiated_ver: u8,
3083        module_channel: u16,
3084        module_epoch: u32,
3085    ) {
3086        let frame = match Frame::build_with_version(
3087            negotiated_ver,
3088            FrameType::Goodbye,
3089            control_flags(),
3090            module_channel,
3091            module_epoch,
3092            0,
3093            Vec::new(),
3094        ) {
3095            Ok(frame) => frame,
3096            Err(err) => {
3097                warn!(
3098                    route_channel = module_channel,
3099                    error = %err,
3100                    "failed to build GOODBYE for abandoned route.bind"
3101                );
3102                return;
3103            }
3104        };
3105        crate::forwarding::send_module_route_goodbye(
3106            &self.counters,
3107            module_sink,
3108            frame,
3109            None,
3110            "abandoned route.bind",
3111        );
3112    }
3113
3114    #[cfg(test)]
3115    fn handle_hello(
3116        &self,
3117        connection_id: ConnectionId,
3118        egress: Option<crate::router::FrameSink>,
3119        frame: Frame,
3120    ) -> Result<Vec<Frame>, RouterError> {
3121        let binding = self.claims.binding(connection_id);
3122        let channel_spawn = self.supervisor.registration_spawn(connection_id);
3123        self.handle_hello_checked(connection_id, egress, frame, false, binding, channel_spawn)
3124    }
3125
3126    fn handle_hello_checked(
3127        &self,
3128        connection_id: ConnectionId,
3129        sink: Option<crate::FrameSink>,
3130        frame: Frame,
3131        wait_for_gate: bool,
3132        prefix_binding: Option<claims::ClaimBinding>,
3133        channel_spawn: Option<Arc<crate::supervise::RegistrationSpawn>>,
3134    ) -> Result<Vec<Frame>, RouterError> {
3135        let primary_sink = channel_spawn.as_ref().and_then(|_| sink.clone());
3136        #[cfg(test)]
3137        if prefix_binding.is_some() {
3138            self.claims.pause_prefix_hello_for_test(5);
3139            match self.claims.take_prefix_hello_failure_for_test() {
3140                Some("error") => {
3141                    return Err(RouterError::backend(
3142                        0,
3143                        frame.header.corr,
3144                        "injected HELLO failure",
3145                    ))
3146                }
3147                Some("panic") => panic!("injected HELLO task panic"),
3148                _ => {}
3149            }
3150        }
3151        if let Some(binding) = &prefix_binding {
3152            let value = serde_json::from_slice::<serde_json::Value>(&frame.body).ok();
3153            let module = value
3154                .as_ref()
3155                .and_then(|value| value["manifest"]["module_id"].as_str());
3156            if !binding.admit_prefix_hello(module)
3157                || !self.prefix_binding_live(binding, connection_id)
3158                || !module.is_some_and(|module| {
3159                    self.supervisor
3160                        .prefix_claim_authorized(&binding.registration.module_id, module)
3161                })
3162                || binding.registration.check().is_err()
3163            {
3164                return Ok(vec![control_error_frame(
3165                    &frame,
3166                    "claim_prefix_refused",
3167                    "HELLO must name the prefix claim's exact module id once",
3168                )?]);
3169            }
3170        }
3171        struct RefuseUnregisteredChannel {
3172            spawn: Option<Arc<crate::supervise::RegistrationSpawn>>,
3173            sink: Option<crate::FrameSink>,
3174        }
3175        impl Drop for RefuseUnregisteredChannel {
3176            fn drop(&mut self) {
3177                if let Some(spawn) = &self.spawn {
3178                    if !spawn.registered() {
3179                        if let Some(sink) = &self.sink {
3180                            sink.terminate_ingress();
3181                        }
3182                        spawn.retire_registration_channel();
3183                    }
3184                }
3185            }
3186        }
3187        let _channel_attempt = RefuseUnregisteredChannel {
3188            spawn: channel_spawn.clone(),
3189            sink: sink.clone(),
3190        };
3191        if let Some(spawn) = &channel_spawn {
3192            let value = serde_json::from_slice::<serde_json::Value>(&frame.body).ok();
3193            if let Err(code) = spawn.admit_hello(
3194                value
3195                    .as_ref()
3196                    .and_then(|value| value["manifest"]["module_id"].as_str()),
3197            ) {
3198                return Ok(vec![control_error_frame(
3199                    &frame,
3200                    code,
3201                    "registration channel HELLO refused",
3202                )?]);
3203            }
3204        }
3205        debug!(
3206            connection_id = connection_id.get(),
3207            corr = frame.header.corr,
3208            "handling HELLO"
3209        );
3210        // A module connection has one identity for its entire lifetime. A second
3211        // registration would leave the old registry owner behind while replacing
3212        // its forwarding endpoint and launch nonce.
3213        if self
3214            .registry
3215            .get_module_by_connection(connection_id)
3216            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3217            .is_some()
3218        {
3219            return Ok(vec![control_error_frame(
3220                &frame,
3221                "invalid_hello",
3222                "connection is already registered as a module",
3223            )?]);
3224        }
3225        let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
3226            Ok(value) => value,
3227            Err(err) => {
3228                return Ok(vec![control_error_frame(
3229                    &frame,
3230                    "invalid_hello",
3231                    format!("malformed HELLO body: {err}"),
3232                )?])
3233            }
3234        };
3235        if let Err(detail) = catalog::validate_hello_declarations(&hello_value) {
3236            return Ok(vec![control_error_frame(
3237                &frame,
3238                error_codes::INVALID_PROVIDER_DECLARATIONS,
3239                detail,
3240            )?]);
3241        }
3242        if let Some(module) = hello_value
3243            .get("manifest")
3244            .and_then(|value| value.get("module_id"))
3245            .and_then(|value| value.as_str())
3246        {
3247            let recovering = self
3248                .config_runtime
3249                .as_ref()
3250                .is_some_and(|owner| owner.check_recovery().unwrap_or(true));
3251            // Defence in depth: RuntimeOwner::module_is_gated in
3252            // config_gate/admission.rs already gates every id during recovery.
3253            // A registration channel still never uses the nonce fallback.
3254            if recovering
3255                && channel_spawn.as_ref().is_none_or(|spawn| {
3256                    !self.supervisor.has_live_verified_channel_binding(spawn)
3257                        || spawn.check().is_err()
3258                })
3259                && !(channel_spawn.is_none()
3260                    && hello_value
3261                        .get("launch_nonce")
3262                        .and_then(|value| value.as_str())
3263                        .is_some_and(|nonce| {
3264                            self.supervisor.spawned_consumer_authorized(module, nonce)
3265                                && self
3266                                    .supervisor
3267                                    .has_live_verified_program_binding(module, nonce)
3268                        }))
3269            {
3270                return Ok(vec![control_error_frame(
3271                    &frame,
3272                    "config_gated",
3273                    "recovery admits only previously bound supervised launches",
3274                )?]);
3275            }
3276            if self
3277                .config_runtime
3278                .as_ref()
3279                .is_some_and(|owner| owner.module_is_gated(module))
3280                && (channel_spawn
3281                    .as_ref()
3282                    .is_none_or(|spawn| !self.supervisor.has_live_verified_channel_binding(spawn))
3283                    || self.config_runtime.as_ref().is_some_and(|owner| {
3284                        !recovering && owner.check_module_hold(module).is_err()
3285                    }))
3286            {
3287                return Ok(vec![control_error_frame(
3288                    &frame,
3289                    "reserved_module",
3290                    "a gated identity belongs only to its supervised child",
3291                )?]);
3292            }
3293        }
3294        if channel_spawn.is_none() {
3295            if let (Some(module), Some(nonce)) = (
3296                hello_value
3297                    .get("manifest")
3298                    .and_then(|value| value.get("module_id"))
3299                    .and_then(|value| value.as_str()),
3300                hello_value
3301                    .get("launch_nonce")
3302                    .and_then(|value| value.as_str()),
3303            ) {
3304                let binding = self.supervisor.has_program_binding_record(module, nonce);
3305                let required = self
3306                    .config_runtime
3307                    .as_ref()
3308                    .is_some_and(|owner| owner.module_is_gated(module));
3309                if let Some(error) = (binding || required)
3310                    .then(|| {
3311                        self.supervisor
3312                            .check_current_program_binding(module, nonce)
3313                            .err()
3314                    })
3315                    .flatten()
3316                {
3317                    let code = if error.code == "program_refused" {
3318                        "program_refused"
3319                    } else {
3320                        "config_delivery_deferred"
3321                    };
3322                    self.request_refusal_close(
3323                        connection_id,
3324                        sink.as_ref(),
3325                        CloseReason::new(code, error.detail.clone()),
3326                    );
3327                    return Ok(vec![control_error_frame(&frame, code, error.detail)?]);
3328                }
3329            }
3330        }
3331        if let Err(err) = validate_hello_display_name(&hello_value) {
3332            return Ok(vec![control_error_frame(
3333                &frame,
3334                "invalid_manifest",
3335                err.to_string(),
3336            )?]);
3337        }
3338        if let Err(err) = validate_hello_capability_grammar(&hello_value) {
3339            return Ok(vec![control_error_frame(
3340                &frame,
3341                "invalid_capability_grammar",
3342                err.to_string(),
3343            )?]);
3344        }
3345        if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
3346            return Ok(vec![control_error_frame(
3347                &frame,
3348                "invalid_manifest",
3349                err.to_string(),
3350            )?]);
3351        }
3352        if let Err(err) = validate_hello_event_declarations(&hello_value) {
3353            return Ok(vec![control_error_frame(
3354                &frame,
3355                "invalid_event_declaration",
3356                err.to_string(),
3357            )?]);
3358        }
3359        if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
3360            return Ok(vec![control_error_frame(
3361                &frame,
3362                "invalid_manifest",
3363                err.to_string(),
3364            )?]);
3365        }
3366        if let Some(provenance) = hello_value
3367            .get("manifest")
3368            .and_then(|manifest| manifest.get("provenance"))
3369        {
3370            if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
3371                return Ok(vec![control_error_frame(
3372                    &frame,
3373                    "invalid_manifest",
3374                    format!("malformed manifest provenance: {err}"),
3375                )?]);
3376            }
3377        }
3378        if let Err(reason) = validate_hello_role_declarations(&hello_value) {
3379            let module_id = hello_value["manifest"]["module_id"]
3380                .as_str()
3381                .unwrap_or("<unknown>");
3382            warn!(module_id, %reason, "dropping whole role_declarations map from HELLO");
3383        }
3384        if let Err(reason) = validate_hello_operation_deferral(&hello_value) {
3385            let module_id = hello_value
3386                .get("manifest")
3387                .and_then(|m| m.get("module_id"))
3388                .and_then(serde_json::Value::as_str)
3389                .unwrap_or("unknown");
3390            warn!(module_id, %reason, "dropping operation_deferral from HELLO");
3391        }
3392        if let Err(reason) = validate_hello_flow_operations(&hello_value) {
3393            let module_id = hello_value["manifest"]["module_id"]
3394                .as_str()
3395                .unwrap_or("<unknown>");
3396            warn!(module_id, %reason, "dropping flow_operations from HELLO");
3397        }
3398        if let Err(reason) = validate_hello_confirm_operations(&hello_value) {
3399            let module_id = hello_value["manifest"]["module_id"]
3400                .as_str()
3401                .unwrap_or("<unknown>");
3402            warn!(module_id, %reason, "dropping confirm_operations from HELLO");
3403        }
3404        let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
3405            Ok(hello) => hello,
3406            Err(err) => {
3407                return Ok(vec![control_error_frame(
3408                    &frame,
3409                    "invalid_hello",
3410                    format!("malformed HELLO body: {err}"),
3411                )?])
3412            }
3413        };
3414
3415        if hello.protocol_ver != hello.manifest.protocol_ver {
3416            return Ok(vec![control_error_frame(
3417                &frame,
3418                "invalid_manifest",
3419                format!(
3420                    "HELLO protocol_ver {} does not match manifest protocol_ver {}",
3421                    hello.protocol_ver, hello.manifest.protocol_ver
3422                ),
3423            )?]);
3424        }
3425
3426        if hello.manifest.module_id.trim().is_empty() {
3427            return Ok(vec![control_error_frame(
3428                &frame,
3429                "invalid_manifest",
3430                "manifest module_id must not be empty",
3431            )?]);
3432        }
3433
3434        let negotiated_ver = match negotiate_version(hello.protocol_ver) {
3435            Ok(negotiated_ver) => negotiated_ver,
3436            Err(message) => {
3437                return Ok(vec![control_error_frame(
3438                    &frame,
3439                    "version_unsupported",
3440                    message,
3441                )?])
3442            }
3443        };
3444
3445        // Registration and forwarding publication use the same module gate as
3446        // stamp, unstamp and spawning. Recheck after taking it; parsing may
3447        // precede those transitions. A program-refusal hold blocks future
3448        // spawns, but does not take this gate or revoke a HELLO whose supervised
3449        // child's image has already passed the check.
3450        let _publication_gate = if let Some(owner) = &self.config_runtime {
3451            let gate = owner
3452                .module_gate(&hello.manifest.module_id)
3453                .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3454            Some(if wait_for_gate {
3455                gate.blocking_lock_owned()
3456            } else {
3457                match gate.try_lock_owned() {
3458                    Ok(guard) => guard,
3459                    Err(_) => {
3460                        return Ok(vec![control_error_frame(
3461                            &frame,
3462                            "config_delivery_deferred",
3463                            "module admission is busy",
3464                        )?])
3465                    }
3466                }
3467            })
3468        } else {
3469            None
3470        };
3471        #[cfg(test)]
3472        if let Some(hold) = &self.hello_before_publication_hold {
3473            hold.wait();
3474            hold.wait();
3475        }
3476        let module = &hello.manifest.module_id;
3477        let required = self
3478            .config_runtime
3479            .as_ref()
3480            .is_some_and(|owner| owner.module_is_gated(module));
3481        let recovering = self
3482            .config_runtime
3483            .as_ref()
3484            .is_some_and(|owner| owner.is_recovering());
3485        // Both entry paths (handle_control -> handle_hello and
3486        // handle_control_frame_timed -> spawn_blocking) enter this same
3487        // handle_hello_checked function. Its fresh, fail-closed store check
3488        // precedes parsing and this publication-gate check; this cached read
3489        // also observes any recovery transition published during that work.
3490        // The nonce-connection-only fallback is defence in depth alongside
3491        // RuntimeOwner::module_is_gated's all-id recovery gate.
3492        if recovering
3493            && channel_spawn.as_ref().is_none_or(|spawn| {
3494                !self.supervisor.has_live_verified_channel_binding(spawn) || spawn.check().is_err()
3495            })
3496            && !(channel_spawn.is_none()
3497                && hello.launch_nonce.as_deref().is_some_and(|nonce| {
3498                    self.supervisor.spawned_consumer_authorized(module, nonce)
3499                        && self
3500                            .supervisor
3501                            .has_live_verified_program_binding(module, nonce)
3502                }))
3503        {
3504            return Ok(vec![control_error_frame(
3505                &frame,
3506                "config_gated",
3507                "recovery admits only previously bound supervised launches",
3508            )?]);
3509        }
3510        if required
3511            && (channel_spawn
3512                .as_ref()
3513                .is_none_or(|spawn| !self.supervisor.has_live_verified_channel_binding(spawn))
3514                || self
3515                    .config_runtime
3516                    .as_ref()
3517                    .is_some_and(|owner| !recovering && owner.check_module_hold(module).is_err()))
3518        {
3519            return Ok(vec![control_error_frame(
3520                &frame,
3521                "reserved_module",
3522                "gated admission requires the current supervised child's registration channel and verified image binding",
3523            )?]);
3524        }
3525        if let Some(spawn) = &channel_spawn {
3526            if let Err(code) = spawn.check() {
3527                return Ok(vec![control_error_frame(
3528                    &frame,
3529                    code,
3530                    "registration launch is no longer live",
3531                )?]);
3532            }
3533        } else if let Some(nonce) = hello.launch_nonce.as_deref() {
3534            if required || self.supervisor.has_program_binding_record(module, nonce) {
3535                if let Err(error) = self.supervisor.check_current_program_binding(module, nonce) {
3536                    let code = if error.code == "program_refused" {
3537                        "program_refused"
3538                    } else {
3539                        "config_delivery_deferred"
3540                    };
3541                    self.request_refusal_close(
3542                        connection_id,
3543                        sink.as_ref(),
3544                        CloseReason::new(code, error.detail.clone()),
3545                    );
3546                    return Ok(vec![control_error_frame(&frame, code, error.detail)?]);
3547                }
3548            }
3549        }
3550
3551        // Swap gate, ahead of the reserved gate on purpose. While a blue/green
3552        // swap is open for this id, the only HELLO admitted as a second process
3553        // is the one carrying the candidate's launch nonce (the swap token), and
3554        // it registers into the candidate slot rather than being refused as a
3555        // duplicate. Run after the reserved gate, a reserved module's candidate
3556        // would be refused `reserved_module` for presenting a nonce that gate
3557        // does not know. See `SupervisorHandle::swap_hello_admission`.
3558        let swap_admission = match &channel_spawn {
3559            Some(spawn) if spawn.is_candidate() => SwapHelloAdmission::Candidate,
3560            Some(_) => SwapHelloAdmission::NotSwapping,
3561            None if prefix_binding.is_some() => SwapHelloAdmission::NotSwapping,
3562            None => self
3563                .supervisor
3564                .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref()),
3565        };
3566        if swap_admission == SwapHelloAdmission::ProtocolNone {
3567            warn!(
3568                module_id = %hello.manifest.module_id,
3569                connection_id = connection_id.get(),
3570                "HELLO refused: the module_id belongs to a protocol:none module, which never registers"
3571            );
3572            // The code stays `swap_token_invalid` for existing callers; only the
3573            // explanation names the actual reason.
3574            return Ok(vec![control_error_frame(
3575                &frame,
3576                "swap_token_invalid",
3577                format!(
3578                    "module_id '{}' is a protocol:none module; a protocol:none module may not register",
3579                    hello.manifest.module_id
3580                ),
3581            )?]);
3582        }
3583        if swap_admission == SwapHelloAdmission::ChannelRequired {
3584            warn!(
3585                module_id = %hello.manifest.module_id,
3586                connection_id = connection_id.get(),
3587                "HELLO refused: the module_id is gated and the HELLO did not arrive over its registration channel"
3588            );
3589            // The code stays `swap_token_invalid` for existing callers; only the
3590            // explanation names the actual reason.
3591            return Ok(vec![control_error_frame(
3592                &frame,
3593                "swap_token_invalid",
3594                format!(
3595                    "module_id '{}' is gated; it must register over its supervised registration channel",
3596                    hello.manifest.module_id
3597                ),
3598            )?]);
3599        }
3600        if swap_admission == SwapHelloAdmission::Refused {
3601            warn!(
3602                module_id = %hello.manifest.module_id,
3603                connection_id = connection_id.get(),
3604                "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
3605            );
3606            return Ok(vec![control_error_frame(
3607                &frame,
3608                "swap_token_invalid",
3609                format!(
3610                    "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
3611                    hello.manifest.module_id
3612                ),
3613            )?]);
3614        }
3615        let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
3616
3617        // A swap candidate proves its identity with the candidate nonce above.
3618        // Other registrations receive this catalog fact only when reserved-module
3619        // policy required and accepted a nonce; unreserved registrations require
3620        // no nonce check.
3621        let verification = if swap_candidate && channel_spawn.is_none() {
3622            ReservedHelloVerification {
3623                nonce_verified: true,
3624                prefix_owner: None,
3625            }
3626        } else {
3627            let verification = if let Some(binding) = &prefix_binding {
3628                if !self.supervisor.prefix_claim_authorized(
3629                    &binding.registration.module_id,
3630                    &hello.manifest.module_id,
3631                ) {
3632                    return Ok(vec![control_error_frame(
3633                        &frame,
3634                        "claim_prefix_refused",
3635                        "prefix ownership changed",
3636                    )?]);
3637                }
3638                Ok(ReservedHelloVerification {
3639                    nonce_verified: true,
3640                    prefix_owner: Some(binding.registration.module_id.clone()),
3641                })
3642            } else if channel_spawn.is_some() {
3643                self.supervisor
3644                    .channel_hello_verification(&hello.manifest.module_id)
3645            } else {
3646                self.supervisor.reserved_hello_verification(
3647                    &hello.manifest.module_id,
3648                    hello.launch_nonce.as_deref(),
3649                )
3650            };
3651            match verification {
3652                Ok(verified) => verified,
3653                Err(rejection) => {
3654                    let message = match rejection {
3655                        ReservedHelloRejection::Exact { module_id } => format!(
3656                            "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
3657                        ),
3658                        ReservedHelloRejection::Prefix {
3659                            prefix,
3660                            owner_module_id,
3661                        } => format!(
3662                            "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
3663                            hello.manifest.module_id
3664                        ),
3665                    };
3666                    return Ok(vec![control_error_frame(
3667                        &frame,
3668                        "reserved_module",
3669                        message,
3670                    )?]);
3671                }
3672            }
3673        };
3674
3675        let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
3676            &hello.manifest.module_id,
3677            hello.manifest.capabilities.as_ref(),
3678        );
3679        if let Some(refusal) = reserved_capability_refusals.first() {
3680            let capability = refusal.capability.clone();
3681            let bound_module = refusal.claimants[0].clone();
3682            log_duplicate_claim_events(reserved_capability_refusals);
3683            return Ok(vec![control_error_frame(
3684                &frame,
3685                "reserved_capability",
3686                format!(
3687                    "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3688                    capability, bound_module, hello.manifest.module_id
3689                ),
3690            )?]);
3691        }
3692
3693        // A connection that already opened client routes must not also register as
3694        // a module: cleanup would then release only one side and leak the other.
3695        if self
3696            .forwarding
3697            .connection_has_client_routes(connection_id)
3698            .map_err(RouterError::Forwarding)?
3699        {
3700            return Ok(vec![control_error_frame(
3701                &frame,
3702                "invalid_hello",
3703                "connection has open client routes and cannot also register as a module",
3704            )?]);
3705        }
3706
3707        // Kept for scope sync authority, which goes only to the connection that
3708        // presented the module's current launch nonce. Recorded before the
3709        // registration is attempted: a connection whose registration then fails
3710        // has no registration, so it cannot sync anyway, and cleanup forgets it.
3711        // A registration channel's authority is the channel itself, never a
3712        // nonce it also presented, and a prefix registration's is its claim:
3713        // neither records one, so neither can fall back to nonce authority.
3714        self.hello_launch_nonces
3715            .lock()
3716            .unwrap_or_else(|poisoned| poisoned.into_inner())
3717            .record(
3718                connection_id,
3719                if prefix_binding.is_some() || channel_spawn.is_some() {
3720                    None
3721                } else {
3722                    hello.launch_nonce.as_deref()
3723                },
3724            );
3725        let control_ops = effective_module_control_ops(hello.control_ops);
3726        let prefix_submodule = verification.prefix_owner.is_some();
3727        // Built before anything is registered so an encoding failure leaves no
3728        // registry or forwarding state behind.
3729        let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
3730        #[cfg(test)]
3731        struct PublicationFinished(Arc<std::sync::Barrier>);
3732        #[cfg(test)]
3733        impl Drop for PublicationFinished {
3734            fn drop(&mut self) {
3735                self.0.wait();
3736            }
3737        }
3738        #[cfg(test)]
3739        let _publication_finished = self.hello_after_launch_check_hold.as_ref().map(|hold| {
3740            hold.wait();
3741            hold.wait();
3742            PublicationFinished(hold.clone())
3743        });
3744        let publication_spawn = channel_spawn
3745            .as_ref()
3746            .or_else(|| prefix_binding.as_ref().map(|binding| &binding.registration));
3747        #[cfg(test)]
3748        if prefix_binding.is_some() {
3749            self.claims.pause_prefix_hello_for_test(7);
3750        }
3751        let _registration_publication = match publication_spawn {
3752            Some(spawn) => Some(match spawn.begin_publication() {
3753                Ok(guard) => guard,
3754                Err(code) => {
3755                    return Ok(vec![control_error_frame(
3756                        &frame,
3757                        code,
3758                        "registration ended before HELLO publication",
3759                    )?])
3760                }
3761            }),
3762            None => None,
3763        };
3764        let (_prefix_live, _prefix_ownership) = if let Some(binding) = &prefix_binding {
3765            let live = self.claims.lock_prefix_binding(binding, connection_id);
3766            let ownership = self.supervisor.prefix_claim_publication(
3767                &binding.registration.module_id,
3768                &hello.manifest.module_id,
3769            );
3770            if live.is_none()
3771                || ownership.is_none()
3772                || !self.claim_primary_registered(binding.spawn, &binding.registration)
3773                || binding.prefix_module_id() != Some(hello.manifest.module_id.as_str())
3774            {
3775                return Ok(vec![control_error_frame(
3776                    &frame,
3777                    "claim_prefix_refused",
3778                    "prefix authority ended before publication",
3779                )?]);
3780            }
3781            (live, ownership)
3782        } else {
3783            (None, None)
3784        };
3785        if let Some(spawn) = publication_spawn {
3786            self.forwarding
3787                .bind_spawn(connection_id, spawn)
3788                .map_err(RouterError::Forwarding)?;
3789        }
3790        // The spawn whose own registration this HELLO is, for `target` on
3791        // the route.open reply: its channel, or the launch its nonce was
3792        // minted for. A prefix child's HELLO names no spawn of its own. It is
3793        // only looked up here. The connection is bound to it and its
3794        // requested declarations are frozen once the registry has accepted
3795        // this HELLO (`admit_hello_authority`), so a HELLO refused as a
3796        // duplicate, or for any other reason, leaves the spawn untouched: its
3797        // closing connection is then no registration of the spawn and ends
3798        // none of the spawn's routes.
3799        let hello_authority = match (&channel_spawn, &prefix_binding) {
3800            (Some(spawn), _) => Some(spawn.authority.clone()),
3801            (None, None) => hello.launch_nonce.as_deref().and_then(|nonce| {
3802                self.supervisor
3803                    .spawned_consumer_record(&hello.manifest.module_id, nonce)
3804            }),
3805            (None, Some(_)) => None,
3806        }
3807        .map(|record| HelloAuthority {
3808            record,
3809            requested: hello
3810                .manifest
3811                .provider_declarations
3812                .clone()
3813                .unwrap_or_default(),
3814            froze: Default::default(),
3815        });
3816        #[cfg(test)]
3817        if prefix_binding.is_some() {
3818            self.claims.pause_prefix_hello_for_test(8);
3819        }
3820        if swap_candidate {
3821            let module_id = hello.manifest.module_id.clone();
3822            let mut differing_declarations = Default::default();
3823            let reply = self.register_swap_candidate(
3824                connection_id,
3825                sink,
3826                &frame,
3827                hello.manifest,
3828                negotiated_ver,
3829                control_ops,
3830                hello_ack,
3831                hello_authority.as_ref(),
3832                &mut differing_declarations,
3833                #[cfg(windows)]
3834                hello.launch_nonce.as_deref(),
3835            )?;
3836            if let Some(spawn) = &channel_spawn {
3837                if self
3838                    .registry
3839                    .get_module_by_connection(connection_id)
3840                    .ok()
3841                    .flatten()
3842                    .is_some()
3843                {
3844                    if let Some(sink) = primary_sink {
3845                        spawn.bind_primary_sink(sink);
3846                    }
3847                    spawn.hello_accepted();
3848                }
3849            } else if self
3850                .registry
3851                .get_module_by_connection(connection_id)
3852                .ok()
3853                .flatten()
3854                .is_some()
3855            {
3856                self.supervisor
3857                    .retire_unused_registration_channel(&module_id, true);
3858            }
3859            if self
3860                .registry
3861                .get_module_by_connection(connection_id)
3862                .ok()
3863                .flatten()
3864                .is_some()
3865            {
3866                self.file_development_widening(
3867                    hello_authority.map(|authority| authority.record),
3868                    differing_declarations,
3869                );
3870            }
3871            return Ok(reply);
3872        }
3873        // Registered unpublished: the module id is claimed, so a duplicate HELLO
3874        // is refused, but `catalog.list` does not show the module and the
3875        // catalog generation does not move until it is published below, once
3876        // its forwarding endpoint exists. Listing it now would advertise a
3877        // module that `route.open` still refuses for having no forwarding
3878        // connection.
3879        let registration = match self
3880            .registry
3881            .register_unpublished_verified_with_control_ops(
3882                hello.manifest,
3883                negotiated_ver,
3884                connection_id,
3885                control_ops,
3886                verification.nonce_verified,
3887            ) {
3888            Ok(registration) => registration,
3889            Err(RegistryError::DuplicateModuleId { module_id }) => {
3890                return Ok(vec![control_error_frame(
3891                    &frame,
3892                    "duplicate_module_id",
3893                    format!(
3894                        "module_id '{module_id}' is already registered; duplicate HELLO rejected"
3895                    ),
3896                )?])
3897            }
3898            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
3899                return Ok(vec![control_error_frame(
3900                    &frame,
3901                    "invalid_module_id",
3902                    err.to_string(),
3903                )?])
3904            }
3905            Err(err) => {
3906                return Ok(vec![control_error_frame(
3907                    &frame,
3908                    "registry_error",
3909                    err.to_string(),
3910                )?])
3911            }
3912        };
3913        let differing_declarations =
3914            match self.admit_hello_authority(connection_id, hello_authority.as_ref()) {
3915                Ok(differing) => differing,
3916                Err(err) => {
3917                    if matches!(
3918                        self.rollback_unpublished_hello(connection_id),
3919                        Ok(r) if !r.is_empty()
3920                    ) {
3921                        crate::supervise::notify_registration_release();
3922                    }
3923                    return Err(RouterError::Forwarding(err));
3924                }
3925            };
3926        #[cfg(test)]
3927        self.hold_between_registry_and_forwarding();
3928
3929        let mut reply = Vec::new();
3930        // Registry readers must wait from before the ACK can be read until
3931        // publication. Nest forwarding inside the registry lock, never reverse.
3932        let publication = self
3933            .registry
3934            .publish_registration_after(connection_id, || {
3935                if let Some(sink) = sink {
3936                    // The forwarding table's module store is also the daemon-to-module
3937                    // control-RPC lane, so every HELLO gets a live endpoint even when the
3938                    // manifest has no routable provider role. Non-routable modules still
3939                    // cannot receive route.bind in production: `handle_route_open` checks
3940                    // the registry manifest with `target_has_required_role` before the
3941                    // only production call to `begin_route_bind_relay_for` below that
3942                    // route.open path. The remaining direct relay callers are unit tests
3943                    // and benchmark harnesses that construct forwarding state explicitly.
3944                    //
3945                    // The HELLO_ACK is queued by the forwarding table itself, before the
3946                    // endpoint becomes visible, and is NOT returned as a reply. A module
3947                    // reads HELLO_ACK first and exits on anything else; a reply is only
3948                    // written after this handler returns, by which time a route.open on
3949                    // another connection could already have queued a route.bind request
3950                    // for this module ahead of it.
3951                    let concurrency = manifest_concurrency(&registration.manifest);
3952                    self.forwarding.register_module_connection_acked(
3953                        connection_id,
3954                        registration.manifest.module_id.clone(),
3955                        negotiated_ver,
3956                        concurrency,
3957                        sink,
3958                        crate::forwarding::ModuleHelloAck {
3959                            frame: hello_ack,
3960                            verified_prefix_owner: verification.prefix_owner,
3961                        },
3962                    )?;
3963                } else {
3964                    // No sink means no forwarding endpoint, so nothing can be routed
3965                    // ahead of the ack; it goes out as the reply.
3966                    reply.push(hello_ack);
3967                }
3968                #[cfg(test)]
3969                self.hold_between_ack_and_publication();
3970                Ok(())
3971            })
3972            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3973        let published = match publication {
3974            Ok(published) => published,
3975            Err(err) => {
3976                // The registry lock has been released, so rollback can take it
3977                // again. No forwarding endpoint or route was published. Keep
3978                // forwarding's weak connection-to-spawn association until the
3979                // connection's read loop ends and removes its remaining routes.
3980                // Removing this unpublished entry does not advance the catalog
3981                // generation.
3982                if matches!(
3983                    self.rollback_unpublished_hello(connection_id),
3984                    Ok(r) if !r.is_empty()
3985                ) {
3986                    crate::supervise::notify_registration_release();
3987                }
3988                self.refuse_hello_authority(connection_id, hello_authority.as_ref());
3989                return Ok(vec![control_error_frame(
3990                    &frame,
3991                    if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
3992                        "invalid_hello"
3993                    } else {
3994                        forwarding_error_code(&err)
3995                    },
3996                    err.to_string(),
3997                )?]);
3998            }
3999        };
4000        if !published {
4001            debug!(
4002                module_id = %registration.manifest.module_id,
4003                connection_id = connection_id.get(),
4004                "registration was removed before it could be published"
4005            );
4006            self.refuse_hello_authority(connection_id, hello_authority.as_ref());
4007            return Ok(reply);
4008        }
4009
4010        drop(_prefix_ownership);
4011        drop(_prefix_live);
4012
4013        // Exposure over assumption: Concurrency's serde default is pinned to the
4014        // pre-field behavior (ModuleManaged), so a management surface that is
4015        // genuinely Serial and just never declared it inherits concurrent
4016        // delivery silently. Logging which registrations RESOLVED BY DEFAULT
4017        // turns "no module has been bitten yet" into the checkable claim "no
4018        // module is exposed" -- one read of the boot log instead of a fleet
4019        // audit. Detected from the raw HELLO bytes because the serde default
4020        // deliberately erases the absent/declared distinction from the type.
4021        if manifest_concurrency_was_defaulted(&frame.body, &registration.manifest) {
4022            info!(
4023                module_id = %registration.manifest.module_id,
4024                "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
4025            );
4026        }
4027        if registration.manifest.registration_channel == Some(false)
4028            && self
4029                .supervisor
4030                .module_is_gated(&registration.manifest.module_id)
4031            && self
4032                .config_runtime
4033                .as_ref()
4034                .and_then(|owner| owner.registration_declaration(&registration.manifest.module_id))
4035                .as_deref()
4036                == Some("declared")
4037        {
4038            warn!(module_id = %registration.manifest.module_id,
4039                "HELLO registration_channel is false despite the approved channel declaration; registration remains admitted");
4040        }
4041
4042        self.apply_registration_capabilities(&registration);
4043
4044        self.supervisor
4045            .record_activity_registration(&registration.manifest.module_id);
4046
4047        info!(
4048            module_id = %registration.manifest.module_id,
4049            module_version = %registration.manifest.module_version,
4050            negotiated_ver,
4051            routable_provider = manifest_provides_routable_role(&registration.manifest),
4052            connection_id = connection_id.get(),
4053            "module registered"
4054        );
4055
4056        if let Some(spawn) = &channel_spawn {
4057            if let Some(sink) = primary_sink {
4058                spawn.bind_primary_sink(sink);
4059            }
4060            spawn.hello_accepted();
4061        } else if !prefix_submodule {
4062            self.supervisor
4063                .retire_unused_registration_channel(&registration.manifest.module_id, false);
4064        }
4065        self.file_development_widening(
4066            hello_authority.map(|authority| authority.record),
4067            differing_declarations,
4068        );
4069        #[cfg(windows)]
4070        if let Some(nonce) = hello.launch_nonce.as_deref() {
4071            self.supervisor
4072                .program_hello_accepted(&registration.manifest.module_id, nonce);
4073        }
4074
4075        #[cfg(test)]
4076        if let Some(hold) = &self.hello_inside_publication_hold {
4077            hold.wait();
4078            hold.wait();
4079        }
4080
4081        Ok(reply)
4082    }
4083
4084    /// Register a HELLO the swap gate admitted into the candidate slot of the
4085    /// registry and of forwarding, where it is reachable over its own
4086    /// connection (its `catalog.update` finds it) but by no by-id lookup, so
4087    /// nothing routes to it until the supervisor cuts over.
4088    ///
4089    /// Registry first (unpublished), then forwarding, then publication, the
4090    /// same order as an ordinary HELLO; a forwarding failure removes the
4091    /// registry entry again. The swap supervisor treats a published, ready
4092    /// candidate as registered and promotes it, and promotion needs the
4093    /// candidate's forwarding endpoint, so the candidate is published only
4094    /// once that endpoint exists. Unlike an ordinary HELLO, this does not
4095    /// recompute capability verdicts or run the capability deny census
4096    /// (`enforce_capability_denies`): both are about the module in the active
4097    /// slot, which `route.open` can target. A candidate sits in the candidate
4098    /// slot, so it neither provides capabilities to other modules nor can be
4099    /// routed to until promotion moves it into the active slot.
4100    #[allow(clippy::too_many_arguments)]
4101    fn register_swap_candidate(
4102        &self,
4103        connection_id: ConnectionId,
4104        sink: Option<crate::FrameSink>,
4105        frame: &Frame,
4106        manifest: ModuleManifest,
4107        negotiated_ver: u8,
4108        control_ops: Vec<String>,
4109        hello_ack: Frame,
4110        hello_authority: Option<&HelloAuthority>,
4111        differing_declarations: &mut ProviderDeclarations,
4112        #[cfg(windows)] launch_nonce: Option<&str>,
4113    ) -> Result<Vec<Frame>, RouterError> {
4114        let module_id = manifest.module_id.clone();
4115        // Closing a swap and publishing its candidate must be ordered. The
4116        // spawn role proves which slot is intended, not that the slot is open.
4117        self.supervisor
4118            .with_swap_candidate_admission(&module_id, || {
4119                let reply = self.publish_swap_candidate(
4120                    connection_id,
4121                    sink,
4122                    frame,
4123                    manifest,
4124                    negotiated_ver,
4125                    control_ops,
4126                    hello_ack,
4127                    hello_authority,
4128                    differing_declarations,
4129                    #[cfg(windows)]
4130                    launch_nonce,
4131                );
4132                let admitted = self
4133                    .registry
4134                    .get_candidate(&module_id)
4135                    .ok()
4136                    .flatten()
4137                    .is_some_and(|candidate| candidate.connection_id == connection_id);
4138                (reply, admitted)
4139            })
4140            .unwrap_or_else(|| {
4141                Ok(vec![control_error_frame(
4142                    frame,
4143                    "swap_token_invalid",
4144                    "swap ended before candidate HELLO publication",
4145                )?])
4146            })
4147    }
4148
4149    #[allow(clippy::too_many_arguments)]
4150    fn publish_swap_candidate(
4151        &self,
4152        connection_id: ConnectionId,
4153        sink: Option<crate::FrameSink>,
4154        frame: &Frame,
4155        manifest: ModuleManifest,
4156        negotiated_ver: u8,
4157        control_ops: Vec<String>,
4158        hello_ack: Frame,
4159        hello_authority: Option<&HelloAuthority>,
4160        differing_declarations: &mut ProviderDeclarations,
4161        #[cfg(windows)] launch_nonce: Option<&str>,
4162    ) -> Result<Vec<Frame>, RouterError> {
4163        let module_id = manifest.module_id.clone();
4164        let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
4165            manifest,
4166            negotiated_ver,
4167            connection_id,
4168            control_ops,
4169        ) {
4170            Ok(registration) => registration,
4171            Err(RegistryError::DuplicateModuleId { module_id }) => {
4172                return Ok(vec![control_error_frame(
4173                    frame,
4174                    "duplicate_module_id",
4175                    format!(
4176                        "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
4177                    ),
4178                )?])
4179            }
4180            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
4181                return Ok(vec![control_error_frame(
4182                    frame,
4183                    "invalid_module_id",
4184                    err.to_string(),
4185                )?])
4186            }
4187            Err(err) => {
4188                return Ok(vec![control_error_frame(
4189                    frame,
4190                    "registry_error",
4191                    err.to_string(),
4192                )?])
4193            }
4194        };
4195        *differing_declarations = match self.admit_hello_authority(connection_id, hello_authority) {
4196            Ok(differing) => differing,
4197            Err(err) => {
4198                if matches!(
4199                    self.rollback_unpublished_hello(connection_id),
4200                    Ok(r) if !r.is_empty()
4201                ) {
4202                    crate::supervise::notify_registration_release();
4203                }
4204                return Err(RouterError::Forwarding(err));
4205            }
4206        };
4207        #[cfg(test)]
4208        self.hold_between_registry_and_forwarding();
4209        let mut reply = Vec::new();
4210        let publication = self
4211            .registry
4212            .publish_registration_after(connection_id, || {
4213                if let Some(sink) = sink {
4214                    // Same ordering as an ordinary HELLO: queue the ACK before
4215                    // inserting the endpoint, and keep registry readers excluded
4216                    // until the candidate is published.
4217                    let concurrency = manifest_concurrency(&registration.manifest);
4218                    self.forwarding.register_candidate_module_connection_acked(
4219                        connection_id,
4220                        module_id.clone(),
4221                        negotiated_ver,
4222                        concurrency,
4223                        sink,
4224                        hello_ack,
4225                    )?;
4226                } else {
4227                    reply.push(hello_ack);
4228                }
4229                #[cfg(test)]
4230                self.hold_between_ack_and_publication();
4231                Ok(())
4232            })
4233            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4234        let published = match publication {
4235            Ok(published) => published,
4236            Err(err) => {
4237                if matches!(
4238                    self.rollback_unpublished_hello(connection_id),
4239                    Ok(r) if !r.is_empty()
4240                ) {
4241                    crate::supervise::notify_registration_release();
4242                }
4243                self.refuse_hello_authority(connection_id, hello_authority);
4244                return Ok(vec![control_error_frame(
4245                    frame,
4246                    forwarding_error_code(&err),
4247                    err.to_string(),
4248                )?]);
4249            }
4250        };
4251        if !published {
4252            debug!(
4253                module_id = %module_id,
4254                connection_id = connection_id.get(),
4255                "swap candidate registration was removed before it could be published"
4256            );
4257            self.refuse_hello_authority(connection_id, hello_authority);
4258            return Ok(reply);
4259        }
4260        info!(
4261            module_id = %module_id,
4262            module_version = %registration.manifest.module_version,
4263            negotiated_ver,
4264            ready = registration.ready,
4265            connection_id = connection_id.get(),
4266            "swap candidate registered; not routable until cutover"
4267        );
4268        #[cfg(windows)]
4269        if let Some(nonce) = launch_nonce {
4270            self.supervisor.program_hello_accepted(&module_id, nonce);
4271        }
4272        Ok(reply)
4273    }
4274
4275    /// Bind `connection` to the spawn its HELLO registers as and freeze the
4276    /// spawn's requested provider declarations. Runs only once the registry
4277    /// has accepted the HELLO, before its endpoint is published, so a route
4278    /// reserved on the endpoint always finds the spawn. Returns the requested
4279    /// entries that differ from the saved approvals, for a widening request.
4280    fn admit_hello_authority(
4281        &self,
4282        connection: ConnectionId,
4283        authority: Option<&HelloAuthority>,
4284    ) -> Result<ProviderDeclarations, ForwardingError> {
4285        let Some(authority) = authority else {
4286            return Ok(ProviderDeclarations::new());
4287        };
4288        let froze = authority
4289            .record
4290            .freeze_declarations(authority.requested.clone());
4291        authority
4292            .froze
4293            .store(froze.is_some(), std::sync::atomic::Ordering::Relaxed);
4294        self.forwarding
4295            .bind_target_authority(connection, authority.record.clone())
4296            .inspect_err(|_| self.refuse_hello_authority(connection, Some(authority)))?;
4297        Ok(froze.unwrap_or_default())
4298    }
4299
4300    /// Undo [`Self::admit_hello_authority`] for a HELLO refused after the
4301    /// registry accepted it, so the refused connection is no registration of
4302    /// the spawn and the spawn's declarations are left for the HELLO that
4303    /// does register it to freeze.
4304    fn refuse_hello_authority(&self, connection: ConnectionId, authority: Option<&HelloAuthority>) {
4305        let Some(authority) = authority else {
4306            return;
4307        };
4308        if let Err(err) = self.forwarding.unbind_target_authority(connection) {
4309            warn!(error = %err, "failed to unbind a refused HELLO from its spawn");
4310        }
4311        if authority
4312            .froze
4313            .swap(false, std::sync::atomic::Ordering::Relaxed)
4314        {
4315            authority.record.unfreeze_refused_declarations();
4316        }
4317    }
4318
4319    fn build_hello_ack(
4320        &self,
4321        frame: &Frame,
4322        negotiated_ver: u8,
4323        module_id: &str,
4324    ) -> Result<Frame, RouterError> {
4325        let ack = ModuleHelloAckBody {
4326            negotiated_ver,
4327            subc_ops: module_subc_ops(),
4328            subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
4329            storage: self
4330                .storage_config
4331                .as_ref()
4332                .map(|cfg| cfg.descriptor_for(module_id)),
4333            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
4334        };
4335        let body = serde_json::to_vec(&ack).map_err(|err| {
4336            RouterError::backend(
4337                0,
4338                frame.header.corr,
4339                format!("failed to encode HELLO_ACK: {err}"),
4340            )
4341        })?;
4342
4343        Frame::build_with_version(
4344            negotiated_ver,
4345            FrameType::HelloAck,
4346            control_flags(),
4347            0,
4348            0,
4349            frame.header.corr,
4350            body,
4351        )
4352        .map_err(RouterError::FrameBuild)
4353    }
4354
4355    async fn handle_client_control_request(
4356        &self,
4357        ctx: &RouteCtx,
4358        frame: Frame,
4359        request: ClientControlRequest,
4360        route_open_timings: &mut Option<RouteOpenTimings>,
4361    ) -> Result<Vec<Frame>, RouterError> {
4362        if let Some(owner) = &self.config_runtime {
4363            let module = match &request {
4364                ClientControlRequest::SupervisorRestart { module_id, .. }
4365                | ClientControlRequest::SupervisorReload { module_id }
4366                | ClientControlRequest::SupervisorSwap { module_id, .. }
4367                | ClientControlRequest::SupervisorSetEnabled {
4368                    module_id,
4369                    enabled: true,
4370                } => Some(module_id.as_str()),
4371                _ => None,
4372            };
4373            let rescan = matches!(
4374                &request,
4375                ClientControlRequest::SupervisorRescan { preview: false }
4376                    | ClientControlRequest::SupervisorRescanChecked { preview: false, .. }
4377            );
4378            let held = if module.is_some() || rescan {
4379                let module = module.map(str::to_owned);
4380                owner
4381                    .on_blocking(move |owner| owner.checked_admission_hold(module.as_deref()))
4382                    .await
4383            } else {
4384                Ok(())
4385            };
4386            if let Err(error) = held {
4387                if error.code != "config_delivery_deferred" {
4388                    let code = if error.code == "store_tampered" {
4389                        "store_tampered"
4390                    } else {
4391                        "config_gated"
4392                    };
4393                    return Ok(vec![control_error_frame(&frame, code, error.detail)?]);
4394                }
4395            }
4396        }
4397        if let Some(operation) = privileged_operation(&request) {
4398            let caller = self.resolve_caller(ctx.connection_id).await;
4399            let op = client_control_request_op(&request);
4400            let text = prompt_text(operation, &caller);
4401            match self.privileged_operator_prompt {
4402                PrivilegedOperatorPrompt::Log => {
4403                    info!(target: "subc_daemon::privileged_prompt", operation = op,
4404                        target = ?operation, caller = %caller.description(), summary = %text,
4405                        "privileged request would prompt the operator");
4406                }
4407                PrivilegedOperatorPrompt::Enforce => {
4408                    let key =
4409                        crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
4410                    let stamp = self
4411                        .operator_apps
4412                        .lock()
4413                        .unwrap_or_else(|p| p.into_inner())
4414                        .get(&ctx.connection_id)
4415                        .cloned();
4416                    let shortcut = crate::operator_confirm::ShortcutContext::eligible_stamp(
4417                        &self.operator_app_policy,
4418                        &self.launch_table,
4419                        stamp.as_ref(),
4420                    )
4421                    .then(|| self.shortcut_context(stamp, None, None));
4422                    let outcome = self
4423                        .forwarding
4424                        .operator_confirms()
4425                        .approve_privileged_with_shortcut(
4426                            ctx,
4427                            &frame,
4428                            op,
4429                            text,
4430                            caller.description(),
4431                            key,
4432                            shortcut,
4433                        )
4434                        .await;
4435                    if outcome.code != "confirmed" {
4436                        return Ok(vec![outcome.frame(&frame)]);
4437                    }
4438                }
4439            }
4440        }
4441        match request {
4442            ClientControlRequest::DaemonShutdownIntent { intent } => {
4443                if intent.as_str() != Some("restart") {
4444                    return Ok(vec![control_error_frame(
4445                        &frame,
4446                        "invalid_params",
4447                        "intent must be 'restart'",
4448                    )?]);
4449                }
4450                // A stamped daemon boots a newer approved generation only from a
4451                // durable restart intent, written here before the daemon exits.
4452                // Failing to write it never blocks the restart: the next start
4453                // then boots the last active generation and the newer one waits.
4454                if let Some(owner) = &self.config_runtime {
4455                    match owner
4456                        .on_blocking(|owner| owner.assign_daemon_restart())
4457                        .await
4458                    {
4459                        Ok(Some(generation)) => {
4460                            info!(
4461                                generation,
4462                                "daemon restart will warm up approved generation"
4463                            )
4464                        }
4465                        Ok(None) => {}
4466                        Err(error) => warn!(
4467                            %error,
4468                            "daemon restart intent not recorded; the next start boots the last active generation"
4469                        ),
4470                    }
4471                }
4472                let response = ClientControlResponse::DaemonShutdownIntent {
4473                    intent: "restart".to_owned(),
4474                    expires_at_ms: self.supervisor.announce_daemon_restart(),
4475                };
4476                Ok(vec![control_response_body_frame(
4477                    &frame,
4478                    &response,
4479                    "ClientControlResponse::DaemonShutdownIntent",
4480                )?])
4481            }
4482            ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
4483            ClientControlRequest::CatalogList { module_id } => {
4484                self.handle_catalog_list(frame, module_id)
4485            }
4486            ClientControlRequest::RouteOpen {
4487                target,
4488                identity,
4489                consumer_identity,
4490                consumer_capabilities,
4491                role_versions,
4492                busy_refusals,
4493                admission_facts,
4494                scope,
4495                ..
4496            } => {
4497                let extensions = match serde_json::from_slice::<subc_control::RouteOpenRequestExt>(
4498                    &frame.body,
4499                ) {
4500                    Ok(extensions) => extensions,
4501                    Err(err) => {
4502                        return Ok(vec![control_error_frame(
4503                            &frame,
4504                            "invalid_control_body",
4505                            format!("malformed route.open extensions: {err}"),
4506                        )?]);
4507                    }
4508                };
4509                self.handle_route_open(
4510                    ctx,
4511                    frame,
4512                    RouteOpenRequest {
4513                        target,
4514                        identity,
4515                        consumer_identity,
4516                        consumer_capabilities,
4517                        role_versions,
4518                        busy_refusals,
4519                        admission_facts,
4520                        scope,
4521                        accepts_close_reasons: extensions.accepts_close_reasons,
4522                    },
4523                    route_open_timings.get_or_insert_with(RouteOpenTimings::new),
4524                )
4525                .await
4526            }
4527            ClientControlRequest::RoutePoll {
4528                route_channel,
4529                route_epoch,
4530                kind,
4531            } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
4532            ClientControlRequest::SupervisorList {} => {
4533                let Ok(permit) = Arc::clone(&self.list_admission).try_acquire_owned() else {
4534                    self.agent_declarations.refuse(
4535                        self,
4536                        ctx,
4537                        control_error_frame(
4538                            &frame,
4539                            "busy",
4540                            "supervisor list work is at capacity; retry",
4541                        )?,
4542                    );
4543                    return Ok(Vec::new());
4544                };
4545                // Filesystem-backed diagnostics must not hold up this connection's reader.
4546                let handler = self.clone();
4547                let sink = ctx.egress.clone();
4548                tokio::spawn(
4549                    async move {
4550                        let _permit = permit;
4551                        let result = handler.handle_supervisor_list(frame.clone()).await;
4552                        let replies = match result {
4553                            Ok(replies) => replies,
4554                            Err(error) => {
4555                                control_error_frame(&frame, "backend_error", error.to_string())
4556                                    .into_iter()
4557                                    .collect()
4558                            }
4559                        };
4560                        for reply in replies {
4561                            if !matches!(
4562                                tokio::time::timeout(Duration::from_millis(200), sink.send(reply))
4563                                    .await,
4564                                Ok(Ok(()))
4565                            ) {
4566                                break;
4567                            }
4568                        }
4569                    }
4570                    .with_current_subscriber(),
4571                );
4572                Ok(Vec::new())
4573            }
4574            ClientControlRequest::SupervisorSpawnSnapshot {} => {
4575                self.handle_supervisor_spawn_snapshot(frame)
4576            }
4577            ClientControlRequest::SupervisorSpawnSubscribe { since } => {
4578                self.handle_supervisor_spawn_subscribe(ctx, frame, since)
4579            }
4580            ClientControlRequest::SupervisorRestart {
4581                module_id,
4582                drain_timeout_ms,
4583            } => {
4584                self.log_supervisor_request_received(
4585                    ctx,
4586                    frame.header.corr,
4587                    ops::SUPERVISOR_RESTART,
4588                    Some(&module_id),
4589                    None,
4590                )?;
4591                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4592                self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
4593                    .await
4594            }
4595            ClientControlRequest::SupervisorSwap {
4596                module_id,
4597                ready_timeout_ms,
4598            } => {
4599                self.log_supervisor_request_received(
4600                    ctx,
4601                    frame.header.corr,
4602                    ops::SUPERVISOR_SWAP,
4603                    Some(&module_id),
4604                    None,
4605                )?;
4606                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4607                self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
4608                    .await
4609            }
4610            ClientControlRequest::SupervisorReload { module_id } => {
4611                self.log_supervisor_request_received(
4612                    ctx,
4613                    frame.header.corr,
4614                    ops::SUPERVISOR_RELOAD,
4615                    Some(&module_id),
4616                    None,
4617                )?;
4618                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4619                self.handle_supervisor_reload(frame, module_id, requester)
4620                    .await
4621            }
4622            ClientControlRequest::SupervisorRescan { preview } => {
4623                if !preview {
4624                    self.log_supervisor_request_received(
4625                        ctx,
4626                        frame.header.corr,
4627                        ops::SUPERVISOR_RESCAN,
4628                        None,
4629                        None,
4630                    )?;
4631                }
4632                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4633                let operation = self.handle_supervisor_rescan(
4634                    frame,
4635                    if preview {
4636                        RescanMode::Preview
4637                    } else {
4638                        RescanMode::Execute
4639                    },
4640                    requester,
4641                );
4642                self.observe_control_wait(ctx, None, ops::SUPERVISOR_RESCAN, operation)
4643                    .await
4644            }
4645            ClientControlRequest::SupervisorRescanChecked {
4646                preview,
4647                candidate,
4648                expect,
4649            } => {
4650                self.observe_control_wait(
4651                    ctx,
4652                    None,
4653                    ops::SUPERVISOR_RESCAN,
4654                    self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect),
4655                )
4656                .await
4657            }
4658            ClientControlRequest::SupervisorReleaseReserved { module_id } => {
4659                self.log_supervisor_request_received(
4660                    ctx,
4661                    frame.header.corr,
4662                    ops::SUPERVISOR_RELEASE_RESERVED,
4663                    Some(&module_id),
4664                    None,
4665                )?;
4666                self.handle_supervisor_release_reserved(frame, module_id)
4667                    .await
4668            }
4669            ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
4670                self.log_supervisor_request_received(
4671                    ctx,
4672                    frame.header.corr,
4673                    ops::SUPERVISOR_SET_ENABLED,
4674                    Some(&module_id),
4675                    Some(enabled),
4676                )?;
4677                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4678                self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
4679                    .await
4680            }
4681            ClientControlRequest::SupervisorHealthProbe { module_id } => {
4682                self.handle_supervisor_health_probe(ctx, frame, module_id)
4683                    .await
4684            }
4685            ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
4686            ClientControlRequest::SupervisorRoutes { module_id } => {
4687                self.handle_supervisor_routes(frame, module_id)
4688            }
4689            ClientControlRequest::SupervisorProvenance { module_id } => {
4690                self.handle_supervisor_provenance(ctx, frame, module_id)
4691                    .await
4692            }
4693            ClientControlRequest::SupervisorStderrTail {
4694                module_id,
4695                max_lines,
4696                max_bytes,
4697            } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
4698            ClientControlRequest::SupervisorTerminals { module_id } => {
4699                self.handle_supervisor_terminals(ctx, frame, module_id)
4700                    .await
4701            }
4702        }
4703    }
4704
4705    fn log_supervisor_request_received(
4706        &self,
4707        ctx: &RouteCtx,
4708        corr: u64,
4709        op: &'static str,
4710        module_id: Option<&str>,
4711        enabled: Option<bool>,
4712    ) -> Result<(), RouterError> {
4713        let caller = self.supervisor_requester(ctx, corr)?;
4714        let caller = principal_label(&caller);
4715        // `Option` fields are recorded only when present, so a request with no
4716        // module id (rescan) or no enabled flag simply omits that field.
4717        info!(
4718            target: "control",
4719            op,
4720            module_id,
4721            enabled,
4722            connection_id = ctx.connection_id.get(),
4723            caller = %caller,
4724            "supervisor request received"
4725        );
4726        Ok(())
4727    }
4728
4729    fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
4730        Ok(self
4731            .registry
4732            .get_module_by_connection(ctx.connection_id)
4733            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
4734            .map(|registration| Principal::Reserved {
4735                module_id: registration.manifest.module_id,
4736            })
4737            .unwrap_or(Principal::Direct))
4738    }
4739
4740    fn handle_module_control_request(
4741        &self,
4742        connection_id: ConnectionId,
4743        frame: Frame,
4744        request: ModuleControlRequestFromModule,
4745    ) -> Result<Vec<Frame>, RouterError> {
4746        match request {
4747            ModuleControlRequestFromModule::RouteAuthorize {
4748                principal,
4749                scope,
4750                target,
4751                expected,
4752            } => self.handle_route_authorize(
4753                connection_id,
4754                frame,
4755                principal,
4756                scope,
4757                target,
4758                expected,
4759            ),
4760            ModuleControlRequestFromModule::CatalogUpdate {
4761                provides,
4762                capabilities,
4763                ready,
4764            } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
4765            ModuleControlRequestFromModule::CatalogUpdateRoleDeclarations { role_declarations } => {
4766                self.handle_catalog_update_role_declarations(
4767                    connection_id,
4768                    frame,
4769                    role_declarations,
4770                )
4771            }
4772            ModuleControlRequestFromModule::LiveRoots {} => {
4773                let registered = self
4774                    .registry
4775                    .get_module_by_connection(connection_id)
4776                    .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4777                let Some(registration) = registered else {
4778                    return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
4779                };
4780                let response = self
4781                    .forwarding
4782                    .live_roots(&registration.manifest.module_id)
4783                    .map_err(RouterError::Forwarding)?;
4784                Ok(vec![control_response_body_frame(
4785                    &frame,
4786                    &response,
4787                    "ModuleControlResponseToModule::LiveRoots",
4788                )?])
4789            }
4790            ModuleControlRequestFromModule::AgentCommands {
4791                started,
4792                finished,
4793                known,
4794            } => self.handle_agent_commands(connection_id, frame, started, finished, known),
4795            ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
4796                self.handle_scope_sync(connection_id, frame, generation, scopes)
4797            }
4798            ModuleControlRequestFromModule::ScopeApply {
4799                generation,
4800                upsert,
4801                end,
4802            } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
4803            ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
4804                self.handle_scope_describe(connection_id, frame, owner, scope_ref)
4805            }
4806            _ => Ok(vec![control_error_frame(
4807                &frame,
4808                "unsupported_control_frame",
4809                "unsupported module control operation",
4810            )?]),
4811        }
4812    }
4813
4814    fn handle_scope_served_by(
4815        &self,
4816        connection_id: ConnectionId,
4817        frame: Frame,
4818    ) -> Result<Vec<Frame>, RouterError> {
4819        let registered = self
4820            .registry
4821            .get_module_by_connection(connection_id)
4822            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4823        let Some(registered) = registered else {
4824            return Ok(vec![control_error_frame(
4825                &frame,
4826                "not_registered",
4827                "scope.served_by requires an active module registration owned by this connection",
4828            )?]);
4829        };
4830        let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
4831        {
4832            Ok(request) => request,
4833            Err(err) => {
4834                return Ok(vec![control_error_frame(
4835                    &frame,
4836                    "invalid_control_body",
4837                    format!("malformed scope.served_by body: {err}"),
4838                )?])
4839            }
4840        };
4841        let answer = self
4842            .scopes
4843            .read()
4844            .unwrap_or_else(|poisoned| poisoned.into_inner())
4845            .served_by(
4846                &registered.manifest.module_id,
4847                &request.scope_ref,
4848                request.scope_epoch,
4849                &request.module_id,
4850            );
4851        let reply = subc_control::ScopeServedByReply::new(
4852            answer,
4853            self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
4854        );
4855        Ok(vec![control_response_body_frame(
4856            &frame,
4857            &reply,
4858            "ScopeServedByReply",
4859        )?])
4860    }
4861
4862    fn handle_operator_confirm(
4863        &self,
4864        ctx: &RouteCtx,
4865        frame: Frame,
4866    ) -> Result<Vec<Frame>, RouterError> {
4867        use crate::operator_confirm::{audit, Outcome};
4868        let registration = self
4869            .registry
4870            .get_module_by_connection(ctx.connection_id)
4871            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4872        let Some(registration) = registration else {
4873            let outcome = Outcome::refusal("not_registered");
4874            audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
4875            return Ok(vec![outcome.frame(&frame)]);
4876        };
4877        let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
4878            Ok(request) => request,
4879            Err(_) => {
4880                let outcome = Outcome::refusal("invalid_control_body");
4881                audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
4882                return Ok(vec![outcome.frame(&frame)]);
4883            }
4884        };
4885        let declared = matches!(
4886            request.kind,
4887            Some(subc_protocol::session::OperatorConfirmKind::Operation)
4888        ) && request.operation.as_ref().is_some_and(|operation| {
4889            registration
4890                .manifest
4891                .confirm_operations
4892                .as_ref()
4893                .is_some_and(|operations| operations.contains(operation))
4894        });
4895        let module_id = registration.manifest.module_id;
4896        // Read the launch nonce (under its own lock) before taking the forwarding
4897        // table's lock below: holding forwarding while waiting on another daemon
4898        // lock risks a lock-order deadlock with paths that take them the other way.
4899        let nonce = self
4900            .hello_launch_nonces
4901            .lock()
4902            .unwrap_or_else(|p| p.into_inner())
4903            .nonce(ctx.connection_id)
4904            .map(str::to_owned);
4905        // A registration channel proves its own spawn; only a connection that
4906        // registered over the launch nonce falls back to the nonce.
4907        let nonce_proven = self
4908            .confirm_channel_proven(ctx.connection_id, &module_id)
4909            .unwrap_or_else(|| {
4910                nonce.as_deref().is_some_and(|nonce| {
4911                    self.supervisor
4912                        .spawned_consumer_authorized(&module_id, nonce)
4913                })
4914            });
4915        let confirms = self.forwarding.operator_confirms();
4916        self.forwarding
4917            .with_operator_route(
4918                ctx.connection_id,
4919                request.route_channel,
4920                request.route_epoch,
4921                |binding| {
4922                    let shortcut = binding
4923                        .filter(|binding| {
4924                            nonce_proven
4925                                && crate::operator_confirm::valid_summary(&request.summary)
4926                                && declared
4927                                && crate::operator_confirm::ShortcutContext::route_eligible(
4928                                    &self.operator_app_policy,
4929                                    &self.launch_table,
4930                                    binding,
4931                                )
4932                        })
4933                        .map(|binding| {
4934                            let phone = crate::operator_confirm::ShortcutContext::phone_fingerprint(
4935                                &self.operator_app_policy,
4936                                binding,
4937                            );
4938                            let stamp = if phone.is_some() {
4939                                None
4940                            } else {
4941                                binding.operator_app.clone()
4942                            };
4943                            self.shortcut_context(
4944                                stamp,
4945                                phone,
4946                                Some((request.route_channel, request.route_epoch)),
4947                            )
4948                        });
4949                    confirms.admit_with_shortcut(
4950                        ctx,
4951                        frame,
4952                        module_id,
4953                        nonce_proven,
4954                        request,
4955                        binding,
4956                        shortcut,
4957                    )
4958                },
4959            )
4960            .map_err(RouterError::Forwarding)
4961    }
4962
4963    fn shortcut_context(
4964        &self,
4965        stamp: Option<crate::operator_apps::OperatorAppStamp>,
4966        phone_fingerprint: Option<String>,
4967        route: Option<(u16, u32)>,
4968    ) -> crate::operator_confirm::ShortcutContext {
4969        crate::operator_confirm::ShortcutContext {
4970            ledger: self.activity.clone(),
4971            forwarding: self.forwarding.clone(),
4972            launches: self.launch_table.clone(),
4973            stamp,
4974            phone_fingerprint,
4975            route,
4976            connections: self.operator_apps.clone(),
4977            #[cfg(test)]
4978            hooks: Default::default(),
4979        }
4980    }
4981
4982    /// Explicit test-only phone policy; daemon configuration cannot select it.
4983    #[cfg(any(test, feature = "test-support"))]
4984    pub fn with_test_phone_shortcut(mut self, enabled: bool) -> Self {
4985        self.operator_app_policy.phone_enabled = enabled;
4986        self
4987    }
4988
4989    /// Resolve folders before taking the scope-table lock: a slow filesystem
4990    /// must not hold up admissions for unrelated scoped routes.
4991    pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
4992        let mut scopes = ScopeRecords::from(scopes);
4993        for (index, record) in scopes.records.iter_mut().enumerate() {
4994            if let Some(refusal) = Self::canonicalize_workdir(record) {
4995                scopes.refusals.insert(index, refusal);
4996            }
4997        }
4998        scopes
4999    }
5000
5001    fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
5002        let workdir = record.workdir.as_mut()?;
5003        let path = std::path::Path::new(workdir);
5004        if !path.is_absolute() {
5005            return Some((
5006                error_codes::INVALID_WORKDIR,
5007                "workdir must be absolute".to_string(),
5008            ));
5009        }
5010        match ProjectRootId::from_path_allowing_missing(path) {
5011            Ok(root) => match root.as_path().to_str() {
5012                Some(canonical) => {
5013                    *workdir = canonical.to_string();
5014                    None
5015                }
5016                None => Some((
5017                    error_codes::INVALID_WORKDIR,
5018                    "canonical workdir is not UTF-8".to_string(),
5019                )),
5020            },
5021            Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
5022        }
5023    }
5024
5025    /// `scope.sync`: the owner is the module registered on this connection.
5026    /// A connection with no registration (every client connection, `direct`
5027    /// included) is refused `not_registered` before the table is consulted.
5028    fn handle_scope_sync(
5029        &self,
5030        connection_id: ConnectionId,
5031        frame: Frame,
5032        generation: u64,
5033        scopes: Vec<ScopeRecord>,
5034    ) -> Result<Vec<Frame>, RouterError> {
5035        self.handle_scope_change(connection_id, frame, generation, scopes, None)
5036    }
5037
5038    fn handle_scope_change(
5039        &self,
5040        connection_id: ConnectionId,
5041        frame: Frame,
5042        generation: u64,
5043        scopes: Vec<ScopeRecord>,
5044        end: Option<Vec<ScopeEnd>>,
5045    ) -> Result<Vec<Frame>, RouterError> {
5046        let is_apply = end.is_some();
5047        let Some(registration) = self
5048            .registry
5049            .get_module_by_connection(connection_id)
5050            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5051        else {
5052            return Ok(vec![control_error_frame(
5053                &frame,
5054                "not_registered",
5055                "scope.sync and scope.apply require an active module registration owned by this connection",
5056            )?]);
5057        };
5058        let owner = registration.manifest.module_id;
5059        let scopes = Self::prepare_scope_records(scopes);
5060        let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
5061        let is_current_launch = |connection: ConnectionId| {
5062            self.connection_is_current_launch(&owner, connection, current_nonce.as_deref())
5063        };
5064        // Lock order is the scope table, then the forwarding table: the new
5065        // tags are published, and the routes the change closes are selected,
5066        // while the scope table is still write-locked, so no admission can read
5067        // a record whose tag is not yet published.
5068        let mut table = self
5069            .scopes
5070            .write()
5071            .unwrap_or_else(|poisoned| poisoned.into_inner());
5072        let outcome = match end {
5073            Some(end) => table.apply(
5074                &owner,
5075                connection_id,
5076                is_current_launch,
5077                generation,
5078                scopes,
5079                end,
5080            ),
5081            None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
5082        };
5083        let drained = match &outcome {
5084            Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
5085            Err(_) => Vec::new(),
5086        };
5087        let has_deadlines = outcome.is_ok() && table.has_deadlines();
5088        drop(table);
5089        if has_deadlines {
5090            self.scope_deadline_added.notify_one();
5091        }
5092        match outcome {
5093            Ok(applied) => {
5094                let counts = ScopeOutcomeCounts::of(&applied.results);
5095                info!(
5096                    owner = %owner,
5097                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
5098                    generation,
5099                    records = applied.results.len(),
5100                    created = counts.created,
5101                    replaced = counts.replaced,
5102                    updated = counts.updated,
5103                    unchanged = counts.unchanged,
5104                    refused = counts.refused,
5105                    ended = applied.ended.len(),
5106                    tag_changes = applied.tag_changes.len(),
5107                    routes_closed = drained.len(),
5108                    "scope change accepted"
5109                );
5110                // An accepted scope change can still refuse individual records, and the
5111                // owner is the only party that sees the reply. Name them here so
5112                // an operator can tell a refused session from a missing one
5113                // without the owner's logs. Capped so a call that refuses
5114                // thousands cannot flood the log; the count above is complete.
5115                for refused in applied
5116                    .results
5117                    .iter()
5118                    .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
5119                    .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
5120                {
5121                    warn!(
5122                        owner = %owner,
5123                        generation,
5124                        scope_ref = %refused.scope_ref,
5125                        scope_epoch = refused.scope_epoch,
5126                        code = refused.code.as_deref().unwrap_or(""),
5127                        "scope record refused"
5128                    );
5129                }
5130                self.close_scope_drained_routes(drained);
5131                let response = if is_apply {
5132                    ModuleControlResponseToModule::ScopeApply {
5133                        generation,
5134                        results: applied.results,
5135                        end_results: applied.end_results,
5136                        ended: applied.ended,
5137                    }
5138                } else {
5139                    ModuleControlResponseToModule::ScopeSync {
5140                        generation,
5141                        results: applied.results,
5142                        ended: applied.ended,
5143                    }
5144                };
5145                Ok(vec![control_response_body_frame(
5146                    &frame,
5147                    &response,
5148                    "scope change response",
5149                )?])
5150            }
5151            Err(refusal) => {
5152                info!(
5153                    owner = %owner,
5154                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
5155                    generation,
5156                    code = refusal.code,
5157                    "scope change refused"
5158                );
5159                Ok(vec![control_error_frame(
5160                    &frame,
5161                    refusal.code,
5162                    refusal.message,
5163                )?])
5164            }
5165        }
5166    }
5167
5168    /// Notify both endpoints of routes drained by a scope change.
5169    fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
5170        scope_expiry::ScopeExpiryContext::new(self).close_scope_drained_routes(drained);
5171    }
5172
5173    /// Re-authorize a stored source on a principal the module supplies.
5174    ///
5175    /// That principal carries no spawn or authority epoch, so this reply never
5176    /// conveys a caller attestation or a first-party mark, and a provider must
5177    /// not apply automatic grants on anything it authorizes.
5178    fn handle_route_authorize(
5179        &self,
5180        connection_id: ConnectionId,
5181        frame: Frame,
5182        principal: Principal,
5183        scope: subc_protocol::scope::RouteAuthorizeScope,
5184        target: String,
5185        expected: subc_protocol::scope::RouteAuthorizeExpected,
5186    ) -> Result<Vec<Frame>, RouterError> {
5187        use crate::scopes::admission::{admit_route, AdmissionMode};
5188        use subc_protocol::scope::{
5189            RouteAuthorizeOutcome, RouteAuthorizeResponse, ROUTE_AUTHORIZE_CAPABILITY,
5190        };
5191        let registry = self
5192            .registry
5193            .route_admission_snapshot()
5194            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5195        let owner_configured = match &scope.owner {
5196            Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
5197            _ => false,
5198        };
5199        let selector = ScopeSelector::new(scope.owner, scope.scope_ref, Some(scope.epoch));
5200        let answer = {
5201            let scopes = self.scopes.read().unwrap_or_else(|p| p.into_inner());
5202            let permitted = registry
5203                .iter()
5204                .find(|r| r.connection_id == connection_id)
5205                .is_some_and(|r| {
5206                    scopes.is_federated_owner(&Principal::Reserved {
5207                        module_id: r.manifest.module_id.clone(),
5208                    }) && r.manifest.capabilities.as_ref().is_some_and(|caps| {
5209                        caps.provides
5210                            .iter()
5211                            .any(|cap| cap == ROUTE_AUTHORIZE_CAPABILITY)
5212                    })
5213                });
5214            if !permitted {
5215                drop(scopes);
5216                return Ok(vec![control_error_frame(&frame, error_codes::ROUTE_AUTHORIZE_NOT_PERMITTED,
5217                    "route.authorize requires the configured federated owner's route-authorize/v1 registration")?]);
5218            }
5219            admit_route(
5220                &scopes,
5221                &registry,
5222                &principal,
5223                &target,
5224                Some(&selector),
5225                owner_configured,
5226                AdmissionMode::Authorize(&expected),
5227            )
5228        };
5229        let response = match answer {
5230            Ok(admission) => {
5231                let mut response = RouteAuthorizeResponse::new(
5232                    RouteAuthorizeOutcome::Authorized,
5233                    "route admission authorized",
5234                );
5235                response.content_version = admission.content_version;
5236                response
5237            }
5238            Err(refusal) => refusal.response(),
5239        };
5240        Ok(vec![control_response_body_frame(
5241            &frame,
5242            &response,
5243            "RouteAuthorizeResponse",
5244        )?])
5245    }
5246
5247    /// `scope.describe`: any registered module may read any scope, because a
5248    /// provider must read the scope a route it serves is stamped with.
5249    fn handle_scope_describe(
5250        &self,
5251        connection_id: ConnectionId,
5252        frame: Frame,
5253        owner: Principal,
5254        scope_ref: String,
5255    ) -> Result<Vec<Frame>, RouterError> {
5256        let registered = self
5257            .registry
5258            .get_module_by_connection(connection_id)
5259            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5260        let Some(registered) = registered else {
5261            return Ok(vec![control_error_frame(
5262                &frame,
5263                "not_registered",
5264                "scope.describe requires an active module registration owned by this connection",
5265            )?]);
5266        };
5267        let scopes = self
5268            .scopes
5269            .read()
5270            .unwrap_or_else(|poisoned| poisoned.into_inner());
5271        if scopes.is_federated_owner(&owner)
5272            && !crate::scopes::target_supports_federated(registered.manifest.capabilities.as_ref())
5273        {
5274            return Ok(vec![control_error_frame(
5275                &frame,
5276                error_codes::SCOPE_FEDERATED_UNSUPPORTED,
5277                "scope.describe of the federated owner requires scope-federated/v1",
5278            )?]);
5279        }
5280        let mut description = scopes.describe(&owner, &scope_ref);
5281        drop(scopes);
5282        if owner
5283            != (Principal::Reserved {
5284                module_id: registered.manifest.module_id.clone(),
5285            })
5286        {
5287            if let Some(facts) = description
5288                .stamp
5289                .as_mut()
5290                .and_then(|stamp| stamp.attributes.federated.as_mut())
5291            {
5292                facts.stamp_required = false;
5293            }
5294        }
5295        // Describe readers also decode strict ScopeAttributes. Attribution is
5296        // safe to omit unless the requesting module declares it understands it.
5297        if !registered
5298            .manifest
5299            .capabilities
5300            .as_ref()
5301            .is_some_and(|caps| {
5302                caps.provides
5303                    .iter()
5304                    .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
5305            })
5306        {
5307            if let Some(stamp) = description.stamp.as_mut() {
5308                stamp.attributes.on_behalf_of_agent = None;
5309            }
5310        }
5311        if !registered
5312            .manifest
5313            .capabilities
5314            .as_ref()
5315            .is_some_and(|caps| {
5316                caps.provides
5317                    .iter()
5318                    .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
5319            })
5320        {
5321            if let Some(stamp) = description.stamp.as_mut() {
5322                stamp.attributes.flow_version = None;
5323                stamp.attributes.flow_code_hash = None;
5324                stamp.attributes.flow_install_id = None;
5325            }
5326        }
5327        let owner_configured = match &owner {
5328            // Ask whether the owner is configured (`is_configured`), not
5329            // whether it is on the roster (`get(..).is_some()`): a supervised
5330            // module's process can register and describe a scope before the
5331            // supervisor has put it on the roster.
5332            Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
5333            _ => false,
5334        };
5335        let response = ModuleControlResponseToModule::ScopeDescribe {
5336            status: description.status,
5337            scope_epoch: description.scope_epoch,
5338            daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
5339            owner_synced: description.owner_synced,
5340            owner_configured,
5341            scope: description.stamp,
5342        };
5343        Ok(vec![control_response_body_frame(
5344            &frame,
5345            &response,
5346            "ModuleControlResponseToModule::ScopeDescribe",
5347        )?])
5348    }
5349
5350    fn handle_catalog_update_role_declarations(
5351        &self,
5352        connection_id: ConnectionId,
5353        frame: Frame,
5354        role_declarations: serde_json::Value,
5355    ) -> Result<Vec<Frame>, RouterError> {
5356        if let Err(err) = validate_role_declarations_value(Some(&role_declarations)) {
5357            return Ok(vec![control_error_frame(
5358                &frame,
5359                "invalid_role_declarations",
5360                err.to_string(),
5361            )?]);
5362        }
5363        let declarations = serde_json::from_value(role_declarations)
5364            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5365        if self
5366            .registry
5367            .replace_role_declarations_for_connection(connection_id, declarations)
5368            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5369            .is_none()
5370        {
5371            return Ok(vec![control_error_frame(
5372                &frame,
5373                "not_registered",
5374                "catalog.update_role_declarations requires an active module registration owned by this connection",
5375            )?]);
5376        }
5377        control_response_body_frame(
5378            &frame,
5379            &ModuleControlResponseToModule::CatalogUpdateRoleDeclarations {},
5380            "ModuleControlResponseToModule::CatalogUpdateRoleDeclarations",
5381        )
5382        .map(|frame| vec![frame])
5383    }
5384
5385    fn handle_catalog_update(
5386        &self,
5387        connection_id: ConnectionId,
5388        frame: Frame,
5389        provides: Vec<ProviderRole>,
5390        capabilities: Option<CapabilityDeclarations>,
5391        ready: Option<bool>,
5392    ) -> Result<Vec<Frame>, RouterError> {
5393        self.refresh_capability_requirements();
5394        let Some(registration) = self
5395            .registry
5396            .get_module_by_connection(connection_id)
5397            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5398        else {
5399            return Ok(vec![control_error_frame(
5400                &frame,
5401                "not_registered",
5402                "catalog.update requires an active module registration owned by this connection",
5403            )?]);
5404        };
5405
5406        if let Some(message) =
5407            catalog_update_frozen_field_message(&registration.manifest, &provides)
5408        {
5409            return Ok(vec![control_error_frame(
5410                &frame,
5411                "catalog_update_frozen_field",
5412                message,
5413            )?]);
5414        }
5415
5416        let mut candidate = registration.manifest.clone();
5417        candidate.provides = provides.clone();
5418        candidate.capabilities = capabilities
5419            .clone()
5420            .or_else(|| registration.manifest.capabilities.clone());
5421        if let Some(rich_ui) = &candidate.rich_ui {
5422            if let Err(err) = rich_ui.validate() {
5423                return Ok(vec![control_error_frame(
5424                    &frame,
5425                    "invalid_manifest",
5426                    err.to_string(),
5427                )?]);
5428            }
5429        }
5430        if let Err(err) = candidate.validate_capability_grammar() {
5431            return Ok(vec![control_error_frame(
5432                &frame,
5433                "invalid_capability_grammar",
5434                err.to_string(),
5435            )?]);
5436        }
5437
5438        // Updates must honor the same reserved owner as initial registration;
5439        // otherwise an empty HELLO could acquire the claim after admission.
5440        let mut conflicts = self
5441            .capability_evaluator
5442            .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
5443        if let Some(conflict) = conflicts.first() {
5444            let message = format!(
5445                "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
5446                conflict.capability, conflict.claimants[0], candidate.module_id
5447            );
5448            for conflict in &mut conflicts {
5449                conflict.source = DuplicateClaimSource::CatalogUpdate;
5450            }
5451            log_duplicate_claim_events(conflicts);
5452            return Ok(vec![control_error_frame(
5453                &frame,
5454                "reserved_capability",
5455                message,
5456            )?]);
5457        }
5458
5459        let updated = match self.registry.replace_catalog_for_connection(
5460            connection_id,
5461            provides,
5462            capabilities,
5463            ready,
5464        ) {
5465            Err(crate::registry::RegistryError::FederatedCapabilityChanged) => {
5466                return Ok(vec![control_error_frame(
5467                    &frame,
5468                    error_codes::CATALOG_UPDATE_FROZEN_CAPABILITY,
5469                    "scope-federated/v1 is fixed at HELLO; restart the module to change it",
5470                )?])
5471            }
5472            Err(crate::registry::RegistryError::CallerAttestationCapabilityChanged) => {
5473                return Ok(vec![control_error_frame(
5474                    &frame,
5475                    error_codes::CATALOG_UPDATE_FROZEN_CAPABILITY,
5476                    "caller-attestation/v1 is fixed at HELLO; restart the module to change it",
5477                )?])
5478            }
5479            result => {
5480                result.map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5481            }
5482        };
5483        if updated.is_none() {
5484            return Ok(vec![control_error_frame(
5485                &frame,
5486                "not_registered",
5487                "catalog.update requires an active module registration owned by this connection",
5488            )?]);
5489        }
5490        if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
5491            log_duplicate_claim_events(
5492                self.capability_evaluator
5493                    .duplicate_claims(DuplicateClaimSource::CatalogUpdate, &registrations),
5494            );
5495        }
5496        if capability_census_trigger(
5497            registration.manifest.capabilities.as_ref(),
5498            updated
5499                .as_ref()
5500                .and_then(|entry| entry.manifest.capabilities.as_ref()),
5501        ) {
5502            self.enforce_capability_denies();
5503        }
5504        self.refresh_capability_requirements();
5505
5506        let response = ModuleControlResponseToModule::CatalogUpdate {};
5507        control_response_body_frame(
5508            &frame,
5509            &response,
5510            "ModuleControlResponseToModule::CatalogUpdate",
5511        )
5512        .map(|frame| vec![frame])
5513    }
5514
5515    pub(crate) fn with_activity_ledger_config(
5516        mut self,
5517        run_dir: &Path,
5518        incarnation: String,
5519        config: crate::activity::ActivityConfig,
5520    ) -> Self {
5521        self.activity = Some(crate::activity::ActivityLedger::open_config(
5522            run_dir.join("activity.jsonl"),
5523            incarnation,
5524            config,
5525        ));
5526        self.supervisor.set_activity(self.activity.clone());
5527        self
5528    }
5529
5530    pub(crate) async fn configure_activity(
5531        &self,
5532        config: crate::activity::ActivityConfig,
5533    ) -> Result<(), &'static str> {
5534        match &self.activity {
5535            Some(ledger) => ledger.configure(config).await,
5536            None => Ok(()),
5537        }
5538    }
5539
5540    fn cancel_activity_subscription(
5541        &self,
5542        ctx: &RouteCtx,
5543        frame: &Frame,
5544    ) -> Result<bool, RouterError> {
5545        let task = self
5546            .activity_streams
5547            .lock()
5548            .unwrap_or_else(|p| p.into_inner())
5549            .remove(&(ctx.connection_id, frame.header.corr));
5550        let Some(task) = task else {
5551            return Ok(false);
5552        };
5553        task.abort();
5554        let end = Frame::build_with_version(
5555            response_version(frame),
5556            FrameType::StreamEnd,
5557            control_flags(),
5558            0,
5559            0,
5560            frame.header.corr,
5561            Vec::new(),
5562        )
5563        .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
5564        let sink = ctx.egress.clone();
5565        tokio::spawn(async move {
5566            let _ = sink.send(end).await;
5567        });
5568        Ok(true)
5569    }
5570
5571    async fn handle_activity_read(
5572        &self,
5573        ctx: &RouteCtx,
5574        frame: Frame,
5575    ) -> Result<Vec<Frame>, RouterError> {
5576        #[cfg(test)]
5577        {
5578            let pause = self
5579                .activity_request_pause
5580                .lock()
5581                .unwrap()
5582                .clone()
5583                .filter(|pause| pause.connection == ctx.connection_id);
5584            if let Some(pause) = pause {
5585                let _cancelled = ActivityRequestCancellation(pause.clone());
5586                pause.started.notify_one();
5587                std::future::pending::<()>().await;
5588            }
5589        }
5590        use subc_control::{
5591            ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
5592            ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
5593        };
5594        #[derive(Deserialize)]
5595        struct Identity {
5596            #[serde(default)]
5597            consumer_identity: Option<ConsumerIdentity>,
5598        }
5599        // A registration channel or claim-bound connection carries its module in
5600        // its binding, so no request field can name another principal, and its
5601        // subscription ends when the bound spawn's authority does.
5602        let mut bound_spawn = None;
5603        let legacy = match self.activity_reader(ctx.connection_id) {
5604            channel_operations::ActivityReader::Module { module_id, spawn } => {
5605                bound_spawn = Some(spawn);
5606                Some(Ok(Principal::Reserved { module_id }))
5607            }
5608            // Refused outright: falling back to `direct` would admit it.
5609            channel_operations::ActivityReader::Refused => {
5610                return Ok(vec![control_error_frame(
5611                    &frame,
5612                    ACTIVITY_READ_FORBIDDEN,
5613                    "helper, relay and retired connections cannot read activity",
5614                )?]);
5615            }
5616            channel_operations::ActivityReader::Legacy => None,
5617        };
5618        // A registered module connection already identifies its caller. Ignore
5619        // consumer_identity there so a module cannot borrow a mirror's identity.
5620        // Only separate consumer connections use the Direct/nonce trust boundary.
5621        let principal = match legacy {
5622            Some(principal) => principal,
5623            None => match self
5624                .forwarding
5625                .module_id_for_connection(ctx.connection_id)
5626                .map_err(RouterError::Forwarding)?
5627            {
5628                Some(module_id) => Ok(Principal::Reserved { module_id }),
5629                None => {
5630                    let identity: Identity = match serde_json::from_slice(&frame.body) {
5631                        Ok(identity) => identity,
5632                        Err(_) => {
5633                            return Ok(vec![control_error_frame(
5634                                &frame,
5635                                ACTIVITY_MALFORMED,
5636                                "malformed activity read",
5637                            )?])
5638                        }
5639                    };
5640                    self.route_open_principal(
5641                        ctx.connection_id,
5642                        &frame,
5643                        identity.consumer_identity,
5644                    )?
5645                }
5646            },
5647        };
5648        let admitted = matches!(principal, Ok(Principal::Direct))
5649            || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
5650        if !admitted {
5651            return Ok(vec![control_error_frame(
5652                &frame,
5653                ACTIVITY_READ_FORBIDDEN,
5654                "activity reads require an operator or trusted mirror principal",
5655            )?]);
5656        }
5657        let Some(ledger) = &self.activity else {
5658            return Ok(vec![control_error_frame(
5659                &frame,
5660                ACTIVITY_UNAVAILABLE,
5661                "activity ledger unavailable",
5662            )?]);
5663        };
5664        if serde_json::from_slice::<ControlOpProbe>(&frame.body)
5665            .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
5666        {
5667            let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
5668                Ok(request) => ledger.query(request).await,
5669                Err(_) => Err(ACTIVITY_MALFORMED),
5670            };
5671            return match result {
5672                Ok(reply) => Ok(vec![control_response_body_frame(
5673                    &frame,
5674                    &reply,
5675                    ops::ACTIVITY_QUERY,
5676                )?]),
5677                Err(code) => Ok(vec![control_error_frame(
5678                    &frame,
5679                    code,
5680                    "activity query refused",
5681                )?]),
5682            };
5683        }
5684        let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
5685            Ok(request) => request,
5686            Err(_) => {
5687                return Ok(vec![control_error_frame(
5688                    &frame,
5689                    ACTIVITY_MALFORMED,
5690                    "malformed activity subscription",
5691                )?])
5692            }
5693        };
5694        let mut subscription = match ledger.subscribe(request.since).await {
5695            Ok(subscription) => subscription,
5696            Err(code) => {
5697                return Ok(vec![control_error_frame(
5698                    &frame,
5699                    code,
5700                    "activity subscription refused",
5701                )?])
5702            }
5703        };
5704        let key = (ctx.connection_id, frame.header.corr);
5705        let sink = ctx.egress.clone();
5706        let mut streams = self
5707            .activity_streams
5708            .lock()
5709            .unwrap_or_else(|p| p.into_inner());
5710        streams.retain(|_, task| !task.is_finished());
5711        if streams.get(&key).is_some_and(|task| !task.is_finished()) {
5712            return Ok(vec![control_error_frame(
5713                &frame,
5714                "duplicate_subscription",
5715                "activity subscription correlation id already in use",
5716            )?]);
5717        }
5718        let stream = async move {
5719            loop {
5720                let entry = if let Some(entry) = subscription.replay.pop_front() {
5721                    Some(entry)
5722                } else {
5723                    subscription.receiver.recv().await
5724                };
5725                let Some(entry) = entry else {
5726                    break;
5727                };
5728                let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
5729                    return;
5730                };
5731                let Ok(next) = Frame::build_with_version(
5732                    response_version(&frame),
5733                    FrameType::StreamData,
5734                    control_flags(),
5735                    0,
5736                    0,
5737                    frame.header.corr,
5738                    body,
5739                ) else {
5740                    return;
5741                };
5742                if sink.send(next).await.is_err() {
5743                    return;
5744                }
5745            }
5746            // Dropping a full subscriber's sender drains its accepted entries
5747            // before the terminal error, preserving their order on the wire.
5748            if let Some(code) = subscription.terminal_code() {
5749                if let Ok(end) =
5750                    control_error_frame(&frame, code, "activity subscriber fell behind")
5751                {
5752                    let _ = sink.send(end).await;
5753                }
5754            }
5755        };
5756        // The spawn's retirement revokes this connection's authority. End the
5757        // subscription then, rather than when the connection's close is
5758        // eventually processed or the next entry fails to send.
5759        let task = tokio::spawn(async move {
5760            match bound_spawn {
5761                Some(spawn) => {
5762                    tokio::select! {
5763                        () = stream => {}
5764                        () = spawn.wait_closed() => {}
5765                    }
5766                }
5767                None => stream.await,
5768            }
5769        });
5770        streams.insert(key, task.abort_handle());
5771        Ok(Vec::new())
5772    }
5773
5774    async fn handle_activity_record(
5775        &self,
5776        ctx: &RouteCtx,
5777        frame: Frame,
5778    ) -> Result<Vec<Frame>, RouterError> {
5779        use subc_control::{
5780            ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
5781            ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
5782        };
5783        let result = async {
5784            let module = self
5785                .forwarding
5786                .module_id_for_connection(ctx.connection_id)
5787                .ok()
5788                .flatten()
5789                .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
5790            let request: ActivityRecordRequest =
5791                serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
5792            crate::activity::validate(&module, &request)?;
5793            let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
5794            let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
5795            let at_ms = std::time::SystemTime::now()
5796                .duration_since(std::time::UNIX_EPOCH)
5797                .unwrap_or_default()
5798                .as_millis() as u64;
5799            ledger.record(module, request, actor, at_ms).await
5800        }
5801        .await;
5802        match result {
5803            Ok(entry) => Ok(vec![control_response_body_frame(
5804                &frame,
5805                &ActivityRecordResponse { seq: entry.seq },
5806                "activity.record",
5807            )?]),
5808            Err(code) => Ok(vec![control_error_frame(
5809                &frame,
5810                code,
5811                "activity recording refused",
5812            )?]),
5813        }
5814    }
5815
5816    fn activity_actor(
5817        &self,
5818        connection: ConnectionId,
5819        module: &str,
5820        on: Option<&subc_control::ActivityOn>,
5821    ) -> Result<subc_control::ActivityActor, &'static str> {
5822        use crate::forwarding::{DataRoute, DataRouteState};
5823        use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
5824        let principal = |id: &str| Principal::Reserved {
5825            module_id: id.to_owned(),
5826        };
5827        let mut actor = ActivityActor {
5828            principal: principal(module),
5829            scope_owner: None,
5830            agent_id: None,
5831            run_id: None,
5832            flow_id: None,
5833            project_root: None,
5834            workdir: None,
5835            project_id: None,
5836        };
5837        let scope = match on {
5838            None => return Ok(actor),
5839            Some(ActivityOn::Scope(scope)) => {
5840                let selector = subc_protocol::scope::ScopeSelector::new(
5841                    principal(module),
5842                    scope.scope_ref.clone(),
5843                    Some(scope.epoch),
5844                );
5845                self.scopes
5846                    .read()
5847                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
5848                    .attribute(&principal(module), module, &selector, true)
5849                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
5850            }
5851            Some(ActivityOn::Route(route)) => {
5852                let DataRoute::Module(DataRouteState::Bound(binding)) = self
5853                    .forwarding
5854                    .lookup_data_route(connection, route.channel, route.epoch)
5855                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
5856                else {
5857                    return Err(ACTIVITY_ACTOR_UNKNOWN);
5858                };
5859                if binding.module_id != module {
5860                    return Err(ACTIVITY_ACTOR_UNKNOWN);
5861                }
5862                actor.principal = binding.principal.clone();
5863                actor.project_root = binding
5864                    .project_root
5865                    .as_ref()
5866                    .map(|root| root.as_path().to_path_buf());
5867                let Some(bound) = &binding.scope else {
5868                    return Ok(actor);
5869                };
5870                let owner = principal(&bound.owner);
5871                let selector = subc_protocol::scope::ScopeSelector::new(
5872                    owner.clone(),
5873                    bound.scope_ref.clone(),
5874                    Some(bound.tag.scope_epoch),
5875                );
5876                let admitted = self
5877                    .scopes
5878                    .read()
5879                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
5880                    .attribute(&owner, module, &selector, true)
5881                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
5882                if admitted.tag != bound.tag {
5883                    return Err(ACTIVITY_ACTOR_UNKNOWN);
5884                }
5885                admitted
5886            }
5887        };
5888        actor.scope_owner = Some(scope.stamp.owner);
5889        actor.agent_id = scope.stamp.attributes.agent_id;
5890        actor.run_id = scope.stamp.attributes.run_id;
5891        actor.flow_id = scope.stamp.attributes.flow_id;
5892        actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
5893        actor.project_id = scope.stamp.project_id;
5894        Ok(actor)
5895    }
5896
5897    fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5898        self.refresh_capability_requirements();
5899        // A bare connection count is ambiguous between many clients holding a
5900        // route each and one client accumulating hundreds, so publish the
5901        // concentration alongside it. Route state is best-effort here: a
5902        // diagnostic endpoint must still answer if the forwarding lock is
5903        // contended.
5904        let mut counters = self.counters.snapshot();
5905        let losses = self.resource_tracker.loss_diagnostics();
5906        if losses.as_object().is_some_and(|losses| !losses.is_empty()) {
5907            if let Some(obj) = counters.as_object_mut() {
5908                obj.insert("agent_command_declaration_loss_by_module".into(), losses);
5909            }
5910        }
5911        if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
5912            obj.insert(
5913                "activity_unparseable_lines".into(),
5914                ledger.read_errors().into(),
5915            );
5916            obj.insert(
5917                "activity_lifecycle_append_failures".into(),
5918                ledger.lifecycle_failures().into(),
5919            );
5920            obj.insert(
5921                "activity_started_digest_skips".into(),
5922                ledger.lifecycle_digest_skips().into(),
5923            );
5924        }
5925        if let (Ok((connections_with_routes, max)), Some(obj)) = (
5926            self.forwarding.client_route_concentration(),
5927            counters.as_object_mut(),
5928        ) {
5929            obj.insert(
5930                "client_connections_with_routes".into(),
5931                connections_with_routes.into(),
5932            );
5933            obj.insert("max_routes_on_one_connection".into(), max.into());
5934        }
5935        // A module that is being fast-refused and a module that is fine look
5936        // identical from a client that retries and succeeds, so name the open
5937        // breakers here. This rides the existing free-form counters object
5938        // rather than a new wire field, so no sibling that deserializes
5939        // `ServerDescribe` has to be rebuilt to keep reading it.
5940        if let (Some(open_breakers), Some(obj)) = (
5941            self.route_bind_breakers.open_snapshot(),
5942            counters.as_object_mut(),
5943        ) {
5944            obj.insert("route_bind_breakers_open".into(), open_breakers);
5945        }
5946        // A daemon whose local endpoint could not be bound keeps serving TCP,
5947        // and the connection file names no endpoint. Name the reason here, in
5948        // the free-form counters object, so an operator reading daemon facts
5949        // sees it without searching the log, and no wire field changes.
5950        if let (Some(reason), Some(obj)) = (
5951            self.local_endpoint_unavailable.clone(),
5952            counters.as_object_mut(),
5953        ) {
5954            obj.insert("local_endpoint_unavailable".into(), reason.into());
5955        }
5956        let response = ClientControlResponse::ServerDescribe {
5957            protocol_ver: PROTOCOL_VERSION,
5958            subc_ops: subc_ops(),
5959            capabilities: self.describe_capabilities(),
5960            connected_clients: self.connected_clients.count(),
5961            counters: Some(counters),
5962            build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
5963            build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
5964            capability_requirements: self.capability_requirement_statuses(),
5965            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
5966            config_path: self
5967                .rescan
5968                .as_ref()
5969                .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
5970        };
5971        Ok(vec![control_response_body_frame(
5972            &frame,
5973            &response,
5974            "ClientControlResponse::ServerDescribe",
5975        )?])
5976    }
5977
5978    fn handle_catalog_list(
5979        &self,
5980        frame: Frame,
5981        module_id: Option<String>,
5982    ) -> Result<Vec<Frame>, RouterError> {
5983        let (generation, modules) = self
5984            .registry
5985            .list_modules_with_nonce_verification()
5986            .map_err(|err| {
5987                RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
5988            })?;
5989        let entries = modules
5990            .into_iter()
5991            .filter(|(registration, _)| {
5992                module_id
5993                    .as_deref()
5994                    .map(|wanted| registration.manifest.module_id == wanted)
5995                    .unwrap_or(true)
5996            })
5997            .map(|(registration, nonce_verified)| {
5998                let entry = self
5999                    .catalog_discovery_entry(&registration, module_id.is_some())
6000                    .map_err(|err| RouterError::backend(0, frame.header.corr, err))?;
6001                let not_ready = self.not_ready_reason(&registration);
6002                let mut entry = entry;
6003                entry.roles = registration.manifest.provides;
6004                entry.control_ops = registration.control_ops;
6005                Ok(entry
6006                    .with_ready(not_ready.is_none())
6007                    .with_display_name(registration.manifest.display_name)
6008                    .with_not_ready(not_ready)
6009                    .with_module_version(Some(registration.manifest.module_version))
6010                    .with_provenance(registration.manifest.provenance)
6011                    .with_capabilities(registration.manifest.capabilities)
6012                    .with_self_signals(registration.manifest.self_signals)
6013                    .with_events(registration.manifest.events)
6014                    .with_operation_delivery(registration.manifest.operation_delivery)
6015                    .with_operation_payload(registration.manifest.operation_payload)
6016                    .with_flow_operations(registration.manifest.flow_operations)
6017                    .with_operation_deferral(registration.manifest.operation_deferral)
6018                    .with_confirm_operations(registration.manifest.confirm_operations)
6019                    .with_rich_ui(registration.manifest.rich_ui)
6020                    .with_role_declarations(registration.manifest.role_declarations)
6021                    .with_nonce_verified(nonce_verified))
6022            })
6023            .collect::<Result<Vec<_>, RouterError>>()?;
6024        let response = ClientControlResponse::CatalogList {
6025            generation,
6026            modules: entries,
6027            subc_ops: subc_ops(),
6028        };
6029        Ok(vec![control_response_body_frame(
6030            &frame,
6031            &response,
6032            "ClientControlResponse::CatalogList",
6033        )?])
6034    }
6035
6036    /// The caller fields for a refusal that only has the raw request frame,
6037    /// such as the connection-level admission refusal sent before the request
6038    /// reaches [`Self::handle_route_open`].
6039    fn route_open_caller_from_frame(&self, ctx: &RouteCtx, frame: &Frame) -> RouteOpenCaller {
6040        let Ok(ClientControlRequest::RouteOpen {
6041            identity,
6042            consumer_identity,
6043            scope,
6044            ..
6045        }) = parse_client_control_request(&frame.body)
6046        else {
6047            return RouteOpenCaller::unknown();
6048        };
6049        let principal = self
6050            .attest_route_open_consumer(ctx.connection_id, consumer_identity.as_ref())
6051            .map(|attested| attested.principal);
6052        RouteOpenCaller::new(principal.as_ref(), scope.as_ref(), &identity)
6053    }
6054
6055    fn route_open_principal(
6056        &self,
6057        connection_id: ConnectionId,
6058        frame: &Frame,
6059        consumer_identity: Option<ConsumerIdentity>,
6060    ) -> Result<Result<Principal, Frame>, RouterError> {
6061        let attested = self
6062            .attest_route_open_consumer(connection_id, consumer_identity.as_ref())
6063            .map(|attested| attested.principal);
6064        self.route_open_principal_attested(frame, consumer_identity, attested)
6065    }
6066
6067    /// [`Self::route_open_principal`] with `attested` already taken by
6068    /// [`Self::attest_route_open_consumer`], so one open resolves its connection
6069    /// claim or launch nonce once.
6070    fn route_open_principal_attested(
6071        &self,
6072        frame: &Frame,
6073        consumer_identity: Option<ConsumerIdentity>,
6074        attested: Option<Principal>,
6075    ) -> Result<Result<Principal, Frame>, RouterError> {
6076        if let Some(principal) = attested {
6077            return Ok(Ok(principal));
6078        }
6079        Ok(Err(control_error_frame(
6080            frame,
6081            "bad_consumer_identity",
6082            consumer_identity.map_or_else(
6083                || "connection claim is no longer authorized for routes".to_string(),
6084                |identity| format!(
6085                    "consumer_identity for module_id '{}' conflicts with its connection claim or did not match a supervised launch nonce",
6086                    identity.module_id
6087                ),
6088            ),
6089        )?))
6090    }
6091
6092    /// Ordinary `route.open` refusals go through here; admission and breaker
6093    /// refusals log separately with their capacity or breaker state. The daemon can
6094    /// attest which code it sent: without the event, a client's "the daemon
6095    /// refused me" and the daemon's own view could only be reconciled by
6096    /// argument. Malformed input (`invalid_project_root`) does not come here;
6097    /// rejecting a request that was never a valid open is not a refusal of one.
6098    #[allow(clippy::too_many_arguments)]
6099    fn route_open_refusal_frame(
6100        &self,
6101        ctx: &RouteCtx,
6102        caller: &RouteOpenCaller,
6103        frame: &Frame,
6104        module_id: &str,
6105        reason: &'static str,
6106        code: &'static str,
6107        message: impl Into<String>,
6108    ) -> Result<Frame, RouterError> {
6109        self.observe_route_open_refusal(ctx, caller, module_id, reason, code);
6110        let detail = if matches!(
6111            reason,
6112            "registration_not_active"
6113                | "supervisor_not_live"
6114                | "no_forwarding_connection"
6115                | "reloading"
6116        ) {
6117            // A planned restart, reload or swap is already replacing the
6118            // process; it does not need a slot in the crash-restart budget.
6119            let expected_back = (reason == "reloading").then_some(true);
6120            self.route_open_recovery_detail(module_id, frame.header.corr, None, expected_back)?
6121        } else {
6122            None
6123        };
6124        control_error_body_frame(
6125            frame,
6126            ErrorBody {
6127                code: code.to_string(),
6128                message: message.into(),
6129                detail,
6130            },
6131        )
6132    }
6133
6134    fn route_open_recovery_detail(
6135        &self,
6136        module_id: &str,
6137        corr: u64,
6138        detail: Option<serde_json::Value>,
6139        expected_back: Option<bool>,
6140    ) -> Result<Option<serde_json::Value>, RouterError> {
6141        let Some((status, _)) = self.supervisor_status(module_id, corr)? else {
6142            return Ok(detail);
6143        };
6144        let mut detail = detail.unwrap_or_else(|| serde_json::json!({}));
6145        detail["state"] = serde_json::json!(status.state.to_string());
6146        if let Some(expected_back) = expected_back {
6147            detail["expected_back"] = serde_json::json!(expected_back);
6148        } else if let Some(module) = self.supervisor.get(module_id) {
6149            match module.will_recover_after_connection_loss() {
6150                Ok(expected_back) => detail["expected_back"] = serde_json::json!(expected_back),
6151                Err(err) => warn!(
6152                    %module_id,
6153                    error = %err,
6154                    "failed to read route.open recovery verdict; omitting expected_back"
6155                ),
6156            }
6157        }
6158        Ok(Some(detail))
6159    }
6160
6161    /// Answer a waiting `route.open` whose relayed bind was refused.
6162    ///
6163    /// `daemon_authored` is provenance, not a code match: the daemon
6164    /// synthesized the refusal itself (a drain, a late reply, a scope that
6165    /// ended mid-bind) or the module refused the bind with its own body. A
6166    /// daemon refusal carries the daemon-origin bit; a module's own
6167    /// rejection is relayed with the bit clear, so a module that refuses
6168    /// with `module_reloading` still cannot mint a daemon refusal.
6169    #[allow(clippy::too_many_arguments)]
6170    fn route_bind_rejection(
6171        &self,
6172        ctx: &RouteCtx,
6173        caller: &RouteOpenCaller,
6174        frame: &Frame,
6175        target_module_id: &str,
6176        body: ErrorBody,
6177        daemon_authored: bool,
6178    ) -> Result<Vec<Frame>, RouterError> {
6179        // The daemon's own commit re-check refused the bind because the
6180        // scope ended or changed after admission. The module accepted;
6181        // counting it as a module rejection would blame the module. This
6182        // branch only runs for daemon-synthesized bodies: a module that
6183        // refuses with a scope code is answered as a module rejection below.
6184        if daemon_authored {
6185            let scope_code = match body.code.as_str() {
6186                error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
6187                error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
6188                _ => None,
6189            };
6190            if let Some(code) = scope_code {
6191                self.observe_route_open_refusal(
6192                    ctx,
6193                    caller,
6194                    target_module_id,
6195                    "scope_changed_before_commit",
6196                    code,
6197                );
6198                return Ok(vec![control_error_body_frame(frame, body)?]);
6199            }
6200        }
6201        self.counters
6202            .increment_route_open_refused("module_rejected");
6203        info!(
6204            target: "control",
6205            code = "module_rejected",
6206            module_code = ?body.code,
6207            module_id = ?target_module_id,
6208            connection_id = ctx.connection_id.get(),
6209            principal = %caller.principal,
6210            scope = %caller.scope,
6211            harness = %caller.harness,
6212            "route.open refused"
6213        );
6214        let refusal = if daemon_authored {
6215            control_error_body_frame(frame, body)?
6216        } else {
6217            control_relayed_error_body_frame(frame, body)?
6218        };
6219        Ok(vec![refusal])
6220    }
6221
6222    /// Refuse a `route.open` because the target module's bind-relay breaker is
6223    /// open, without attempting the relay.
6224    ///
6225    /// The wire code is `module_timeout`, which is the truth (the module has
6226    /// not been answering binds) and which both SDKs already classify as
6227    /// retryable with capped backoff. Reusing it is what keeps this change out
6228    /// of both SDKs; the daemon-side distinction lives in the counter key
6229    /// instead.
6230    ///
6231    /// DELIBERATELY NOT LOGGED PER OCCURRENCE, unlike every other refusal.
6232    /// While a breaker is open this fires on every open to that module, and the
6233    /// stall written up in `docs/designs/route-open-head-of-line.md` already
6234    /// produced 261 lines about a single module inside 3000 lines of daemon
6235    /// log. The rare transitions are logged at warn/info instead and the volume
6236    /// is carried by the counter, so the evidence survives without the flood.
6237    /// The debug line keeps a per-refusal record reachable for whoever turns
6238    /// the level up.
6239    #[allow(clippy::too_many_arguments)]
6240    fn route_open_breaker_refusal_frame(
6241        &self,
6242        ctx: &RouteCtx,
6243        caller: &RouteOpenCaller,
6244        frame: &Frame,
6245        module_id: &str,
6246        consecutive_timeouts: u32,
6247        retry_in: Duration,
6248        probe_in_flight: bool,
6249    ) -> Result<Frame, RouterError> {
6250        self.counters
6251            .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
6252        debug!(
6253            target: "control",
6254            code = "module_timeout",
6255            module_id = ?module_id,
6256            connection_id = ctx.connection_id.get(),
6257            principal = %caller.principal,
6258            scope = %caller.scope,
6259            harness = %caller.harness,
6260            consecutive_timeouts,
6261            retry_in_ms = retry_in.as_millis() as u64,
6262            probe_in_flight,
6263            "route.open refused by open bind-relay breaker"
6264        );
6265        // Say what a caller can act on. An open bind-relay breaker means the
6266        // module timed out accepting several new routes in a row. The module
6267        // is still running and its established routes keep working; only new
6268        // route.open requests are refused until the cooldown ends and one
6269        // test route (the probe) gets through. A message that only counts
6270        // failed relays reads as "the module is down" to a worker that sees it.
6271        let detail = if probe_in_flight {
6272            "one test route is already being tried; retry once it settles".to_string()
6273        } else {
6274            format!("retrying new routes in {}s", retry_in.as_secs().max(1))
6275        };
6276        control_error_frame(
6277            frame,
6278            "module_timeout",
6279            format!(
6280                "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
6281                 timed out in a row); {detail}; its established routes are unaffected"
6282            ),
6283        )
6284    }
6285
6286    /// `code` is daemon vocabulary and prints plainly; `module_id` is the
6287    /// requester's bytes (an unknown target is whatever the client sent) and
6288    /// is Debug-formatted so control characters land in the log escaped
6289    /// rather than as terminal sequences for whoever tails it.
6290    ///
6291    /// `reason` names the check that refused, because one wire code has
6292    /// several senders: after a module registers, `target_unavailable` can
6293    /// come from a missing role, an inactive registration, a supervisor that
6294    /// has not marked the process live, a missing forwarding connection, or a
6295    /// failed relay, and a log that records only the code cannot say which of
6296    /// them fired. It is a static, daemon-chosen label per branch, so it is
6297    /// safe to print plainly and stays a closed set.
6298    fn observe_route_open_refusal(
6299        &self,
6300        ctx: &RouteCtx,
6301        caller: &RouteOpenCaller,
6302        module_id: &str,
6303        reason: &'static str,
6304        code: &'static str,
6305    ) {
6306        self.counters.increment_route_open_refused(code);
6307        info!(
6308            target: "control",
6309            code,
6310            reason,
6311            module_id = ?module_id,
6312            connection_id = ctx.connection_id.get(),
6313            principal = %caller.principal,
6314            scope = %caller.scope,
6315            harness = %caller.harness,
6316            "route.open refused"
6317        );
6318        if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
6319            self.route_outages.record_not_serving(module_id, reason);
6320        }
6321    }
6322
6323    /// Record an ACCEPTED route.open.
6324    ///
6325    /// Refusals have been logged and counted since the attestation work; accepts
6326    /// were invisible, so the daemon knew every principal it stamped and wrote
6327    /// none of them down. The party that attests the identity was the only party
6328    /// not recording it, which left a credential vault unable to name the sender
6329    /// of a call that reached it (claustrum #43) and left the launch-nonce
6330    /// concurrency question unanswerable from the outside.
6331    ///
6332    /// FIELD NAMES MATCH `route.open refused` DELIBERATELY, so one grep over
6333    /// `code`/`module_id`/`connection_id` returns both directions of the same
6334    /// decision rather than two shapes a reader has to join by hand.
6335    ///
6336    /// `module_id` IS RENDERED BARE HERE AND DEBUG-ESCAPED ON THE REFUSAL PATH,
6337    /// and the difference carries information rather than being an
6338    /// inconsistency. This line is only reachable after a successful bind to a
6339    /// REGISTERED module, so the value has already passed HELLO validation
6340    /// including the path-hazard refusal and cannot contain control bytes. A
6341    /// refused id may be arbitrary attacker-chosen bytes and must stay escaped.
6342    /// So A QUOTED `module_id` IN THE LOG MEANS THE VALUE WAS NEVER VALIDATED.
6343    ///
6344    /// Bare is also what every other daemon line already emits (`module
6345    /// registered`, `configured module supervised`). Shipping `?module_id` here
6346    /// made this instrument the only one in the file whose ids did not answer
6347    /// `grep module_id=broca` -- 3 hits against 342 for the escaped form, in a
6348    /// line whose whole purpose is being grepped beside its sibling.
6349    ///
6350    /// THIS RENDERING IS UNFENCED AND THE REASON IS WORTH KNOWING: the in-crate
6351    /// `EventCapture` test layer implements only `record_debug`, so `Visit`
6352    /// forwards every field type through it and a bare `&str` and a `?`-escaped
6353    /// one are recorded identically. A test written against that harness passes
6354    /// either way -- I wrote one, measured it, and deleted it rather than ship a
6355    /// green assertion that cannot fail. The same limit applies to the escaping
6356    /// assertion in `route_open_supervised_absence_emits_refusal_fields_and_counts_code`:
6357    /// it reads as a guard on the Debug escaping and cannot detect its removal.
6358    /// Fencing either needs the real formatter, not the capture layer.
6359    ///
6360    /// `peer_addr` is NOT here and cannot be: `SO_PEERCRED`/`LOCAL_PEERPID` are
6361    /// unix-socket options and subc is loopback TCP, so there is no peer identity
6362    /// to record. The ephemeral port would decay within minutes and answer only a
6363    /// live question. The identity question is instead answered by counting
6364    /// distinct live connections presenting one module's `consumer_identity` --
6365    /// "is anyone else holding this secret" rather than "is this the right
6366    /// process".
6367    fn observe_route_open_accept(
6368        &self,
6369        ctx: &RouteCtx,
6370        module_id: &str,
6371        principal: &str,
6372        scope: &str,
6373    ) {
6374        self.route_outages.record_accepted(module_id);
6375        self.counters.increment_route_open_accepted(principal);
6376        info!(
6377            target: "control",
6378            principal,
6379            module_id,
6380            connection_id = ctx.connection_id.get(),
6381            scope = %scope,
6382            "route.open accepted"
6383        );
6384    }
6385
6386    fn supervised_absent_route_open_refusal_frame(
6387        &self,
6388        ctx: &RouteCtx,
6389        caller: &RouteOpenCaller,
6390        frame: &Frame,
6391        module_id: &str,
6392        code: &'static str,
6393        status: &crate::supervise::ModuleStatus,
6394    ) -> Result<Frame, RouterError> {
6395        self.counters.increment_route_open_refused(code);
6396        info!(
6397            target: "control",
6398            code,
6399            reason = "supervised_not_registered",
6400            module_id = ?module_id,
6401            connection_id = ctx.connection_id.get(),
6402            principal = %caller.principal,
6403            scope = %caller.scope,
6404            harness = %caller.harness,
6405            state = %status.state,
6406            enabled = status.enabled,
6407            live = status.live,
6408            "route.open refused"
6409        );
6410        // A supervised module whose process has not registered is not
6411        // serving, whatever the reason; the supervisor knows this id, so it is
6412        // safe to track.
6413        self.route_outages
6414            .record_not_serving(module_id, "supervised_not_registered");
6415        let message = if let Some(pid) = status.waiting_for_previous_pid {
6416            status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
6417                format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
6418            })
6419        } else {
6420            format!(
6421                "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
6422                status.state, status.enabled, status.live
6423            )
6424        };
6425        // `module_warming` here means the process is starting, running or
6426        // being restarted and has not registered yet. Nothing has to be
6427        // restarted for it to serve, so the crash budget does not decide it.
6428        let expected_back = (code == error_codes::MODULE_WARMING).then_some(true);
6429        control_error_body_frame(
6430            frame,
6431            ErrorBody {
6432                code: code.to_string(),
6433                message,
6434                detail: self.route_open_recovery_detail(
6435                    module_id,
6436                    frame.header.corr,
6437                    None,
6438                    expected_back,
6439                )?,
6440            },
6441        )
6442    }
6443
6444    async fn handle_route_open(
6445        &self,
6446        ctx: &RouteCtx,
6447        frame: Frame,
6448        request: RouteOpenRequest,
6449        timings: &mut RouteOpenTimings,
6450    ) -> Result<Vec<Frame>, RouterError> {
6451        let RouteOpenRequest {
6452            target,
6453            mut identity,
6454            consumer_identity,
6455            consumer_capabilities,
6456            role_versions,
6457            busy_refusals,
6458            admission_facts,
6459            scope,
6460            accepts_close_reasons,
6461        } = request;
6462        let target_module_id = target_module_id(&target).to_string();
6463        timings.module_id.clone_from(&target_module_id);
6464        if self
6465            .registry
6466            .get_module_by_connection(ctx.connection_id)
6467            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6468            .is_some()
6469        {
6470            return Ok(vec![control_error_frame(
6471                &frame,
6472                "invalid_request",
6473                "module connections cannot open client routes",
6474            )?]);
6475        }
6476        debug!(
6477            connection_id = ctx.connection_id.get(),
6478            corr = frame.header.corr,
6479            module_id = %target_module_id,
6480            "handling route.open"
6481        );
6482
6483        // The caller's attested principal, requested scope and harness, for
6484        // every `route.open refused` line below. The connection's claim
6485        // binding or launch nonce is checked here, once, and the answer is
6486        // reused where the principal is enforced further down, so the order
6487        // of refusals is unchanged.
6488        let attested =
6489            self.attest_route_open_consumer(ctx.connection_id, consumer_identity.as_ref());
6490        let mut caller = RouteOpenCaller::new(
6491            attested.as_ref().map(|attested| &attested.principal),
6492            scope.as_ref(),
6493            &identity,
6494        );
6495
6496        // A malformed declaration is refused first, before anything about the
6497        // target is looked up: the same body would be refused against any
6498        // module, so the caller learns nothing by retrying or waiting. An empty
6499        // map declares nothing and travels as no field at all, so a provider
6500        // only ever sees a missing field or a non-empty one.
6501        let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
6502        if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
6503            self.observe_route_open_refusal(
6504                ctx,
6505                &caller,
6506                &target_module_id,
6507                "invalid_role_versions",
6508                error_codes::INVALID_REQUEST,
6509            );
6510            return Ok(vec![control_error_body_frame(
6511                &frame,
6512                ErrorBody {
6513                    code: error_codes::INVALID_REQUEST.to_string(),
6514                    message: error.to_string(),
6515                    detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
6516                },
6517            )?]);
6518        }
6519
6520        if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
6521            if status.waiting_for_previous_pid.is_some() {
6522                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
6523                    ctx,
6524                    &caller,
6525                    &frame,
6526                    &target_module_id,
6527                    "module_warming",
6528                    &status,
6529                )?]);
6530            }
6531        }
6532
6533        // WHY THESE REPLIES DISCRIMINATE FREELY, since the usual rule is the
6534        // opposite. Below, a caller learns whether a module is unregistered,
6535        // supervised-but-down (with state/enabled/live), or registered without the
6536        // requested role. Elsewhere that is an enumeration leak: a probe learning
6537        // the shape of a fleet it cannot otherwise see.
6538        //
6539        // It is not one here, and the reason is the ACCESS MODEL rather than
6540        // anything about these errors. Reaching route.open requires the
6541        // pre-envelope HMAC handshake, whose key lives in a 0600 user-owned
6542        // connection file, so any caller who completes it already runs as this
6543        // user -- and can read subc.jsonc for the module list and `ck module
6544        // status` for live state. The reply discloses nothing the caller cannot
6545        // read more easily from disk, while the precision is load-bearing:
6546        // `unknown_module` is retryable and a missing role is not.
6547        //
6548        // IF THE HANDSHAKE EVER ADMITS A PRINCIPAL THAT IS NOT THIS USER -- a
6549        // remote transport, a sandboxed caller, a shared-host mode -- THAT
6550        // PREMISE DIES AND THESE THREE REPLIES MUST COLLAPSE INTO ONE.
6551        let admission_registry = self
6552            .registry
6553            .route_admission_snapshot()
6554            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
6555        let Some(registration) =
6556            crate::scopes::admission::registered_target(&admission_registry, &target_module_id)
6557        else {
6558            if let Some((status, warming)) =
6559                self.supervisor_status(&target_module_id, frame.header.corr)?
6560            {
6561                // BEFORE the two availability codes below, because for a module
6562                // that speaks no subc wire both of them are false comfort: they
6563                // say "not right now" and are retried, and this module will
6564                // never register no matter how long the caller waits. The
6565                // absence here is the declaration being honoured, not a module
6566                // that is late.
6567                if status.protocol == ModuleProtocol::None {
6568                    return Ok(vec![self.route_open_refusal_frame(
6569                        ctx,
6570                        &caller,
6571                        &frame,
6572                        &target_module_id,
6573                        "protocol_none",
6574                        error_codes::MODULE_NO_PROTOCOL,
6575                        format!(
6576                            "module_id '{target_module_id}' is declared protocol: none; \
6577                             it speaks no subc wire and serves no routes"
6578                        ),
6579                    )?]);
6580                }
6581                let code = if warming {
6582                    "module_warming"
6583                } else {
6584                    "target_unavailable"
6585                };
6586                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
6587                    ctx,
6588                    &caller,
6589                    &frame,
6590                    &target_module_id,
6591                    code,
6592                    &status,
6593                )?]);
6594            }
6595            if let Some(removed_ago_ms) =
6596                self.supervisor.removal_tombstone_age_ms(&target_module_id)
6597            {
6598                return Ok(vec![self.route_open_refusal_frame(
6599                    ctx,
6600                    &caller,
6601                    &frame,
6602                    &target_module_id,
6603                    "removed",
6604                    error_codes::MODULE_REMOVED,
6605                    format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
6606                )?]);
6607            }
6608            return Ok(vec![self.route_open_refusal_frame(
6609                ctx,
6610                &caller,
6611                &frame,
6612                &target_module_id,
6613                "not_registered",
6614                error_codes::UNKNOWN_MODULE,
6615                format!("module_id '{target_module_id}' is not registered"),
6616            )?]);
6617        };
6618
6619        // Best-effort only: registry readiness and forwarding reservation use
6620        // different locks, so a module can flip readiness between this read and
6621        // the relay. Modules must still tolerate an `on_bind` while not ready.
6622        if crate::scopes::admission::target_registration(&admission_registry, &target_module_id)
6623            .is_err()
6624        {
6625            self.counters
6626                .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
6627            info!(
6628                target: "control",
6629                code = error_codes::MODULE_WARMING,
6630                module_id = ?target_module_id,
6631                connection_id = ctx.connection_id.get(),
6632                principal = %caller.principal,
6633                scope = %caller.scope,
6634                harness = %caller.harness,
6635                reason = "declared_not_ready",
6636                "route.open refused"
6637            );
6638            // The module is registered but says it cannot take work, which is
6639            // an outage from the caller's side even though its process is up.
6640            self.route_outages
6641                .record_not_serving(&target_module_id, "declared_not_ready");
6642            // The registered process can declare itself ready without a
6643            // restart, so an exhausted crash budget does not prevent serving.
6644            return Ok(vec![control_error_body_frame(
6645                &frame,
6646                ErrorBody {
6647                    code: error_codes::MODULE_WARMING.to_string(),
6648                    message: format!(
6649                        "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
6650                    ),
6651                    detail: self.route_open_recovery_detail(
6652                        &target_module_id,
6653                        frame.header.corr,
6654                        Some(serde_json::json!({ "reason": "declared_not_ready" })),
6655                        Some(true),
6656                    )?,
6657                },
6658            )?]);
6659        }
6660
6661        // Effective readiness, second half: a module that declares a capability
6662        // `need: required` is not routable while that capability has no
6663        // registered provider. It is enforced HERE, as a retryable routing
6664        // refusal, and deliberately not as spawn ordering or a boot block. The
6665        // module is still started and registered and can make its own calls;
6666        // spawn ordering is a promise that cannot be kept once a provider
6667        // crashes at runtime, and refusing to boot would stop the whole
6668        // machine, including the tools needed to fix its configuration.
6669        //
6670        // "Provided" is the evaluator's verdict, which counts a provider as
6671        // soon as it has REGISTERED, not once it is ready. Two modules that
6672        // require each other's capabilities are therefore both routable once
6673        // both register; counting readiness instead would deadlock them.
6674        //
6675        // Only new opens are refused. Routes already bound when a provider
6676        // goes away stay bound: nothing here tears them down, and the module
6677        // answers them as it can. Like the readiness read above this is
6678        // best-effort against a provider registering or leaving concurrently.
6679        if let Some(capability) = self.first_unprovided_required_capability(registration) {
6680            self.counters
6681                .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
6682            info!(
6683                target: "control",
6684                code = error_codes::MODULE_WARMING,
6685                module_id = ?target_module_id,
6686                connection_id = ctx.connection_id.get(),
6687                principal = %caller.principal,
6688                scope = %caller.scope,
6689                harness = %caller.harness,
6690                reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
6691                capability = %capability,
6692                "route.open refused"
6693            );
6694            return Ok(vec![control_error_body_frame(
6695                &frame,
6696                ErrorBody {
6697                    code: error_codes::MODULE_WARMING.to_string(),
6698                    message: format!(
6699                        "module_id '{target_module_id}' requires capability '{capability}', \
6700                         which no registered module provides; retry"
6701                    ),
6702                    detail: Some(serde_json::json!({
6703                        "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
6704                        "capability": capability,
6705                    })),
6706                },
6707            )?]);
6708        }
6709
6710        if !target_has_required_role(&target, &registration.manifest.provides) {
6711            return Ok(vec![self.route_open_refusal_frame(
6712                ctx,
6713                &caller,
6714                &frame,
6715                &target_module_id,
6716                "role_not_provided",
6717                "target_unavailable",
6718                format!("module_id '{target_module_id}' does not provide the requested target"),
6719            )?]);
6720        }
6721
6722        if registration.state != ChannelState::Active {
6723            return Ok(vec![self.route_open_refusal_frame(
6724                ctx,
6725                &caller,
6726                &frame,
6727                &target_module_id,
6728                "registration_not_active",
6729                "target_unavailable",
6730                format!("module_id '{target_module_id}' is not active"),
6731            )?]);
6732        }
6733
6734        if self
6735            .forwarding
6736            .module_is_draining(&target_module_id)
6737            .map_err(RouterError::Forwarding)?
6738        {
6739            return Ok(vec![self.route_open_refusal_frame(
6740                ctx,
6741                &caller,
6742                &frame,
6743                &target_module_id,
6744                "reloading",
6745                "module_reloading",
6746                format!("module_id '{target_module_id}' is reloading"),
6747            )?]);
6748        }
6749
6750        if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
6751            process_liveness.process_live(&target_module_id) == Some(false)
6752        }) {
6753            // A module the supervisor is restarting or reloading can still hold
6754            // a registration: the old process before its connection closes, or
6755            // a new one that registered while the supervisor was draining. The
6756            // forwarding table does not see that as draining, but the consumer
6757            // should still be told to retry soon, exactly as for the drain
6758            // above, rather than that the target is unavailable.
6759            if process_liveness.process_replacing(&target_module_id) {
6760                return Ok(vec![self.route_open_refusal_frame(
6761                    ctx,
6762                    &caller,
6763                    &frame,
6764                    &target_module_id,
6765                    "reloading",
6766                    "module_reloading",
6767                    format!("module_id '{target_module_id}' is reloading"),
6768                )?]);
6769            }
6770            return Ok(vec![self.route_open_refusal_frame(
6771                ctx,
6772                &caller,
6773                &frame,
6774                &target_module_id,
6775                "supervisor_not_live",
6776                "target_unavailable",
6777                format!("module_id '{target_module_id}' is not live"),
6778            )?]);
6779        }
6780
6781        if !self
6782            .forwarding
6783            .has_live_module_connection(&target_module_id)
6784            .map_err(RouterError::Forwarding)?
6785        {
6786            return Ok(vec![self.route_open_refusal_frame(
6787                ctx,
6788                &caller,
6789                &frame,
6790                &target_module_id,
6791                "no_forwarding_connection",
6792                "target_unavailable",
6793                format!("module_id '{target_module_id}' has no live forwarding connection"),
6794            )?]);
6795        }
6796
6797        if let Some(error) =
6798            self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
6799        {
6800            self.observe_route_open_refusal(
6801                ctx,
6802                &caller,
6803                &target_module_id,
6804                "op_not_allowed",
6805                "op_not_allowed",
6806            );
6807            return Ok(vec![error]);
6808        }
6809
6810        let principal = match (&attested, consumer_identity.is_some()) {
6811            // A claim-bound connection whose launch ended attests nothing,
6812            // and it must never fall back to opening as `direct`.
6813            (None, false) => {
6814                self.observe_route_open_refusal(
6815                    ctx,
6816                    &caller,
6817                    &target_module_id,
6818                    "registration_channel_closed",
6819                    crate::supervise::REGISTRATION_CHANNEL_CLOSED,
6820                );
6821                return Ok(vec![control_error_frame(
6822                    &frame,
6823                    crate::supervise::REGISTRATION_CHANNEL_CLOSED,
6824                    "the claim binding's launch has ended",
6825                )?]);
6826            }
6827            _ => match self.route_open_principal_attested(
6828                &frame,
6829                consumer_identity,
6830                attested.as_ref().map(|attested| attested.principal.clone()),
6831            )? {
6832                Ok(principal) => principal,
6833                Err(error) => {
6834                    self.observe_route_open_refusal(
6835                        ctx,
6836                        &caller,
6837                        &target_module_id,
6838                        "bad_consumer_identity",
6839                        "bad_consumer_identity",
6840                    );
6841                    return Ok(vec![error]);
6842                }
6843            },
6844        };
6845        let attested = attested.expect("an attested principal was resolved above");
6846        timings.principal = principal_label(&principal);
6847
6848        // The opener's spawn authority, read once here and carried with the
6849        // relay; nothing below reads its record again.
6850        let Ok(mut caller_capture) = attested.capture() else {
6851            self.observe_route_open_refusal(
6852                ctx,
6853                &caller,
6854                &target_module_id,
6855                "registration_channel_closed",
6856                crate::supervise::REGISTRATION_CHANNEL_CLOSED,
6857            );
6858            return Ok(vec![control_error_frame(
6859                &frame,
6860                crate::supervise::REGISTRATION_CHANNEL_CLOSED,
6861                "the opener's launch has ended",
6862            )?]);
6863        };
6864        // The caller-side checks below read the opener's own spawn: its
6865        // module from its record and its manifest from its own registration,
6866        // never another launch's active registration under the same id.
6867        let caller_module_id = match &attested.spawn {
6868            Some((record, _)) => Some(record.module_id().to_owned()),
6869            None => match &principal {
6870                Principal::Reserved { module_id } => Some(module_id.clone()),
6871                _ => None,
6872            },
6873        };
6874
6875        // Only Callosum can attest a production phone build. Consume the claim
6876        // here so it can be kept on the route without reaching any module.
6877        let build_class = identity.build_class.take().filter(|build_class| {
6878            if caller_module_id.as_deref() == Some("callosum") {
6879                build_class == subc_protocol::BUILD_CLASS_PRODUCTION
6880            } else {
6881                if self
6882                    .build_class_warned
6883                    .lock()
6884                    .unwrap_or_else(|p| p.into_inner())
6885                    .insert(ctx.connection_id)
6886                {
6887                    warn!(connection_id = ctx.connection_id.get(), principal = %timings.principal,
6888                        "ignoring build_class from a non-Callosum opener");
6889                }
6890                false
6891            }
6892        });
6893
6894        // This is attested, control-plane policy for supervised module origins.
6895        // Keep it before route reservation and out of the opaque forwarding hot
6896        // path: data frames must never acquire a per-frame capability check.
6897        if let Some(opening_registration) = self.caller_registration(&attested, &admission_registry)
6898        {
6899            let opening_module_id = opening_registration.manifest.module_id.as_str();
6900            if let Some(capability) =
6901                denied_capability(&opening_registration.manifest, &registration.manifest)
6902            {
6903                warn!(
6904                    opening_module_id,
6905                    target_module_id,
6906                    capability,
6907                    "refusing route.open because an attested capability deny edge matches"
6908                );
6909                return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
6910                    &frame,
6911                    &target_module_id,
6912                    "capability_deny_edge",
6913                    "capability_forbidden",
6914                    format!(
6915                        "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
6916                    ),
6917                )?]);
6918            }
6919        }
6920
6921        if admission_facts.is_some() {
6922            let carrier_matches = caller_module_id.is_some()
6923                && self.admission_facts_carrier_module_id.as_deref() == caller_module_id.as_deref();
6924            if !carrier_matches {
6925                return Ok(vec![self.route_open_refusal_frame(
6926                    ctx,
6927                    &caller,
6928                    &frame,
6929                    &target_module_id,
6930                    "admission_facts_carrier_not_permitted",
6931                    "admission_facts_not_permitted",
6932                    "admission facts may only be carried by the configured reserved module",
6933                )?]);
6934            }
6935
6936            let target_allowed = self
6937                .admission_facts_targets
6938                .as_ref()
6939                .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
6940            if !target_allowed {
6941                return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
6942                    &frame,
6943                    &target_module_id,
6944                    "admission_facts_target_not_listed",
6945                    "admission_facts_target_not_allowed",
6946                    format!(
6947                        "admission facts are not permitted for target module_id '{target_module_id}'"
6948                    ),
6949                )?]);
6950            }
6951
6952            // Keep the value opaque to subc. The downstream admission validator owns
6953            // schema and semantic checks; this daemon only enforces carrier authority
6954            // and the configured destination allowlist.
6955        }
6956
6957        // Scope admission, on the attested principal above and never on the
6958        // request body. The tag read here travels with the pending bind and is
6959        // compared with the published one at commit, so a sync between here
6960        // and the module's ack refuses the open instead of binding a stamp
6961        // that is no longer true.
6962        let (bound_scope, mut scope_stamp, stamp_required) = match &scope {
6963            None => (None, None, false),
6964            Some(selector) => {
6965                let owner_configured = match &selector.owner {
6966                    // A reserved owner counts as configured from before its
6967                    // process is spawned (see `SupervisorHandle::is_configured`).
6968                    // So an owner that has not synced its scopes yet is refused
6969                    // as retryable (`scope_not_synced`), not as one that will
6970                    // never sync.
6971                    Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
6972                    _ => false,
6973                };
6974                let mut scopes = self
6975                    .scopes
6976                    .write()
6977                    .unwrap_or_else(|poisoned| poisoned.into_inner());
6978                let admitted = crate::scopes::admission::admit_route(
6979                    &scopes,
6980                    &admission_registry,
6981                    &principal,
6982                    &target_module_id,
6983                    Some(selector),
6984                    owner_configured,
6985                    crate::scopes::admission::AdmissionMode::Open,
6986                )
6987                .map(|route| {
6988                    let admission = route.scope.expect("scoped admission");
6989                    let bound = scopes.reserve_bind(
6990                        &admission,
6991                        &principal,
6992                        ctx.connection_id,
6993                        selector.hold_presence,
6994                    );
6995                    (admission, bound)
6996                });
6997                match admitted {
6998                    Ok((admission, bound)) => {
6999                        (Some(bound), Some(admission.stamp), admission.stamp_required)
7000                    }
7001                    Err(refusal) => {
7002                        return Ok(vec![self.route_open_refusal_frame(
7003                            ctx,
7004                            &caller,
7005                            &frame,
7006                            &target_module_id,
7007                            refusal.code.unwrap_or(error_codes::TARGET_UNAVAILABLE),
7008                            refusal.code.unwrap_or(error_codes::TARGET_UNAVAILABLE),
7009                            refusal.message,
7010                        )?]);
7011                    }
7012                }
7013            }
7014        };
7015        // From here on a refusal names the scope by the stamp admission
7016        // issued, exactly as the `route.open accepted` line does.
7017        if let Some(stamp) = &scope_stamp {
7018            caller.scope = route_open_scope_label(stamp);
7019        }
7020        // Decided from the admitted stamp, before any field of it is stripped
7021        // for a target that lacks the capability to read it.
7022        if scope_stamp.as_ref().is_some_and(scope_acts_for_another) {
7023            caller_capture = caller_capture.map(crate::spawn_authority::CallerCapture::relayed);
7024        }
7025
7026        // Allow resolvable missing-tail paths so a vanished directory does not
7027        // prevent reopening an existing session for cleanup. Binding does not
7028        // authorize creation of durable provider state.
7029        //
7030        // Resolve real paths rather than lexical cleanup: canonicalize the
7031        // longest existing ancestor and append the missing tail, retaining the
7032        // path's real existing ancestry even when a leaf is missing.
7033        timings.enter(RouteOpenPhase::ProjectRoot);
7034        let legacy_project = match crate::bind_compat::LegacyProject::from_open(&frame.body) {
7035            Ok(project) => project,
7036            Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
7037        };
7038        timings.enter(RouteOpenPhase::Admission);
7039        let project_root = match legacy_project.effective_root(scope_stamp.as_ref()) {
7040            Ok(root) => root,
7041            Err(err) => {
7042                return Ok(vec![control_error_frame(
7043                    &frame,
7044                    "invalid_project_root",
7045                    err.to_string(),
7046                )?])
7047            }
7048        };
7049        // Compute compatibility project fields before stripping optional stamp
7050        // members. Preserve the canonical opener root and its project ID,
7051        // including absence. For rootless openers, use stamp workdir or `/`, and
7052        // opener ID, else stamp ID, else no ID. Emit these identity fields only
7053        // to targets without bind-no-project/v1.
7054        let legacy_bind_project = legacy_project.for_target(scope_stamp.as_ref());
7055
7056        // Last gate before any relay work, and deliberately after the cheap
7057        // registry and availability checks above: those name a more precise
7058        // condition (unknown, removed, reloading) and a caller is better served
7059        // by the precise code than by this one.
7060        //
7061        // Everything below this point costs an egress permit, a reserved handle
7062        // pair and, if the module does not answer, the whole relay budget. The
7063        // reader no longer waits for that budget, so cap each target explicitly;
7064        // serial dispatch used to provide the accidental cap of one relay per
7065        // connection. Admission is a mutex-protected count and never waits.
7066        let _concurrency_guard = match self
7067            .route_bind_concurrency
7068            .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
7069        {
7070            Ok(guard) => guard,
7071            Err(in_flight) => {
7072                return Ok(vec![self.route_open_target_capacity_refusal(
7073                    ctx,
7074                    &caller,
7075                    &frame,
7076                    &target_module_id,
7077                    in_flight,
7078                )?]);
7079            }
7080        };
7081
7082        // A module that has already burned the whole budget `threshold` times
7083        // in a row does not get to charge it again until a probe says it recovered.
7084        let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
7085            RouteBindAdmission::Admitted { guard, probe } => {
7086                if probe {
7087                    info!(
7088                        module_id = %target_module_id,
7089                        connection_id = ctx.connection_id.get(),
7090                        "route.bind breaker half-open: admitting one probe"
7091                    );
7092                }
7093                guard
7094            }
7095            RouteBindAdmission::Refused {
7096                consecutive_timeouts,
7097                retry_in,
7098                probe_in_flight,
7099            } => {
7100                return Ok(vec![self.route_open_breaker_refusal_frame(
7101                    ctx,
7102                    &caller,
7103                    &frame,
7104                    &target_module_id,
7105                    consecutive_timeouts,
7106                    retry_in,
7107                    probe_in_flight,
7108                )?]);
7109            }
7110        };
7111
7112        // Resolve the per-module budget here so the wait matches the operator's
7113        // intent for this specific target. A per-module override in
7114        // `subc.jsonc` (or `with_route_bind_relay_timeouts` for embedded
7115        // daemons) wins over the daemon-wide default.
7116        let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
7117        let relay_deadline = Instant::now() + route_bind_relay_timeout;
7118        let operator_app = self
7119            .operator_apps
7120            .lock()
7121            .unwrap_or_else(|p| p.into_inner())
7122            .get(&ctx.connection_id)
7123            .cloned();
7124        timings.enter(RouteOpenPhase::EgressReservation);
7125        let pending = match self
7126            .forwarding
7127            .begin_route_bind_relay_for(
7128                ctx.connection_id,
7129                ctx.egress.clone(),
7130                response_version(&frame),
7131                frame.header.corr,
7132                &target_module_id,
7133                principal.clone(),
7134                bound_scope,
7135                project_root,
7136                identity.harness.clone(),
7137                build_class,
7138                operator_app.clone(),
7139                crate::forwarding::RouteAuthority {
7140                    incarnation: self.supervisor.daemon_incarnation(),
7141                    caller: caller_capture
7142                        .as_ref()
7143                        .map(|caller| caller.authority.pair()),
7144                },
7145                relay_deadline,
7146                busy_refusals == Some(true),
7147                accepts_close_reasons,
7148            )
7149            .await
7150        {
7151            Ok(pending) => pending,
7152            Err(err) => {
7153                return Ok(vec![self.route_open_refusal_frame(
7154                    ctx,
7155                    &caller,
7156                    &frame,
7157                    &target_module_id,
7158                    "relay_reservation_failed",
7159                    forwarding_error_code(&err),
7160                    err.to_string(),
7161                )?])
7162            }
7163        };
7164        timings.enter(RouteOpenPhase::Admission);
7165        let crate::forwarding::PendingRouteBindRelay {
7166            endpoint,
7167            module_sink,
7168            negotiated_ver,
7169            client_channel,
7170            client_epoch,
7171            module_channel,
7172            module_epoch,
7173            corr: relay_corr,
7174            receiver,
7175        } = pending;
7176        let mut reservation =
7177            RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
7178
7179        // Reserving egress can wait while a module reconnects or a swap cuts
7180        // over. Check the relay's captured connection, not the earlier by-id
7181        // lookup: the original module's flow/run capabilities cannot authorize
7182        // its replacement. The captured sink stays bound to that connection.
7183        let relay_registration = self
7184            .registry
7185            .get_module_by_connection(endpoint.connection_id)
7186            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
7187        let relay_capabilities = relay_registration
7188            .as_ref()
7189            .and_then(|registration| registration.manifest.capabilities.as_ref());
7190        if let Some(selector) = &scope {
7191            // Egress reservation may wait across a scope update. Recheck the
7192            // proof against the current record before sending the captured bind.
7193            if let Err(refusal) = self
7194                .scopes
7195                .read()
7196                .unwrap_or_else(|poisoned| poisoned.into_inner())
7197                .check_bind_proof(&principal, selector)
7198            {
7199                reservation.release_and_disarm();
7200                return Ok(vec![self.route_open_refusal_frame(
7201                    ctx,
7202                    &caller,
7203                    &frame,
7204                    &target_module_id,
7205                    refusal.code,
7206                    refusal.code,
7207                    refusal.message,
7208                )?]);
7209            }
7210        }
7211        if let Some(stamp) = scope_stamp.as_ref() {
7212            if let Err(refusal) = crate::scopes::check_target_required_stamp(
7213                stamp_required,
7214                &target_module_id,
7215                relay_capabilities,
7216            )
7217            .and_then(|()| {
7218                crate::scopes::check_target_flow_support(
7219                    stamp,
7220                    &target_module_id,
7221                    relay_capabilities,
7222                )
7223            })
7224            .and_then(|()| {
7225                crate::scopes::check_target_agent_run_support(
7226                    stamp,
7227                    &target_module_id,
7228                    relay_capabilities,
7229                )
7230            }) {
7231                reservation.release_and_disarm();
7232                return Ok(vec![self.route_open_refusal_frame(
7233                    ctx,
7234                    &caller,
7235                    &frame,
7236                    &target_module_id,
7237                    refusal.code,
7238                    refusal.code,
7239                    refusal.message,
7240                )?]);
7241            }
7242        }
7243
7244        // Older readers may reject unknown fields. Select optional fields from
7245        // the captured endpoint's declaration, including on unscoped binds.
7246        if scope_stamp
7247            .as_ref()
7248            .is_some_and(|stamp| stamp.kind == subc_protocol::scope::ScopeKind::Federated)
7249            && !crate::scopes::target_supports_federated(relay_capabilities)
7250        {
7251            // Keep bound_scope: even an unstamped legacy route belongs to this epoch.
7252            scope_stamp = None;
7253        }
7254        if !relay_capabilities.is_some_and(|caps| {
7255            caps.provides
7256                .iter()
7257                .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
7258        }) {
7259            if let Some(stamp) = scope_stamp.as_mut() {
7260                stamp.workdir = None;
7261                stamp.project_id = None;
7262            }
7263        }
7264        // A worker run has no agent_id; on_behalf_of_agent is its only agent
7265        // attribution. The run gate above has already required the target to
7266        // declare `worker-runs/v1`, which promises to read both
7267        // `on_behalf_of_agent` and `call_mode`.
7268        let worker_run = scope_stamp
7269            .as_ref()
7270            .is_some_and(crate::scopes::is_worker_run)
7271            && relay_capabilities.is_some_and(|caps| {
7272                caps.provides
7273                    .iter()
7274                    .any(|cap| cap == subc_protocol::scope::WORKER_RUNS_CAPABILITY)
7275            });
7276        if !worker_run
7277            && !relay_capabilities.is_some_and(|caps| {
7278                caps.provides
7279                    .iter()
7280                    .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
7281            })
7282        {
7283            if let Some(stamp) = scope_stamp.as_mut() {
7284                stamp.attributes.on_behalf_of_agent = None;
7285            }
7286        }
7287        if !relay_capabilities.is_some_and(|caps| {
7288            caps.provides
7289                .iter()
7290                .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
7291        }) {
7292            if let Some(stamp) = scope_stamp.as_mut() {
7293                stamp.attributes.flow_version = None;
7294                stamp.attributes.flow_code_hash = None;
7295                stamp.attributes.flow_install_id = None;
7296            }
7297        }
7298        let call_mode = if worker_run
7299            || relay_capabilities
7300                .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
7301        {
7302            Some(derive_call_mode(scope_stamp.as_ref()))
7303        } else {
7304            if let Some(stamp) = scope_stamp.as_mut() {
7305                stamp.expires_at_ms = None;
7306            }
7307            None
7308        };
7309
7310        debug!(
7311            connection_id = ctx.connection_id.get(),
7312            client_channel,
7313            client_epoch,
7314            module_channel,
7315            module_epoch,
7316            "reserved route handle pair"
7317        );
7318        // Rendered BEFORE the move into the relay, because the accept arm below
7319        // is where it is logged and the principal is gone by then.
7320        let route_principal_label = principal_label(&principal);
7321        let relay = ModuleControlRequest::RouteBind {
7322            route_channel: module_channel,
7323            epoch: module_epoch,
7324            target,
7325            identity,
7326            principal: Some(principal),
7327            consumer_capabilities,
7328            role_versions,
7329            admission_facts,
7330            scope: scope_stamp,
7331            call_mode,
7332        };
7333        let peer = RouteBindPeer::new()
7334            .with_observed_peer(
7335                relay_capabilities
7336                    .is_some_and(|caps| {
7337                        caps.provides
7338                            .iter()
7339                            .any(|cap| cap == OBSERVED_PEER_CAPABILITY)
7340                    })
7341                    .then(|| {
7342                        self.observed_peers
7343                            .lock()
7344                            .unwrap_or_else(|p| p.into_inner())
7345                            .get(&ctx.connection_id)
7346                            .cloned()
7347                            .unwrap_or_else(ObservedPeer::unobserved)
7348                    }),
7349            )
7350            .with_operator_app(
7351                self.connection_operator_app(operator_app.as_ref())
7352                    .filter(|_| {
7353                        relay_capabilities.is_some_and(|caps| {
7354                            caps.provides
7355                                .iter()
7356                                .any(|cap| cap == OPERATOR_APP_CAPABILITY)
7357                        })
7358                    }),
7359            )
7360            .with_caller(caller_for_target(
7361                caller_capture.as_ref(),
7362                relay_capabilities,
7363                &target_module_id,
7364            ));
7365        let relay_body = crate::bind_compat::encode_bind(
7366            &relay,
7367            &peer,
7368            &legacy_bind_project,
7369            relay_capabilities,
7370        )
7371        .map_err(|err| {
7372            RouterError::backend(
7373                0,
7374                frame.header.corr,
7375                format!("failed to encode route.bind request: {err}"),
7376            )
7377        })?;
7378        let relay_frame = Frame::build_with_version(
7379            negotiated_ver,
7380            FrameType::Request,
7381            control_flags(),
7382            0,
7383            0,
7384            relay_corr,
7385            relay_body,
7386        )
7387        .map_err(RouterError::FrameBuild)?;
7388
7389        timings.enter(RouteOpenPhase::BindSend);
7390        let sent = module_sink.send(relay_frame).await;
7391        timings.enter(RouteOpenPhase::Admission);
7392        if let Err(err) = sent {
7393            reservation.release_and_disarm();
7394            return Ok(vec![self.route_open_refusal_frame(
7395                ctx,
7396                &caller,
7397                &frame,
7398                &target_module_id,
7399                "relay_send_failed",
7400                "target_unavailable",
7401                err.to_string(),
7402            )?]);
7403        }
7404
7405        #[cfg(test)]
7406        if let Some(hold) = &self.route_bind_relay_sent_hold {
7407            // First wait: tell the test the bind is queued. Second wait: stay
7408            // here until the test has driven whatever it wants to happen
7409            // between queueing the bind and marking it enqueued.
7410            hold.wait().await;
7411            hold.wait().await;
7412        }
7413
7414        // A missing relay is not by itself an abandoned bind: the module may
7415        // already have acked it and the route may be committed. Only a relay
7416        // dropped without any module reply needs this GOODBYE; one a reply
7417        // settled has already had its GOODBYE decided by the reply path.
7418        if self
7419            .forwarding
7420            .mark_route_bind_relay_enqueued(endpoint, relay_corr)
7421            .map_err(RouterError::Forwarding)?
7422            == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
7423        {
7424            self.send_abandoned_route_bind_goodbye(
7425                &module_sink,
7426                negotiated_ver,
7427                module_channel,
7428                module_epoch,
7429            );
7430        }
7431
7432        timings.enter(RouteOpenPhase::BindAckWait);
7433        let outcome = timeout_at(relay_deadline, receiver).await;
7434        timings.enter(RouteOpenPhase::Admission);
7435        match outcome {
7436            Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
7437                reservation.disarm();
7438                if breaker.record_accepted() {
7439                    info!(
7440                        module_id = %target_module_id,
7441                        "route.bind breaker closed: the probe was accepted"
7442                    );
7443                }
7444                self.observe_route_open_accept(
7445                    ctx,
7446                    &target_module_id,
7447                    &route_principal_label,
7448                    &caller.scope,
7449                );
7450                Ok(Vec::new())
7451            }
7452            Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
7453                reservation.release_and_disarm();
7454                // A module that says no in microseconds is healthy. Rejection
7455                // is a different condition with its own refusal and must not
7456                // move the breaker.
7457                breaker.record_inconclusive();
7458                self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, false)
7459            }
7460            Ok(Ok(RouteBindRelayOutcome::DaemonRejected(body))) => {
7461                reservation.release_and_disarm();
7462                breaker.record_inconclusive();
7463                self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, true)
7464            }
7465            Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
7466                reservation.release_and_disarm();
7467                breaker.record_inconclusive();
7468                // Fires when the module's connection closes while a relayed
7469                // bind is pending -- typically a caller racing a module restart
7470                // whose bind was relayed BEFORE the drain mark went up. Logged
7471                // because the caller sees only its own error and the fleet has
7472                // already spent one diagnosis round unable to tell this arm
7473                // from a relay timeout without daemon-side evidence.
7474                tracing::warn!(
7475                    module_id = %target_module_id,
7476                    "route.bind relay abandoned: {message}"
7477                );
7478                Ok(vec![self.route_open_refusal_frame(
7479                    ctx,
7480                    &caller,
7481                    &frame,
7482                    &target_module_id,
7483                    "relay_abandoned",
7484                    "target_unavailable",
7485                    message,
7486                )?])
7487            }
7488            Ok(Err(_)) => {
7489                reservation.release_and_disarm();
7490                breaker.record_inconclusive();
7491                Ok(vec![self.route_open_refusal_frame(
7492                    ctx,
7493                    &caller,
7494                    &frame,
7495                    &target_module_id,
7496                    "relay_waiter_canceled",
7497                    "target_unavailable",
7498                    "route.bind relay waiter was canceled before the module responded",
7499                )?])
7500            }
7501            Err(_) => {
7502                reservation.release_and_disarm();
7503                // THE ONLY ARM THAT MOVES THE BREAKER. Budget exhausted with no
7504                // answer at all is the one condition a fast refusal can
7505                // usefully stand in for; every other arm already answered.
7506                if let Some(opened) = breaker.record_timeout(
7507                    self.route_bind_breaker_threshold,
7508                    self.route_bind_breaker_cooldown,
7509                ) {
7510                    warn!(
7511                        module_id = %target_module_id,
7512                        consecutive_timeouts = opened.consecutive_timeouts,
7513                        cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
7514                        reopened_after_probe = opened.reopened_after_probe,
7515                        "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
7516                    );
7517                }
7518                // The generous budget just burned to no answer: the module is
7519                // registered and its connection is up, but its bind handler sat
7520                // on the ack for the full budget (warm-on-bind, cold configure,
7521                // or a wedged handler). Every earlier unavailability shape
7522                // fast-refuses BEFORE the relay, so this arm firing means the
7523                // slowness is module-side -- log it so the per-module timeline
7524                // is reconstructable without client audit rows.
7525                tracing::warn!(
7526                    module_id = %target_module_id,
7527                    timeout_ms = route_bind_relay_timeout.as_millis() as u64,
7528                    "route.bind relay timed out: module did not ack within budget"
7529                );
7530                Ok(vec![self.route_open_refusal_frame(
7531                    ctx,
7532                    &caller,
7533                    &frame,
7534                    &target_module_id,
7535                    "relay_timed_out",
7536                    "module_timeout",
7537                    format!(
7538                        "module_id '{target_module_id}' did not answer route.bind within {:?}",
7539                        route_bind_relay_timeout
7540                    ),
7541                )?])
7542            }
7543        }
7544    }
7545
7546    fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
7547        let response = ClientControlResponse::SupervisorSpawnSnapshot {
7548            snapshot: self.supervisor.spawn_snapshot(),
7549        };
7550        Ok(vec![control_response_body_frame(
7551            &frame,
7552            &response,
7553            "ClientControlResponse::SupervisorSpawnSnapshot",
7554        )?])
7555    }
7556
7557    fn handle_supervisor_spawn_subscribe(
7558        &self,
7559        ctx: &RouteCtx,
7560        frame: Frame,
7561        since: Option<SpawnCursor>,
7562    ) -> Result<Vec<Frame>, RouterError> {
7563        match self.supervisor.subscribe_spawns(
7564            ctx.connection_id,
7565            frame.header.corr,
7566            response_version(&frame),
7567            since,
7568            ctx.egress.clone(),
7569        ) {
7570            Ok(()) => Ok(Vec::new()),
7571            Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
7572                Ok(vec![control_error_body_frame(
7573                    &frame,
7574                    ErrorBody {
7575                        code: "spawn_cursor_incarnation_mismatch".to_string(),
7576                        message: "spawn cursor belongs to a different daemon incarnation"
7577                            .to_string(),
7578                        detail: Some(serde_json::json!({
7579                            "current_daemon_incarnation": current
7580                        })),
7581                    },
7582                )?])
7583            }
7584            Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
7585                &frame,
7586                ErrorBody {
7587                    code: "spawn_cursor_too_old".to_string(),
7588                    message: "spawn cursor predates the retained event ring".to_string(),
7589                    detail: Some(serde_json::json!({
7590                        "oldest_retained_cursor": oldest
7591                    })),
7592                },
7593            )?]),
7594            Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
7595                0,
7596                frame.header.corr,
7597                format!("failed to open supervisor spawn subscription: {error}"),
7598            )),
7599        }
7600    }
7601
7602    async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
7603        // Status/configuration snapshots may wait on synchronous supervisor
7604        // locks. Keep that work off the connection's async runtime thread.
7605        let handler = self.clone();
7606        let runtime = tokio::runtime::Handle::current();
7607        // The blocking pool does not inherit the control request's tracing
7608        // subscriber. Preserve it so slow snapshot-lock warnings stay visible.
7609        let subscriber = tracing::dispatcher::get_default(Clone::clone);
7610        let corr = frame.header.corr;
7611        tokio::task::spawn_blocking(move || {
7612            runtime.block_on(
7613                list_diagnostics::scope(handler.handle_supervisor_list_inner(frame))
7614                    .with_subscriber(subscriber),
7615            )
7616        })
7617        .await
7618        .map_err(|error| RouterError::backend(0, corr, error.to_string()))?
7619    }
7620
7621    async fn handle_supervisor_list_inner(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
7622        use crate::config_gate::runtime::ApprovedDaemonView;
7623        let publication = self
7624            .config_runtime
7625            .as_ref()
7626            .and_then(|owner| owner.polled_snapshot().ok());
7627        let approved = publication
7628            .as_ref()
7629            .map(|view| view.approved_daemon.clone())
7630            .unwrap_or_else(|| {
7631                if self.config_runtime.is_some() {
7632                    ApprovedDaemonView::Unavailable
7633                } else {
7634                    ApprovedDaemonView::NotStamped
7635                }
7636            });
7637        let generation = self
7638            .registry
7639            .generation()
7640            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
7641        let supervised = self.supervisor.list();
7642        let mut statuses = BTreeMap::new();
7643        let mut not_running = BTreeSet::new();
7644        for module in &supervised {
7645            let status = module.status_for_control("list").map_err(|err| {
7646                RouterError::backend(
7647                    0,
7648                    frame.header.corr,
7649                    format!("failed to read supervisor status: {err}"),
7650                )
7651            })?;
7652            if !status.enabled || !status.process_alive {
7653                not_running.insert(module.module_id().to_owned());
7654            }
7655            statuses.insert(module.module_id().to_owned(), status);
7656        }
7657        // Legacy per-process diagnostics keep their shared optional budget.
7658        // The cached tree and accounting export below does not spend it.
7659        let mut process_resources = BTreeMap::new();
7660        for module in &supervised {
7661            process_resources.insert(
7662                module.module_id().to_owned(),
7663                crate::subtree_resources::read_process(module.clone()).await,
7664            );
7665        }
7666        // Image evidence uses the same optional deadline, but an exhausted
7667        // budget must not suppress the committed resource view.
7668        let mut image_readings = BTreeMap::new();
7669        for module in &supervised {
7670            image_readings.insert(
7671                module.module_id().to_owned(),
7672                module.running_image_agreement().await,
7673            );
7674        }
7675        let readings = self.resource_tracker.list(supervised.clone());
7676        let mut modules = Vec::new();
7677        for module in supervised {
7678            let status = statuses
7679                .remove(module.module_id())
7680                .expect("each supervised module has a status snapshot");
7681            let (configured, _) = module.configuration().map_err(|err| {
7682                RouterError::backend(
7683                    0,
7684                    frame.header.corr,
7685                    format!("failed to read module configuration: {err}"),
7686                )
7687            })?;
7688            let next_retry_at_ms = module.next_retry_at_ms().map_err(|err| {
7689                RouterError::backend(
7690                    0,
7691                    frame.header.corr,
7692                    format!("failed to read retry time: {err}"),
7693                )
7694            })?;
7695            let image = image_readings
7696                .remove(module.module_id())
7697                .expect("each supervised module has an image snapshot");
7698            let resources = process_resources.remove(module.module_id());
7699            let mut verdict =
7700                reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
7701            verdict.spec = Some(launch_spec_agreement(
7702                &configured,
7703                status.process_alive,
7704                status.launch_spec_digest.as_ref(),
7705            ));
7706            let pending_reload = Some(verdict);
7707            let pending_entry = match &approved {
7708                ApprovedDaemonView::Approved { config, .. } => config
7709                    .modules
7710                    .iter()
7711                    .find(|m| m.module_id == module.module_id()),
7712                _ => None,
7713            };
7714            let pending = self
7715                .pending_module_fields(
7716                    &module,
7717                    pending_entry,
7718                    false,
7719                    Self::registration_channel_in_force(
7720                        publication
7721                            .as_ref()
7722                            .and_then(|view| view.registration_declaration(module.module_id()))
7723                            .as_deref(),
7724                        pending_entry.map(|entry| entry.registration_channel),
7725                    ),
7726                )
7727                .map_err(|err| RouterError::backend(0, frame.header.corr, err))?;
7728            let script = configured.script.as_ref().map(|entry| {
7729                let approved = self
7730                    .config_runtime
7731                    .as_ref()
7732                    .and_then(|owner| owner.approved_script(&status.module_id, &entry.path));
7733                crate::supervise::script::status(entry, approved.as_ref())
7734            });
7735            let configured_enabled = match &approved {
7736                ApprovedDaemonView::Approved {
7737                    config: approved, ..
7738                } => approved
7739                    .modules
7740                    .iter()
7741                    .find(|module| module.module_id == status.module_id)
7742                    .map(|module| module.enabled),
7743                ApprovedDaemonView::NotStamped => self
7744                    .capability_evaluator
7745                    .configured_enabled(&status.module_id),
7746                ApprovedDaemonView::Unavailable => None,
7747            };
7748            let admission_refused = status
7749                .admission_refused
7750                .clone()
7751                .filter(|_| status.enabled && !status.process_alive)
7752                .filter(|_| {
7753                    publication
7754                        .as_ref()
7755                        .and_then(|view| view.registration_declaration(&status.module_id))
7756                        .as_deref()
7757                        == Some("undeclared")
7758                });
7759            // Keep the retired policy field on the wire for one release so
7760            // existing status consumers still receive the platform policy.
7761            let mut entry = SupervisorEntry::new(
7762                status.module_id,
7763                status.state.to_string(),
7764                status.enabled,
7765                status.live,
7766                status.health.status,
7767            )
7768            .with_configured_enabled(configured_enabled)
7769            .with_script(script)
7770            .with_admission_refused(admission_refused)
7771            .with_next_retry_at_ms(next_retry_at_ms)
7772            .with_launch_nonce_env(Some(!cfg!(unix)))
7773            .with_launch_nonce_source(status.launch_nonce_source)
7774            .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
7775            .with_protocol(status.protocol)
7776            .with_pending_reload(pending_reload)
7777            .with_last_probe_ms(status.health.last_probe_ms)
7778            .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
7779            .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
7780            .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
7781            .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
7782            .with_restart_count(Some(status.restart_count))
7783            .with_max_restarts(Some(status.max_restarts))
7784            .with_lifetime_restarts(Some(status.lifetime_restarts))
7785            .with_spawn_generation(Some(status.spawn_generation))
7786            .with_restart_window_secs(Some(status.restart_window.as_secs()))
7787            .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
7788            .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
7789            .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
7790            .with_resources(resources);
7791            entry.pending = pending;
7792            modules.push(entry);
7793        }
7794        for module in &mut modules {
7795            if let Some((tree, hour)) = readings.modules.get(&module.module_id) {
7796                module.tree = tree.clone();
7797                module.cpu_last_hour = hour.clone();
7798            }
7799            module.tree_unavailable = readings
7800                .reasons
7801                .get(&module.module_id)
7802                .map(|reason| (*reason).to_owned());
7803            if module.tree.is_none() {
7804                module.tree_unavailable = Some(
7805                    if not_running.contains(&module.module_id) {
7806                        "not_running"
7807                    } else if let Some(reason) = readings.reasons.get(&module.module_id) {
7808                        reason
7809                    } else {
7810                        "not_yet_measured"
7811                    }
7812                    .to_owned(),
7813                );
7814            }
7815        }
7816        let response = ClientControlResponse::SupervisorList {
7817            generation,
7818            modules,
7819            daemon: readings.daemon,
7820            agent_commands: readings.agent_commands,
7821        };
7822        Ok(vec![control_response_body_frame(
7823            &frame,
7824            &response,
7825            "ClientControlResponse::SupervisorList",
7826        )?])
7827    }
7828
7829    fn handle_supervisor_stderr_tail(
7830        &self,
7831        frame: Frame,
7832        module_id: String,
7833        max_lines: Option<u32>,
7834        max_bytes: Option<u32>,
7835    ) -> Result<Vec<Frame>, RouterError> {
7836        let Some(module) = self.supervisor.get(&module_id) else {
7837            return Ok(vec![control_error_frame(
7838                &frame,
7839                "unknown_module",
7840                format!("module_id '{module_id}' is not supervised"),
7841            )?]);
7842        };
7843
7844        let snapshot = module.stderr_tail(
7845            max_lines.map(|value| value as usize),
7846            max_bytes.map(|value| value as usize),
7847        );
7848
7849        let response = ClientControlResponse::SupervisorStderrTail {
7850            module_id,
7851            tail: StderrTail {
7852                capture: match snapshot.capture {
7853                    CaptureState::Captured => StderrCaptureState::Captured,
7854                    CaptureState::Incomplete { reason } => {
7855                        StderrCaptureState::Incomplete { reason }
7856                    }
7857                    CaptureState::NotCaptured { reason } => {
7858                        StderrCaptureState::NotCaptured { reason }
7859                    }
7860                },
7861                entries: snapshot
7862                    .entries
7863                    .into_iter()
7864                    .map(|entry| match entry {
7865                        TailEntry::Line {
7866                            text,
7867                            truncated,
7868                            at_ms,
7869                        } => StderrTailEntry::Line {
7870                            text,
7871                            truncated,
7872                            at_ms,
7873                        },
7874                        TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
7875                    })
7876                    .collect(),
7877                dropped_lines: snapshot.dropped_lines,
7878            },
7879        };
7880        Ok(vec![control_response_body_frame(
7881            &frame,
7882            &response,
7883            "ClientControlResponse::SupervisorStderrTail",
7884        )?])
7885    }
7886
7887    async fn observe_control_wait<F: std::future::Future>(
7888        &self,
7889        ctx: &RouteCtx,
7890        target: Option<&str>,
7891        kind: &'static str,
7892        future: F,
7893    ) -> F::Output {
7894        self.counters
7895            .observe_reader_wait(
7896                || {
7897                    let module = self
7898                        .forwarding
7899                        .module_id_for_connection(ctx.connection_id)
7900                        .ok()
7901                        .flatten();
7902                    let principal = if module.is_some() { "module" } else { "direct" };
7903                    (
7904                        self.resolve_reader_wait_target(kind, target)
7905                            .or(module)
7906                            .unwrap_or_else(|| crate::observability::CONTROL_WAIT_BUCKET.into()),
7907                        principal.into(),
7908                    )
7909                },
7910                kind,
7911                future,
7912            )
7913            .await
7914    }
7915
7916    /// Resolve the reader-wait module ID only from supervisor or registry
7917    /// records; never attribute a wait from the request selector alone.
7918    pub(crate) fn resolve_reader_wait_target(
7919        &self,
7920        op: &str,
7921        selector: Option<&str>,
7922    ) -> Option<String> {
7923        let selector = selector?;
7924        match op {
7925            ops::SUPERVISOR_TERMINALS | ops::SUPERVISOR_PROVENANCE => self
7926                .supervisor
7927                .get(selector)
7928                .map(|module| module.module_id().to_owned()),
7929            ops::SUPERVISOR_HEALTH_PROBE => self
7930                .registry
7931                .get_module(selector)
7932                .ok()
7933                .flatten()
7934                .map(|registration| registration.manifest.module_id),
7935            _ => None,
7936        }
7937    }
7938
7939    async fn handle_supervisor_terminals(
7940        &self,
7941        ctx: &RouteCtx,
7942        frame: Frame,
7943        module_id: String,
7944    ) -> Result<Vec<Frame>, RouterError> {
7945        let Some(module) = self.supervisor.get(&module_id) else {
7946            return Ok(vec![control_error_frame(
7947                &frame,
7948                "unknown_module",
7949                format!("module_id '{module_id}' is not supervised"),
7950            )?]);
7951        };
7952
7953        // The journal read runs on a blocking thread: it can be megabytes of
7954        // file I/O and must not occupy a runtime worker.
7955        let terminals = self
7956            .observe_control_wait(
7957                ctx,
7958                Some(module.module_id()),
7959                ops::SUPERVISOR_TERMINALS,
7960                module.read_durable_terminal_history(),
7961            )
7962            .await
7963            .map_err(|error| {
7964                RouterError::backend(
7965                    0,
7966                    frame.header.corr,
7967                    format!("failed to read terminal history: {error}"),
7968                )
7969            })?;
7970        let response = ClientControlResponse::SupervisorTerminals {
7971            module_id,
7972            terminals,
7973        };
7974        Ok(vec![control_response_body_frame(
7975            &frame,
7976            &response,
7977            "ClientControlResponse::SupervisorTerminals",
7978        )?])
7979    }
7980
7981    fn handle_supervisor_routes(
7982        &self,
7983        frame: Frame,
7984        module_id: Option<String>,
7985    ) -> Result<Vec<Frame>, RouterError> {
7986        let modules = self
7987            .forwarding
7988            .route_census(module_id.as_deref())
7989            .map_err(RouterError::Forwarding)?
7990            .into_iter()
7991            .map(|(module_id, routes)| SupervisorRouteModule {
7992                module_id,
7993                routes: routes
7994                    .into_iter()
7995                    .map(|route| SupervisorRoute {
7996                        consumer: match route.principal {
7997                            Principal::Reserved { module_id } => {
7998                                SupervisorRouteConsumer::Reserved { module_id }
7999                            }
8000                            Principal::Direct | Principal::Unverified => {
8001                                SupervisorRouteConsumer::Direct {
8002                                    connection_id: route.goodbye_target.connection_id.get(),
8003                                }
8004                            }
8005                        },
8006                        age_ms: Instant::now()
8007                            .saturating_duration_since(route.bound_at)
8008                            .as_millis()
8009                            .try_into()
8010                            .unwrap_or(u64::MAX),
8011                        draining: route.draining,
8012                        drain_reason: route.drain_reason,
8013                    })
8014                    .collect(),
8015            })
8016            .collect();
8017        let response = ClientControlResponse::SupervisorRoutes { modules };
8018        Ok(vec![control_response_body_frame(
8019            &frame,
8020            &response,
8021            "ClientControlResponse::SupervisorRoutes",
8022        )?])
8023    }
8024
8025    fn running_registration_method(
8026        &self,
8027        module: &str,
8028    ) -> Option<subc_control::RegistrationMethod> {
8029        let registration = self.registry.get_module(module).ok()??;
8030        let connection = registration.connection_id;
8031        let channel = self.supervisor.registration_spawn(connection).is_some();
8032        let nonce = if channel {
8033            None
8034        } else {
8035            self.hello_launch_nonces
8036                .lock()
8037                .ok()?
8038                .nonce(connection)
8039                .map(str::to_owned)
8040        };
8041        if !channel && nonce.is_none() {
8042            return None;
8043        }
8044        self.supervisor
8045            .registration_snapshot_is_current_live(
8046                &self.registry,
8047                module,
8048                connection,
8049                nonce.as_deref(),
8050            )
8051            .then_some(if channel {
8052                subc_control::RegistrationMethod::Channel
8053            } else {
8054                subc_control::RegistrationMethod::Nonce
8055            })
8056    }
8057
8058    async fn handle_supervisor_provenance(
8059        &self,
8060        ctx: &RouteCtx,
8061        frame: Frame,
8062        module_id: Option<String>,
8063    ) -> Result<Vec<Frame>, RouterError> {
8064        let targeted = module_id.is_some();
8065        let selected = if let Some(module_id) = module_id {
8066            let Some(module) = self.supervisor.get(&module_id) else {
8067                return Ok(vec![control_error_frame(
8068                    &frame,
8069                    "unknown_module",
8070                    format!("module_id '{module_id}' is not supervised"),
8071                )?]);
8072            };
8073            vec![module]
8074        } else {
8075            self.supervisor.list()
8076        };
8077
8078        let target = targeted.then(|| selected[0].module_id());
8079        let declarations = if let Some(owner) = &self.config_runtime {
8080            owner
8081                .on_blocking(|owner| owner.registration_declarations())
8082                .await
8083                .ok()
8084        } else {
8085            None
8086        };
8087        self.observe_control_wait(ctx, target, ops::SUPERVISOR_PROVENANCE, async {
8088            let mut modules = Vec::with_capacity(selected.len());
8089            let mut extensions = Vec::with_capacity(selected.len());
8090            for module in &selected {
8091                let (status, observed_image) = module
8092                    .status_and_running_image_agreement()
8093                    .await
8094                    .map_err(|err| {
8095                        RouterError::backend(
8096                            0,
8097                            frame.header.corr,
8098                            format!("failed to read supervisor status: {err}"),
8099                        )
8100                    })?;
8101                let module_declared = self
8102                    .registry
8103                    .get_module(&status.module_id)
8104                    .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
8105                    .and_then(|registration| registration.manifest.provenance)
8106                    .map(|build| ModuleDeclaredProvenance::Reported { build })
8107                    .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
8108                #[cfg(test)]
8109                let running_image = match &self.provenance_probe_override {
8110                    Some(result) => result.clone(),
8111                    None => observed_image,
8112                };
8113                #[cfg(not(test))]
8114                let running_image = observed_image;
8115                let registration = self.running_registration_method(&status.module_id);
8116                let mut extension = subc_control::SupervisorModuleProvenanceExt::new()
8117                    .with_registration(registration);
8118                extension.registration_declaration = declarations
8119                    .as_ref()
8120                    .and_then(|entries| entries.get(&status.module_id))
8121                    .cloned();
8122                if status.process_alive && status.registration_declaration.is_some() {
8123                    extension.registration_declaration = status.registration_declaration.clone();
8124                }
8125                extensions.push(extension);
8126                modules.push(SupervisorModuleProvenance {
8127                    module_id: status.module_id,
8128                    module_declared,
8129                    daemon_observed: SupervisorObservedProcess {
8130                        launch_nonce_source: status.launch_nonce_source,
8131                        pid: status.pid,
8132                        spawned_at_ms: status.spawned_at_ms,
8133                        spawned_from: status.spawned_from,
8134                        running_image,
8135                    },
8136                });
8137            }
8138            let daemon = SupervisorDaemonProvenance {
8139                daemon_build: self.daemon_provenance.build.clone(),
8140                daemon_observed: DaemonObservedProcess {
8141                    pid: self.daemon_provenance.pid,
8142                    started_at_ms: self
8143                        .daemon_provenance
8144                        .start_clock
8145                        .map(|clock| clock.started_at_ms())
8146                        .or(self.daemon_provenance.started_at_ms),
8147                    running_image: self
8148                        .daemon_provenance
8149                        .probe
8150                        .observe(
8151                            self.daemon_provenance.pid,
8152                            self.daemon_provenance.executable_path.as_deref(),
8153                            self.daemon_provenance.executable_identity.clone(),
8154                            self.daemon_provenance.process_start_time,
8155                        )
8156                        .await,
8157                },
8158            };
8159            let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
8160            let mut response = serde_json::to_value(response)
8161                .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
8162            for (entry, extension) in response["modules"]
8163                .as_array_mut()
8164                .expect("provenance modules are an array")
8165                .iter_mut()
8166                .zip(extensions)
8167            {
8168                extension.write_to(entry).map_err(|error| {
8169                    RouterError::backend(0, frame.header.corr, error.to_string())
8170                })?;
8171            }
8172            Ok(vec![control_response_body_frame(
8173                &frame,
8174                &response,
8175                "ClientControlResponse::SupervisorProvenance",
8176            )?])
8177        })
8178        .await
8179    }
8180
8181    fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
8182        self.refresh_capability_requirements();
8183        let publication = self
8184            .config_runtime
8185            .as_ref()
8186            .and_then(|owner| owner.polled_snapshot().ok());
8187        let generation = self
8188            .registry
8189            .generation()
8190            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
8191        let modules = self
8192            .supervisor
8193            .list()
8194            .into_iter()
8195            .map(|module| {
8196                let status = module.status_for_control("health").map_err(|err| {
8197                    RouterError::backend(
8198                        0,
8199                        frame.header.corr,
8200                        format!("failed to read supervisor health: {err}"),
8201                    )
8202                })?;
8203                let module_id = status.module_id;
8204                let capability_detail = self
8205                    .capability_evaluator
8206                    .required_problem_detail(&module_id);
8207                let declaration_warning = self.registry.get_module(&module_id).ok().flatten()
8208                    .filter(|registration| registration.manifest.registration_channel == Some(false))
8209                    .filter(|_| publication.as_ref().and_then(|view| view.registration_declaration(&module_id))
8210                        .as_deref().map(|next| if status.process_alive { status.registration_declaration.as_deref().unwrap_or(next) } else { next }) == Some("declared"))
8211                    .map(|_| format!("{module_id}: HELLO registration_channel is false despite the approved channel declaration"));
8212                let capability_detail = append_capability_problem_detail(capability_detail, declaration_warning);
8213                Ok(SupervisorHealthEntry {
8214                    module_id,
8215                    status: status.health.status,
8216                    detail: append_capability_problem_detail(
8217                        status.health.detail,
8218                        capability_detail,
8219                    ),
8220                    metrics: status.health.metrics,
8221                    consecutive_failures: status.health.consecutive_failures,
8222                    late_answer_count: status.health.late_answer_count,
8223                    last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
8224                    last_action: status.health.last_action,
8225                    last_action_ms: status.health.last_action_ms,
8226                    last_probe_ms: status.health.last_probe_ms,
8227                })
8228            })
8229            .collect::<Result<Vec<_>, RouterError>>()?;
8230        let response = ClientControlResponse::SupervisorHealth {
8231            generation,
8232            modules,
8233        };
8234        Ok(vec![control_response_body_frame(
8235            &frame,
8236            &response,
8237            "ClientControlResponse::SupervisorHealth",
8238        )?])
8239    }
8240
8241    async fn handle_supervisor_restart(
8242        &self,
8243        frame: Frame,
8244        module_id: String,
8245        drain_timeout_ms: Option<u64>,
8246        requester: Principal,
8247    ) -> Result<Vec<Frame>, RouterError> {
8248        let operation_lock = self.supervisor.operation_lock();
8249        let _operation_guard = operation_lock.lock().await;
8250        let Some(module) = self.supervisor.get(&module_id) else {
8251            return Ok(vec![control_error_frame(
8252                &frame,
8253                "unknown_module",
8254                format!("module_id '{module_id}' is not supervised"),
8255            )?]);
8256        };
8257
8258        self.route_outages.mark_operator_action(&module_id);
8259        if let Err(err) = module
8260            .restart_requested(drain_timeout_ms, Some(requester))
8261            .await
8262        {
8263            self.route_outages
8264                .operator_action_ended_unrefused(&module_id);
8265            let (code, message) = match err {
8266                _ if err.is_config_gate_error() => ("config_gated", err.to_string()),
8267                crate::supervise::SuperviseError::Disabled { .. } => {
8268                    ("module_disabled", err.to_string())
8269                }
8270                crate::supervise::SuperviseError::SwapInProgress { .. } => {
8271                    ("swap_in_progress", err.to_string())
8272                }
8273                _ => (
8274                    "target_unavailable",
8275                    format!("failed to restart module_id '{module_id}': {err}"),
8276                ),
8277            };
8278            return Ok(vec![control_error_frame(&frame, code, message)?]);
8279        }
8280
8281        let response = ClientControlResponse::SupervisorAck {
8282            module_id,
8283            applied: true,
8284        };
8285        Ok(vec![control_response_body_frame(
8286            &frame,
8287            &response,
8288            "ClientControlResponse::SupervisorAck",
8289        )?])
8290    }
8291
8292    /// `supervisor.swap`. Answered when the swap has cut over or failed, not
8293    /// when the old process has finished draining: a caller whose own lane
8294    /// rides the old process must get its reply before that drain waits on it.
8295    async fn handle_supervisor_swap(
8296        &self,
8297        frame: Frame,
8298        module_id: String,
8299        ready_timeout_ms: Option<u64>,
8300        requester: Principal,
8301    ) -> Result<Vec<Frame>, RouterError> {
8302        // The daemon-wide operation lock is held only to resolve the handle,
8303        // not across the swap. The swap can take its whole readiness budget,
8304        // and `supervisor.set_enabled` (ck module stop) takes the same lock:
8305        // holding it here would park an operator's stop behind the swap it is
8306        // meant to abort. A rescan or stop that reaches the module during the
8307        // swap is served by the swap itself (see `supervise_swap`).
8308        let module = {
8309            let operation_lock = self.supervisor.operation_lock();
8310            let _operation_guard = operation_lock.lock().await;
8311            self.supervisor.get(&module_id)
8312        };
8313        let Some(module) = module else {
8314            return Ok(vec![control_error_frame(
8315                &frame,
8316                "unknown_module",
8317                format!("module_id '{module_id}' is not supervised"),
8318            )?]);
8319        };
8320
8321        self.route_outages.mark_operator_action(&module_id);
8322        if let Err(err) = module
8323            .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
8324            .await
8325        {
8326            self.route_outages
8327                .operator_action_ended_unrefused(&module_id);
8328            use crate::supervise::SuperviseError;
8329            let message = err.to_string();
8330            let error = match err {
8331                SuperviseError::ConfigHeld { .. } => ErrorBody::new("config_gated", message),
8332                SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
8333                SuperviseError::SwapRefused { reason, .. } => ErrorBody {
8334                    code: "swap_refused".to_string(),
8335                    message,
8336                    detail: Some(serde_json::json!({ "reason": reason.as_str() })),
8337                },
8338                SuperviseError::SwapFailed {
8339                    arm,
8340                    candidate_exit,
8341                    ..
8342                } => ErrorBody {
8343                    code: "swap_failed".to_string(),
8344                    message,
8345                    detail: Some(serde_json::json!({
8346                        "arm": arm.as_str(),
8347                        "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
8348                        "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
8349                    })),
8350                },
8351                _ => ErrorBody::new(
8352                    "target_unavailable",
8353                    format!("failed to swap module_id '{module_id}': {message}"),
8354                ),
8355            };
8356            return Ok(vec![control_error_body_frame(&frame, error)?]);
8357        }
8358        // A completed swap kept the incumbent serving until cutover, so it
8359        // usually opened no outage; a mark left behind would make the next,
8360        // unrelated outage read as requested.
8361        self.route_outages
8362            .operator_action_ended_unrefused(&module_id);
8363
8364        let response = ClientControlResponse::SupervisorAck {
8365            module_id,
8366            applied: true,
8367        };
8368        Ok(vec![control_response_body_frame(
8369            &frame,
8370            &response,
8371            "ClientControlResponse::SupervisorAck",
8372        )?])
8373    }
8374
8375    async fn handle_supervisor_reload(
8376        &self,
8377        frame: Frame,
8378        module_id: String,
8379        requester: Principal,
8380    ) -> Result<Vec<Frame>, RouterError> {
8381        let operation_lock = self.supervisor.operation_lock();
8382        let _operation_guard = operation_lock.lock().await;
8383        let Some(module) = self.supervisor.get(&module_id) else {
8384            return Ok(vec![control_error_frame(
8385                &frame,
8386                "unknown_module",
8387                format!("module_id '{module_id}' is not supervised"),
8388            )?]);
8389        };
8390
8391        self.route_outages.mark_operator_action(&module_id);
8392        if let Err(err) = module.reload_requested(Some(requester)).await {
8393            self.route_outages
8394                .operator_action_ended_unrefused(&module_id);
8395            let (code, message) = match err {
8396                _ if err.is_config_gate_error() => ("config_gated", err.to_string()),
8397                crate::supervise::SuperviseError::Disabled { .. } => {
8398                    ("module_disabled", err.to_string())
8399                }
8400                crate::supervise::SuperviseError::SwapInProgress { .. } => {
8401                    ("swap_in_progress", err.to_string())
8402                }
8403                _ => (
8404                    "reload_failed",
8405                    format!("failed to reload module_id '{module_id}': {err}"),
8406                ),
8407            };
8408            return Ok(vec![control_error_frame(&frame, code, message)?]);
8409        }
8410
8411        let response = ClientControlResponse::SupervisorAck {
8412            module_id,
8413            applied: true,
8414        };
8415        Ok(vec![control_response_body_frame(
8416            &frame,
8417            &response,
8418            "ClientControlResponse::SupervisorAck",
8419        )?])
8420    }
8421
8422    /// `supervisor.rescan` and `supervisor.rescan_checked`. `mode` says where
8423    /// the configuration comes from and whether to apply it; every mode then
8424    /// runs the same comparison, so a preview describes the apply it previews.
8425    async fn handle_supervisor_rescan(
8426        &self,
8427        frame: Frame,
8428        mode: RescanMode,
8429        requester: Principal,
8430    ) -> Result<Vec<Frame>, RouterError> {
8431        let preview = mode.previews();
8432        let mut approved_preview = None;
8433        if matches!(mode, RescanMode::Preview) {
8434            if let Some(owner) = &self.config_runtime {
8435                match owner
8436                    .on_blocking(|owner| {
8437                        owner.observe_file_proposal()?;
8438                        owner.approved_daemon_configuration()
8439                    })
8440                    .await
8441                {
8442                    Ok(config) => approved_preview = config,
8443                    Err(error) => return Ok(vec![config_gate::config_error_frame(&frame, error)?]),
8444                }
8445            }
8446        }
8447        let Some(context) = self.rescan.clone() else {
8448            return Ok(vec![control_error_frame(
8449                &frame,
8450                "rescan_unavailable",
8451                "the daemon was not started with a reloadable config path".to_string(),
8452            )?]);
8453        };
8454
8455        let operation_lock = self.supervisor.operation_lock();
8456        let _operation_guard = operation_lock.lock().await;
8457        // Loaded under the operation lock, so no other rescan can change the
8458        // staged configuration between a checked apply's comparison and the
8459        // apply itself.
8460        let loaded = match self.load_rescan_config(&mode, &context) {
8461            Ok(config) => config,
8462            Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
8463        };
8464        // `load` reports a missing file as Ok(None), which is correct at boot
8465        // (no config, nothing to supervise) and catastrophic here: rescan treats
8466        // "not in the config" as "remove it", so an absent file would read as an
8467        // empty module list and retire the entire running fleet. An editor
8468        // writing via write-new-then-rename, or a half-finished edit, is enough
8469        // to open that window. Refuse instead: a config that cannot be read
8470        // carries no instruction to remove anything.
8471        let Some((config, activity_config)) = loaded else {
8472            return Ok(vec![control_error_frame(
8473                &frame,
8474                "invalid_daemon_config",
8475                format!(
8476                    "daemon config not found at {}; refusing to rescan (an absent config would \
8477                     retire every supervised module)",
8478                    context.config_path.display()
8479                ),
8480            )?]);
8481        };
8482        let (
8483            configured_port,
8484            storage_config,
8485            admission_facts_carrier_module_id,
8486            admission_facts_targets,
8487            scope_authority_owners,
8488            federated_scope_owner,
8489            privileged_operator_prompt,
8490            modules,
8491            reserved_capabilities,
8492        ) = (
8493            config.port,
8494            config.storage,
8495            config.admission_facts_carrier_module_id,
8496            config.admission_facts_targets,
8497            config.scope_authority_owners,
8498            config.federated_scope_owner,
8499            config.privileged_operator_prompt,
8500            config.modules,
8501            config.reserved_capabilities,
8502        );
8503
8504        // Collect the sections rescan cannot apply, so the REPLY carries them.
8505        //
8506        // The warning below has always been correct and has always gone only to
8507        // the journal -- addressed to whoever reads logs, while the person who
8508        // just edited the config is looking at the CLI. Naming each section
8509        // individually rather than setting a flag: "something outside modules
8510        // changed" sends the operator back to diffing their own file, which is
8511        // the work this is meant to save.
8512        let mut restart_required = Vec::new();
8513        for section in RestartRequiredSection::ALL {
8514            let changed = match section {
8515                RestartRequiredSection::Port => configured_port != context.configured_port,
8516                RestartRequiredSection::Storage => storage_config != context.storage_config,
8517                RestartRequiredSection::AdmissionFactsCarrierModuleId => {
8518                    admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
8519                }
8520                RestartRequiredSection::AdmissionFactsTargets => {
8521                    admission_facts_targets != context.admission_facts_targets
8522                }
8523                RestartRequiredSection::ScopeAuthorityOwners => {
8524                    scope_authority_owners != context.scope_authority_owners
8525                }
8526                RestartRequiredSection::FederatedScopeOwner => {
8527                    federated_scope_owner != context.federated_scope_owner
8528                }
8529                RestartRequiredSection::PrivilegedOperatorPrompt => {
8530                    privileged_operator_prompt != context.privileged_operator_prompt
8531                }
8532            };
8533            if changed {
8534                restart_required.push(section.label().to_string());
8535            }
8536        }
8537        if !restart_required.is_empty() {
8538            warn!(
8539                config_path = %context.config_path.display(),
8540                sections = %restart_required.join(", "),
8541                "daemon config changed outside the modules section; restart the daemon to apply those changes"
8542            );
8543        }
8544
8545        for configured in &modules {
8546            if let Err(err) = validate_spec(&configured.module_spec()) {
8547                return Ok(vec![control_error_frame(
8548                    &frame,
8549                    "invalid_daemon_config",
8550                    format!("supervisor rescan rejected daemon config: {err}"),
8551                )?]);
8552            }
8553        }
8554
8555        let configured_capabilities = modules
8556            .iter()
8557            .map(|module| (module.module_id.clone(), module.enabled))
8558            .collect::<Vec<_>>();
8559        let preview_capability_warnings = if preview {
8560            let (_, registrations) = self.runtime_capability_snapshot()?;
8561            let current_modules = self
8562                .supervisor
8563                .list()
8564                .into_iter()
8565                .map(|module| module.module_id().to_string())
8566                .collect::<BTreeSet<_>>();
8567            let resulting_modules = configured_capabilities.clone();
8568            let removed = current_modules
8569                .into_iter()
8570                .filter(|module_id| {
8571                    !resulting_modules
8572                        .iter()
8573                        .any(|(configured_id, _)| configured_id == module_id)
8574                })
8575                .collect::<Vec<_>>();
8576            // A candidate preview also names providers the candidate disables.
8577            // `supervisor.rescan`'s own preview keeps its removal-only warnings.
8578            let disabled = match &mode {
8579                RescanMode::Candidate(_) => resulting_modules
8580                    .iter()
8581                    .filter(|(_, enabled)| !enabled)
8582                    .map(|(module_id, _)| module_id.clone())
8583                    .collect::<Vec<_>>(),
8584                _ => Vec::new(),
8585            };
8586            self.capability_evaluator.preview_warnings(
8587                resulting_modules,
8588                &removed,
8589                &disabled,
8590                &registrations,
8591            )
8592        } else {
8593            Vec::new()
8594        };
8595        let reconciled = match (&mode, approved_preview.as_ref()) {
8596            (RescanMode::Preview, Some(approved)) => Ok((
8597                config_gate::preview_approved_daemon(approved, &modules),
8598                Vec::new(),
8599            )),
8600            (RescanMode::Candidate(candidate), _) => self
8601                .preview_candidate(&context, modules, candidate)
8602                .map(|result| (result, Vec::new())),
8603            _ => {
8604                self.reconcile_supervised_modules(
8605                    &context.supervisor,
8606                    modules,
8607                    preview,
8608                    requester.clone(),
8609                )
8610                .await
8611            }
8612        };
8613        let (result, changed) = match reconciled {
8614            Ok(result) => result,
8615            Err(message) => {
8616                return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
8617            }
8618        };
8619        if !preview {
8620            if let Some(ledger) = &self.activity {
8621                ledger.module_rescanned(&result.added, &result.removed, requester.clone());
8622                ledger.config_changed(&result.added, &result.removed, &changed, requester);
8623            }
8624            if let Err(code) = self.configure_activity(activity_config).await {
8625                warn!(
8626                    code,
8627                    "activity retention could not be applied during rescan"
8628                );
8629            }
8630            self.capability_evaluator
8631                .configure(configured_capabilities, reserved_capabilities);
8632            self.capability_evaluator.wake_deadline_loop();
8633            self.refresh_capability_requirements();
8634        }
8635        let mut result = result;
8636        result.restart_required = restart_required;
8637        result.capability_warnings = preview_capability_warnings;
8638        let response = ClientControlResponse::SupervisorRescan { result };
8639        Ok(vec![control_response_body_frame(
8640            &frame,
8641            &response,
8642            "ClientControlResponse::SupervisorRescan",
8643        )?])
8644    }
8645
8646    async fn handle_supervisor_release_reserved(
8647        &self,
8648        frame: Frame,
8649        module_id: String,
8650    ) -> Result<Vec<Frame>, RouterError> {
8651        let Some(context) = self.rescan.clone() else {
8652            return Ok(vec![control_error_frame(
8653                &frame,
8654                "release_unavailable",
8655                "reserved-id release requires a daemon started with a reloadable config path",
8656            )?]);
8657        };
8658        let operation_lock = self.supervisor.operation_lock();
8659        let _operation_guard = operation_lock.lock().await;
8660        let approved = if let Some(owner) = &self.config_runtime {
8661            // Observe the file as a proposal, but decide membership solely from
8662            // approved bytes. A proposed re-add must not retain a removed id.
8663            match owner
8664                .on_blocking(|owner| {
8665                    owner.observe_file_proposal()?;
8666                    owner.approved_daemon_configuration()
8667                })
8668                .await
8669            {
8670                Ok(config) => config,
8671                Err(error) => return Ok(vec![config_gate::config_error_frame(&frame, error)?]),
8672            }
8673        } else {
8674            None
8675        };
8676        let loaded = if let Some(approved) = approved {
8677            Ok(Some(approved))
8678        } else {
8679            crate::daemon_config::load(&context.config_path)
8680        };
8681        let loaded = match loaded {
8682            Ok(Some(config)) => config,
8683            Ok(None) => {
8684                return Ok(vec![control_error_frame(
8685                    &frame,
8686                    "invalid_daemon_config",
8687                    format!(
8688                        "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
8689                        context.config_path.display()
8690                    ),
8691                )?])
8692            }
8693            Err(err) => {
8694                return Ok(vec![control_error_frame(
8695                    &frame,
8696                    "invalid_daemon_config",
8697                    format!("unable to verify reserved-id release against daemon config: {err}"),
8698                )?])
8699            }
8700        };
8701        if loaded
8702            .modules
8703            .iter()
8704            .any(|configured| configured.module_id == module_id)
8705        {
8706            return Ok(vec![control_error_frame(
8707                &frame,
8708                "reserved_module_configured",
8709                format!(
8710                    "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
8711                ),
8712            )?]);
8713        }
8714        if !self.supervisor.release_retained_reserved_gate(&module_id) {
8715            return Ok(vec![control_error_frame(
8716                &frame,
8717                "reserved_gate_not_retained",
8718                format!(
8719                    "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
8720                ),
8721            )?]);
8722        }
8723
8724        let response = ClientControlResponse::SupervisorAck {
8725            module_id,
8726            applied: true,
8727        };
8728        Ok(vec![control_response_body_frame(
8729            &frame,
8730            &response,
8731            "ClientControlResponse::SupervisorAck",
8732        )?])
8733    }
8734
8735    /// Compare the configured module set with the supervised one, changing
8736    /// nothing. The staged spec each supervised module holds is what the
8737    /// configured entry is compared with.
8738    fn plan_rescan(
8739        &self,
8740        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
8741    ) -> Result<RescanPlan, String> {
8742        let mut current = BTreeMap::new();
8743        for module in self.supervisor.list() {
8744            let (spec, health) = module.configuration().map_err(|err| {
8745                format!(
8746                    "failed to read configuration for module_id '{}': {err}",
8747                    module.module_id()
8748                )
8749            })?;
8750            let enabled = module
8751                .status()
8752                .map_err(|err| {
8753                    format!(
8754                        "failed to read status for module_id '{}': {err}",
8755                        module.module_id()
8756                    )
8757                })?
8758                .enabled;
8759            current.insert(
8760                module.module_id().to_string(),
8761                (module, spec, health, enabled),
8762            );
8763        }
8764        let configured = configured_modules
8765            .into_iter()
8766            .map(|module| (module.module_id.clone(), module))
8767            .collect::<BTreeMap<_, _>>();
8768
8769        let added = configured
8770            .keys()
8771            .filter(|module_id| !current.contains_key(*module_id))
8772            .cloned()
8773            .collect::<Vec<_>>();
8774        let removed = current
8775            .keys()
8776            .filter(|module_id| !configured.contains_key(*module_id))
8777            .cloned()
8778            .collect::<Vec<_>>();
8779        let mut changed_pending_reload = Vec::new();
8780        let mut changed_pending = Vec::new();
8781        let mut configuration_changes = BTreeSet::new();
8782        let mut enabled_changes = BTreeSet::new();
8783        let mut unchanged = 0_u32;
8784        let publication = self
8785            .config_runtime
8786            .as_ref()
8787            .and_then(|owner| owner.polled_snapshot().ok());
8788
8789        for (module_id, configured_module) in &configured {
8790            let Some((module, current_spec, current_health, current_enabled)) =
8791                current.get(module_id)
8792            else {
8793                continue;
8794            };
8795            // Compare the whole launch spec so a future launch field cannot
8796            // accidentally become a live-only policy change. Health is stored
8797            // separately and applies live without replacing the process.
8798            let launch_changed = *current_spec != configured_module.module_spec();
8799            let configuration_changed =
8800                launch_changed || *current_health != configured_module.health;
8801            let pending = self.pending_module_fields(
8802                module,
8803                Some(configured_module),
8804                configuration_changed,
8805                Self::registration_channel_in_force(
8806                    publication
8807                        .as_ref()
8808                        .and_then(|view| view.registration_declaration(module_id))
8809                        .as_deref(),
8810                    publication
8811                        .as_ref()
8812                        .and_then(|view| match &view.approved_daemon {
8813                            crate::config_gate::runtime::ApprovedDaemonView::Approved {
8814                                config,
8815                                ..
8816                            } => config
8817                                .modules
8818                                .iter()
8819                                .find(|entry| entry.module_id == *module_id)
8820                                .map(|entry| entry.registration_channel),
8821                            _ => None,
8822                        }),
8823                ),
8824            )?;
8825            let has_pending = !pending.is_empty();
8826            if has_pending {
8827                changed_pending.push(subc_control::SupervisorPendingModule {
8828                    module_id: module_id.clone(),
8829                    fields: pending
8830                        .into_iter()
8831                        .map(|mut field| {
8832                            field.value = None;
8833                            field
8834                        })
8835                        .collect(),
8836                });
8837            }
8838            // Applying a rescan also replaces the recorded config value, even
8839            // when an operator already put the runtime in the requested state.
8840            let enabled_changed = *current_enabled != configured_module.enabled
8841                || self
8842                    .capability_evaluator
8843                    .configured_enabled(module_id)
8844                    .is_some_and(|enabled| enabled != configured_module.enabled);
8845            if configuration_changed {
8846                configuration_changes.insert(module_id.clone());
8847            }
8848            if launch_changed {
8849                changed_pending_reload.push(module_id.clone());
8850            }
8851            if enabled_changed {
8852                enabled_changes.insert(module_id.clone());
8853            }
8854            if !configuration_changed && !enabled_changed && !has_pending {
8855                unchanged = unchanged.saturating_add(1);
8856            }
8857        }
8858        Ok(RescanPlan {
8859            current,
8860            configured,
8861            added,
8862            removed,
8863            changed_pending_reload,
8864            changed_pending,
8865            configuration_changes,
8866            enabled_changes,
8867            unchanged,
8868        })
8869    }
8870
8871    /// Reconcile the running module set against the configured one.
8872    ///
8873    /// With `preview` set, the diff is computed and returned WITHOUT applying any
8874    /// of it: nothing is retired, reconfigured, enabled or spawned. The preview
8875    /// deliberately shares this function with the executing path rather than
8876    /// computing the same diff somewhere else -- two implementations of one
8877    /// decision agree until they do not, and the whole value of a preview is that
8878    /// it describes the operation that will actually run.
8879    async fn reconcile_supervised_modules(
8880        &self,
8881        supervisor: &Supervisor,
8882        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
8883        preview: bool,
8884        requester: Principal,
8885    ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
8886        let plan = self.plan_rescan(configured_modules)?;
8887
8888        // Everything above this point is pure computation over two snapshots.
8889        // Everything below MUTATES. The preview returns here so the boundary is a
8890        // single early return rather than a condition repeated at each mutation
8891        // site, where one missed guard would apply part of a change the caller was
8892        // told would not happen.
8893        if preview {
8894            return Ok((plan.preview_result(), Vec::new()));
8895        }
8896        let changed = plan
8897            .configuration_changes
8898            .union(&plan.enabled_changes)
8899            .cloned()
8900            .collect::<Vec<_>>();
8901        let RescanPlan {
8902            current,
8903            configured,
8904            added,
8905            removed,
8906            changed_pending_reload,
8907            changed_pending,
8908            configuration_changes,
8909            enabled_changes,
8910            unchanged,
8911        } = plan;
8912        // The staged generation moves before the first change rather than
8913        // after the last, so a rescan that fails partway still invalidates
8914        // every earlier `supervisor.rescan_checked` preview: part of its diff
8915        // may have run.
8916        if !(added.is_empty()
8917            && removed.is_empty()
8918            && configuration_changes.is_empty()
8919            && changed_pending.is_empty()
8920            && enabled_changes.is_empty())
8921        {
8922            if let Some(context) = &self.rescan {
8923                context
8924                    .generation
8925                    .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
8926            }
8927        }
8928
8929        for module_id in &removed {
8930            let module = &current
8931                .get(module_id)
8932                .expect("removed module came from current supervisor state")
8933                .0;
8934            module
8935                .retire_requested(Some(requester.clone()))
8936                .await
8937                .map_err(|err| {
8938                    format!("failed to retire module_id '{module_id}' during rescan: {err}")
8939                })?;
8940            // TOMBSTONE BEFORE RETIRE, and the order is the whole fix.
8941            //
8942            // `handle_route_open` resolves an absent module in three steps:
8943            // registry, then supervisor status, then tombstone. Retiring first
8944            // opens a window where ALL THREE ARE ABSENT -- the registry entry
8945            // went with the teardown above, the supervisor entry went with
8946            // `retire`, and the tombstone does not exist yet -- so a route.open
8947            // landing in it gets `unknown_module` (RETRYABLE, "never heard of
8948            // it") for a module that was deliberately removed and whose caller
8949            // should get `module_removed` (TERMINAL, carrying a removal age).
8950            //
8951            // Writing the tombstone first closes it: during the window the
8952            // supervisor entry still answers, so the caller gets
8953            // `target_unavailable` -- retryable, and TRUE, because the module
8954            // is mid-teardown. After both statements it is `module_removed`.
8955            // No instant remains where a removed module reads as one that
8956            // never existed.
8957            //
8958            // NOT DETERMINISTICALLY TESTABLE FROM HERE, said plainly because
8959            // the absence of a test beside a fix invites deletion: these are
8960            // two sync statements with no await between them, so reaching the
8961            // window needs a second worker thread to land exactly between them
8962            // and there is no hook to force it. MEASURED: the 25 daemon_config
8963            // tests pass identically with the old order and the new one, so
8964            // the existing suite cannot see this and a green run is not
8965            // evidence either way. What the suite does hold is the
8966            // post-condition -- a removed module answers `module_removed` --
8967            // which this preserves.
8968            //
8969            // Found by an Athena panel reading the shipped tree against a
8970            // design note (2026-09-19), as the one concrete instance of that
8971            // note's class that survived contact with source. Direction is
8972            // benign: retryable where terminal was intended, never the reverse.
8973            self.supervisor.record_rescan_removal(module_id);
8974            self.supervisor.retire(module_id);
8975            self.route_outages.forget(module_id);
8976        }
8977
8978        for module_id in configured.keys() {
8979            let Some((module, _, _, current_enabled)) = current.get(module_id) else {
8980                continue;
8981            };
8982            let configured_module = configured
8983                .get(module_id)
8984                .expect("configured module id came from configured map");
8985            if configuration_changes.contains(module_id) {
8986                module
8987                    .update_configuration(
8988                        configured_module.module_spec(),
8989                        configured_module.health.clone(),
8990                        configured_module.drain_timeout_ms,
8991                    )
8992                    .await
8993                    .map_err(|err| {
8994                        format!(
8995                            "failed to update module_id '{module_id}' configuration during rescan: {err}"
8996                        )
8997                    })?;
8998            }
8999            // Config-only drift is applied by the evaluator's config update;
9000            // do not turn it into a runtime start or stop.
9001            if *current_enabled != configured_module.enabled {
9002                // A rescan that starts or stops a module applies an operator's
9003                // edit to the config, so the resulting outage was asked for.
9004                self.route_outages.mark_operator_action(module_id);
9005                module
9006                    .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
9007                    .await
9008                    .map_err(|err| {
9009                        self.route_outages.operator_action_ended_unrefused(module_id);
9010                        format!(
9011                            "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
9012                            configured_module.enabled
9013                        )
9014                    })?;
9015            }
9016        }
9017
9018        for module_id in &added {
9019            let configured_module = configured
9020                .get(module_id)
9021                .expect("added module id came from configured map");
9022            supervisor
9023                .supervise_configured_requested_async(
9024                    configured_module.module_spec(),
9025                    configured_module.enabled,
9026                    configured_module.health.clone(),
9027                    configured_module.drain_timeout_ms,
9028                    configured_module.restart,
9029                    configured_module.register_timeout,
9030                    Some(requester.clone()),
9031                )
9032                .await
9033                .map_err(|err| {
9034                    format!("failed to add module_id '{module_id}' during rescan: {err}")
9035                })?;
9036        }
9037
9038        if let Some(context) = &self.rescan {
9039            let mut entries = context
9040                .module_entries
9041                .lock()
9042                .unwrap_or_else(|e| e.into_inner());
9043            entries.record(&configured, &added, &removed);
9044        }
9045
9046        Ok((
9047            SupervisorRescanResult {
9048                added,
9049                removed,
9050                changed_pending_reload,
9051                changed_pending,
9052                enabled_changes: enabled_changes.iter().cloned().collect(),
9053                unchanged,
9054                preview: false,
9055                // Filled by the caller, which is the only layer that can see the
9056                // previous config to diff against.
9057                restart_required: Vec::new(),
9058                capability_warnings: Vec::new(),
9059                module_actions: Vec::new(),
9060                earlier_pending: Vec::new(),
9061                expect: None,
9062            },
9063            changed,
9064        ))
9065    }
9066
9067    async fn handle_supervisor_set_enabled(
9068        &self,
9069        frame: Frame,
9070        module_id: String,
9071        enabled: bool,
9072        requester: Principal,
9073    ) -> Result<Vec<Frame>, RouterError> {
9074        let operation_lock = self.supervisor.operation_lock();
9075        let _operation_guard = operation_lock.lock().await;
9076        let Some(module) = self.supervisor.get(&module_id) else {
9077            return Ok(vec![control_error_frame(
9078                &frame,
9079                "unknown_module",
9080                format!("module_id '{module_id}' is not supervised"),
9081            )?]);
9082        };
9083
9084        // Enabling counts as well as disabling: a module an operator starts
9085        // is refused until it registers, and that wait was asked for.
9086        self.route_outages.mark_operator_action(&module_id);
9087        let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
9088            Ok(applied) => applied,
9089            Err(err) => {
9090                self.route_outages
9091                    .operator_action_ended_unrefused(&module_id);
9092                return Ok(vec![control_error_frame(
9093                    &frame,
9094                    if err.is_config_gate_error() {
9095                        "config_gated"
9096                    } else {
9097                        "target_unavailable"
9098                    },
9099                    format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
9100                )?]);
9101            }
9102        };
9103        if !applied {
9104            // Already in the requested state: nothing was made unavailable,
9105            // so the mark must not outlive this request.
9106            self.route_outages
9107                .operator_action_ended_unrefused(&module_id);
9108        }
9109
9110        self.capability_evaluator.wake_deadline_loop();
9111        self.refresh_capability_requirements();
9112        let response = ClientControlResponse::SupervisorAck { module_id, applied };
9113        Ok(vec![control_response_body_frame(
9114            &frame,
9115            &response,
9116            "ClientControlResponse::SupervisorAck",
9117        )?])
9118    }
9119
9120    async fn handle_supervisor_health_probe(
9121        &self,
9122        ctx: &RouteCtx,
9123        frame: Frame,
9124        module_id: String,
9125    ) -> Result<Vec<Frame>, RouterError> {
9126        self.refresh_capability_requirements();
9127        let Some(registration) = self
9128            .registry
9129            .get_module(&module_id)
9130            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
9131        else {
9132            return Ok(vec![control_error_frame(
9133                &frame,
9134                "unknown_module",
9135                format!("module_id '{module_id}' is not registered"),
9136            )?]);
9137        };
9138
9139        // This guard's ACCEPT direction is fenced, but only INCIDENTALLY: no test is
9140        // named for it. Making `module_registration_grants_op` return false
9141        // unconditionally reddens five tests, and every one is named for something
9142        // else -- capability relay, probe/bind demultiplexing, supervision-only
9143        // probing. They exercise a successful advertisement check on the way to their
9144        // own subject.
9145        //
9146        // Real protection, fragile in a specific way: narrowing any of those tests to
9147        // focus on its stated subject would silently remove coverage nobody knows
9148        // they are carrying. Recorded here rather than as a sixth test, because the
9149        // useful fact is WHICH tests hold the guard up -- a new test would add
9150        // coverage without telling the next person what the existing ones quietly do.
9151        if !module_registration_grants_op(&registration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
9152        {
9153            return Ok(vec![control_error_frame(
9154                &frame,
9155                "health_not_advertised",
9156                format!("module_id '{module_id}' did not advertise health.check"),
9157            )?]);
9158        }
9159
9160        self.observe_control_wait(
9161            ctx,
9162            Some(&registration.manifest.module_id),
9163            ops::SUPERVISOR_HEALTH_PROBE,
9164            async {
9165                let deadline = Instant::now() + self.health_probe_timeout;
9166                let pending = match self.forwarding.begin_module_control_rpc_for(
9167                    &module_id,
9168                    MODULE_CONTROL_OP_HEALTH_CHECK,
9169                    deadline,
9170                ) {
9171                    Ok(pending) => pending,
9172                    Err(err) => {
9173                        return Ok(vec![control_error_frame(
9174                            &frame,
9175                            forwarding_error_code(&err),
9176                            err.to_string(),
9177                        )?])
9178                    }
9179                };
9180
9181                let PendingModuleControlRpc {
9182                    endpoint,
9183                    module_sink,
9184                    negotiated_ver,
9185                    corr: probe_corr,
9186                    receiver,
9187                } = pending;
9188                let mut guard =
9189                    ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
9190                let probe_body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {})
9191                    .map_err(|err| {
9192                        RouterError::backend(
9193                            0,
9194                            frame.header.corr,
9195                            format!("failed to encode health.check request: {err}"),
9196                        )
9197                    })?;
9198                let probe_frame = Frame::build_with_version(
9199                    negotiated_ver,
9200                    FrameType::Request,
9201                    control_flags(),
9202                    0,
9203                    0,
9204                    probe_corr,
9205                    probe_body,
9206                )
9207                .map_err(RouterError::FrameBuild)?;
9208
9209                // Queue admission and the reply share one deadline, so a module that
9210                // stops reading cannot park the requesting connection's reader forever.
9211                match timeout_at(deadline, module_sink.send(probe_frame)).await {
9212                    Ok(Ok(())) => {}
9213                    Ok(Err(err)) => {
9214                        return Ok(vec![control_error_frame(
9215                            &frame,
9216                            "target_unavailable",
9217                            err.to_string(),
9218                        )?]);
9219                    }
9220                    Err(_) => {
9221                        return Ok(vec![control_error_frame(
9222                            &frame,
9223                            "module_timeout",
9224                            format!(
9225                                "module_id '{module_id}' did not answer health.check within {:?}",
9226                                self.health_probe_timeout
9227                            ),
9228                        )?]);
9229                    }
9230                }
9231
9232                match timeout_at(deadline, receiver).await {
9233                    Ok(Ok(ModuleControlRpcOutcome::RawResponse(_))) => {
9234                        Ok(vec![control_error_frame(
9235                            &frame,
9236                            "invalid_control_body",
9237                            "health.check returned an unexpected config response",
9238                        )?])
9239                    }
9240                    Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
9241                        guard.disarm();
9242                        let Some(report) = response.health_report() else {
9243                            return Ok(vec![control_error_frame(
9244                                &frame,
9245                                "invalid_control_body",
9246                                "health.check RPC returned a non-health response",
9247                            )?]);
9248                        };
9249                        // Metrics go out whole here. The supervisor's cached snapshot
9250                        // caps this blob (see truncate_health_metrics), and this path
9251                        // exists precisely to answer without that cap -- so applying it
9252                        // here would leave no way to see what the cached view drops.
9253                        let HealthReport {
9254                            status,
9255                            detail,
9256                            metrics,
9257                        } = report;
9258                        let capability_detail = self
9259                            .capability_evaluator
9260                            .required_problem_detail(&module_id);
9261                        let response = ClientControlResponse::SupervisorHealthProbe {
9262                            module_id,
9263                            status,
9264                            detail: append_capability_problem_detail(detail, capability_detail),
9265                            metrics,
9266                        };
9267                        Ok(vec![control_response_body_frame(
9268                            &frame,
9269                            &response,
9270                            "ClientControlResponse::SupervisorHealthProbe",
9271                        )?])
9272                    }
9273                    Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
9274                        guard.disarm();
9275                        // The module refused the health probe with its own body, so
9276                        // the refusal is relayed without the daemon-origin flag
9277                        // (bit 6): a client must not read it as the daemon's own
9278                        // refusal, which it may treat as never forwarded.
9279                        Ok(vec![control_relayed_error_body_frame(&frame, body)?])
9280                    }
9281                    Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
9282                        guard.disarm();
9283                        Ok(vec![control_error_frame(
9284                            &frame,
9285                            "target_unavailable",
9286                            message,
9287                        )?])
9288                    }
9289                    Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
9290                        guard.disarm();
9291                        Ok(vec![control_error_frame(
9292                            &frame,
9293                            "invalid_control_body",
9294                            message,
9295                        )?])
9296                    }
9297                    Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
9298                        guard.disarm();
9299                        Ok(vec![control_error_frame(
9300                            &frame,
9301                            "invalid_control_body",
9302                            format!("expected module-control op '{expected}', got '{actual}'"),
9303                        )?])
9304                    }
9305                    Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
9306                        guard.disarm();
9307                        Ok(vec![control_error_frame(
9308                            &frame,
9309                            "module_timeout",
9310                            format!(
9311                                "module_id '{module_id}' answered health.check after {:?}",
9312                                self.health_probe_timeout
9313                            ),
9314                        )?])
9315                    }
9316                    Ok(Err(_)) => Ok(vec![control_error_frame(
9317                        &frame,
9318                        "target_unavailable",
9319                        "health.check waiter was canceled before the module responded",
9320                    )?]),
9321                    Err(_) => Ok(vec![control_error_frame(
9322                        &frame,
9323                        "module_timeout",
9324                        format!(
9325                            "module_id '{module_id}' did not answer health.check within {:?}",
9326                            self.health_probe_timeout
9327                        ),
9328                    )?]),
9329                }
9330            },
9331        )
9332        .await
9333    }
9334
9335    fn supervisor_status(
9336        &self,
9337        module_id: &str,
9338        corr: u64,
9339    ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
9340        self.supervisor
9341            .get(module_id)
9342            .map(|module| {
9343                let warming = module.is_warming_for_control("status").map_err(|err| {
9344                    RouterError::backend(
9345                        0,
9346                        corr,
9347                        format!(
9348                            "failed to read supervisor warming state for module_id '{module_id}': {err}"
9349                        ),
9350                    )
9351                })?;
9352                module.status_for_control("status").map_err(|err| {
9353                    RouterError::backend(
9354                        0,
9355                        corr,
9356                        format!(
9357                            "failed to read supervisor status for module_id '{module_id}': {err}"
9358                        ),
9359                    )
9360                }).map(|status| (status, warming))
9361            })
9362            .transpose()
9363    }
9364
9365    fn guard_module_control_op(
9366        &self,
9367        frame: &Frame,
9368        module_id: &str,
9369        op: &str,
9370    ) -> Result<Option<Frame>, RouterError> {
9371        if self.module_grants_op(module_id, op, frame.header.corr)? {
9372            return Ok(None);
9373        }
9374
9375        Ok(Some(control_error_frame(
9376            frame,
9377            "op_not_allowed",
9378            format!("module_id '{module_id}' did not grant control op '{op}'"),
9379        )?))
9380    }
9381
9382    fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
9383        let Some(registration) = self
9384            .registry
9385            .get_module(module_id)
9386            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
9387        else {
9388            return Ok(false);
9389        };
9390        Ok(module_registration_grants_op(&registration.control_ops, op))
9391    }
9392
9393    fn handle_status_update(
9394        &self,
9395        endpoint: ModuleEndpointId,
9396        frame: Frame,
9397    ) -> Result<Vec<Frame>, RouterError> {
9398        let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
9399            Ok(update) => update,
9400            Err(err) => {
9401                // Forward-compat: a newer module may push a channel-0 op this subc
9402                // version doesn't know. The control contract says unknown push ops
9403                // are IGNORED, never answered with an error. Only a malformed body
9404                // for an op we DO know is a real error worth surfacing.
9405                if is_known_module_push_op(&frame.body) {
9406                    return Ok(vec![control_error_frame(
9407                        &frame,
9408                        "invalid_control_body",
9409                        format!("malformed module control push body: {err}"),
9410                    )?]);
9411                }
9412                return Ok(Vec::new());
9413            }
9414        };
9415
9416        match update {
9417            ModuleControlPush::RouteStatus {
9418                route_channel,
9419                route_epoch,
9420                status,
9421            } => {
9422                self.forwarding
9423                    .cache_status(endpoint, route_channel, route_epoch, status)
9424                    .map_err(RouterError::Forwarding)?;
9425            }
9426        }
9427        Ok(Vec::new())
9428    }
9429
9430    fn handle_route_poll(
9431        &self,
9432        ctx: &RouteCtx,
9433        frame: Frame,
9434        route_channel: u16,
9435        route_epoch: u32,
9436        kind: PollKind,
9437    ) -> Result<Vec<Frame>, RouterError> {
9438        let snapshot = self
9439            .forwarding
9440            .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
9441            .map_err(RouterError::Forwarding)?;
9442        let response = match (kind, snapshot) {
9443            (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
9444                ClientControlResponse::RoutePoll {
9445                    route_channel,
9446                    route_epoch,
9447                    status,
9448                    live: None,
9449                }
9450            }
9451            (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
9452                route_channel,
9453                route_epoch,
9454                status: None,
9455                live: None,
9456            },
9457            (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
9458                // ABSENCE HERE MEANS "NOT SUPERVISED", NOT "UNKNOWN", and that
9459                // is what makes reporting `true` correct rather than a
9460                // confident guess. `process_live` returns None only when the
9461                // module id has no supervisor snapshot at all -- an
9462                // externally-started module the daemon did not spawn -- and
9463                // for those the supervisor has no opinion to offer, ever. It
9464                // is never None for a supervised module in an unknown state:
9465                // a supervised module always has a snapshot, and the answer
9466                // comes from `state == Running && process_alive`.
9467                //
9468                // The route is Bound, so the module completed a HELLO on a
9469                // live connection; "the process this route points at is
9470                // running" is therefore attested by the binding rather than
9471                // assumed. Reporting `false` for an unsupervised module would
9472                // be the actual lie -- it would tell a client its healthy
9473                // route is dead because the daemon does not manage the
9474                // process.
9475                //
9476                // IF `process_live` EVER GAINS A THIRD CASE -- a supervised
9477                // module whose liveness is genuinely unknown, e.g. a snapshot
9478                // that has not been populated yet -- THIS DEFAULT BECOMES
9479                // WRONG and must split: unsupervised stays true, unknown
9480                // becomes null so the client can tell the two apart. The
9481                // response field is already `Option<bool>`, so the wire can
9482                // carry that distinction today.
9483                let live = self
9484                    .process_liveness
9485                    .as_ref()
9486                    .and_then(|source| source.process_live(&module_id))
9487                    .unwrap_or(true);
9488                ClientControlResponse::RoutePoll {
9489                    route_channel,
9490                    route_epoch,
9491                    status: None,
9492                    live: Some(live),
9493                }
9494            }
9495            (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
9496                route_channel,
9497                route_epoch,
9498                status: None,
9499                live: Some(false),
9500            },
9501        };
9502
9503        Ok(vec![control_response_body_frame(
9504            &frame,
9505            &response,
9506            "ClientControlResponse::RoutePoll",
9507        )?])
9508    }
9509
9510    pub(crate) fn observe_module_control_completion(
9511        &self,
9512        completion: ModuleControlRpcCompletion,
9513    ) -> bool {
9514        match completion {
9515            ModuleControlRpcCompletion::Unknown => false,
9516            ModuleControlRpcCompletion::Settled => true,
9517            ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
9518                let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
9519                info!(
9520                    module_id = %module_id,
9521                    latency_ms,
9522                    "late health.check answer proves the module is alive"
9523                );
9524                match self
9525                    .supervisor
9526                    .record_late_health_answer(&module_id, latency_ms)
9527                {
9528                    Ok(true) => {}
9529                    Ok(false) => debug!(
9530                        module_id = %module_id,
9531                        latency_ms,
9532                        "late health.check answer has no active supervisor snapshot"
9533                    ),
9534                    Err(err) => warn!(
9535                        module_id = %module_id,
9536                        latency_ms,
9537                        error = %err,
9538                        "failed to record late health.check answer"
9539                    ),
9540                }
9541                true
9542            }
9543        }
9544    }
9545
9546    /// Decide whether a failure while settling a relayed `route.bind` belongs to
9547    /// the module connection whose frame is being handled, or to the client that
9548    /// relay was opened for.
9549    ///
9550    /// This runs on the MODULE connection's frame handler, where returning `Err`
9551    /// ends that connection -- and a module connection carries every client's
9552    /// routes to that module, so ending it costs the whole fleet its tools.
9553    /// `ConnectionClosing` carries the id of the connection that is closing, and
9554    /// when that id is a CLIENT's, the condition is entirely about that one
9555    /// client's route.open. A client-scoped condition has no authority over a
9556    /// shared module connection, so it is logged and the single relay is dropped:
9557    /// the client is going away, and `complete_pending_relay` already removed the
9558    /// relay before failing, so there is nothing left to settle. Anything that
9559    /// relay still reserved is released by that client's own connection teardown,
9560    /// which is already under way -- that is what "closing" means.
9561    ///
9562    /// Every other failure is a statement about THIS connection and stays fatal:
9563    /// a poisoned forwarding lock, a stale module endpoint, and the module's own
9564    /// id in `ConnectionClosing` all mean this connection cannot keep serving
9565    /// frames correctly.
9566    fn refuse_to_end_module_connection_for_a_client(
9567        &self,
9568        module_connection_id: ConnectionId,
9569        corr: u64,
9570        err: ForwardingError,
9571    ) -> Result<(), RouterError> {
9572        if let ForwardingError::ConnectionClosing { connection_id } = err {
9573            if connection_id != module_connection_id {
9574                warn!(
9575                    module_connection_id = module_connection_id.get(),
9576                    client_connection_id = connection_id.get(),
9577                    corr,
9578                    "dropping a route.bind response for a closing client; the module connection keeps serving"
9579                );
9580                return Ok(());
9581            }
9582        }
9583        Err(RouterError::Forwarding(err))
9584    }
9585
9586    fn handle_module_relay_response(
9587        &self,
9588        connection_id: ConnectionId,
9589        frame: Frame,
9590    ) -> Result<Vec<Frame>, RouterError> {
9591        let mut secondary_error = None;
9592        let outcome = match frame.header.ty {
9593            FrameType::Response => match module_control_response_probe(&frame.body) {
9594                Ok(probe) if probe.op == "route.bind" => {
9595                    match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
9596                        Ok(ModuleControlResponse::RouteBindAck {}) => {
9597                            RouteBindRelayOutcome::Accepted
9598                        }
9599                        Ok(other) => {
9600                            let message =
9601                                format!("route.bind response carried unexpected body: {other:?}");
9602                            secondary_error = Some(control_error_frame(
9603                                &frame,
9604                                "invalid_control_body",
9605                                message.clone(),
9606                            )?);
9607                            RouteBindRelayOutcome::ModuleGone(message)
9608                        }
9609                        Err(err) => {
9610                            let message = format!("malformed route.bind response body: {err}");
9611                            secondary_error = Some(control_error_frame(
9612                                &frame,
9613                                "invalid_control_body",
9614                                message.clone(),
9615                            )?);
9616                            RouteBindRelayOutcome::ModuleGone(message)
9617                        }
9618                    }
9619                }
9620                Ok(probe) => {
9621                    let outcome = if matches!(probe.op.as_str(), "config.review" | "consent/v1") {
9622                        ModuleControlRpcOutcome::RawResponse(frame.body.clone())
9623                    } else {
9624                        match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
9625                            Ok(response) => ModuleControlRpcOutcome::Response(response),
9626                            Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
9627                                "malformed {} response body: {err}",
9628                                probe.op
9629                            )),
9630                        }
9631                    };
9632                    let completion = self
9633                        .forwarding
9634                        .complete_module_control_rpc(
9635                            connection_id,
9636                            frame.header.corr,
9637                            Some(&probe.op),
9638                            outcome,
9639                        )
9640                        .map_err(RouterError::Forwarding)?;
9641                    if !self.observe_module_control_completion(completion) {
9642                        debug!(
9643                            connection_id = connection_id.get(),
9644                            corr = frame.header.corr,
9645                            op = %probe.op,
9646                            "dropping late or unknown module-control RPC response"
9647                        );
9648                    }
9649                    return Ok(Vec::new());
9650                }
9651                Err(err) => {
9652                    if let Some(expected_op) = self
9653                        .forwarding
9654                        .pending_module_control_op(connection_id, frame.header.corr)
9655                        .map_err(RouterError::Forwarding)?
9656                    {
9657                        let completion = self
9658                            .forwarding
9659                            .complete_module_control_rpc(
9660                                connection_id,
9661                                frame.header.corr,
9662                                None,
9663                                ModuleControlRpcOutcome::MalformedResponse(format!(
9664                                    "malformed {expected_op} response body: {err}"
9665                                )),
9666                            )
9667                            .map_err(RouterError::Forwarding)?;
9668                        if !self.observe_module_control_completion(completion) {
9669                            debug!(
9670                                connection_id = connection_id.get(),
9671                                corr = frame.header.corr,
9672                                "dropping late malformed module-control RPC response"
9673                            );
9674                        }
9675                        return Ok(Vec::new());
9676                    }
9677                    let message = format!("malformed route.bind response body: {err}");
9678                    secondary_error = Some(control_error_frame(
9679                        &frame,
9680                        "invalid_control_body",
9681                        message.clone(),
9682                    )?);
9683                    RouteBindRelayOutcome::ModuleGone(message)
9684                }
9685            },
9686            FrameType::Error => {
9687                if self
9688                    .forwarding
9689                    .pending_module_control_op(connection_id, frame.header.corr)
9690                    .map_err(RouterError::Forwarding)?
9691                    .is_some()
9692                {
9693                    let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
9694                        Ok(body) => ModuleControlRpcOutcome::Rejected(body),
9695                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
9696                            "malformed module-control ERROR body: {err}"
9697                        )),
9698                    };
9699                    let completion = self
9700                        .forwarding
9701                        .complete_module_control_rpc(
9702                            connection_id,
9703                            frame.header.corr,
9704                            None,
9705                            outcome,
9706                        )
9707                        .map_err(RouterError::Forwarding)?;
9708                    if !self.observe_module_control_completion(completion) {
9709                        debug!(
9710                            connection_id = connection_id.get(),
9711                            corr = frame.header.corr,
9712                            "dropping late or unknown module-control RPC error"
9713                        );
9714                    }
9715                    return Ok(Vec::new());
9716                }
9717                match serde_json::from_slice::<ErrorBody>(&frame.body) {
9718                    Ok(body) => RouteBindRelayOutcome::Rejected(body),
9719                    Err(err) => {
9720                        let message = format!("malformed route.bind ERROR body: {err}");
9721                        secondary_error = Some(control_error_frame(
9722                            &frame,
9723                            "invalid_control_body",
9724                            message.clone(),
9725                        )?);
9726                        RouteBindRelayOutcome::ModuleGone(message)
9727                    }
9728                }
9729            }
9730            ty => {
9731                return Ok(vec![control_error_frame(
9732                    &frame,
9733                    "unsupported_control_frame",
9734                    format!("unsupported module channel-0 frame {ty:?}"),
9735                )?])
9736            }
9737        };
9738
9739        let settled =
9740            self.forwarding
9741                .complete_pending_relay(connection_id, frame.header.corr, outcome);
9742        let completion = match settled {
9743            Ok(completion) => completion,
9744            Err(err) => {
9745                self.refuse_to_end_module_connection_for_a_client(
9746                    connection_id,
9747                    frame.header.corr,
9748                    err,
9749                )?;
9750                return Ok(secondary_error.into_iter().collect());
9751            }
9752        };
9753        if let Some(target) = completion.abandoned.as_ref() {
9754            send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
9755        }
9756        if !completion.settled {
9757            debug!(
9758                connection_id = connection_id.get(),
9759                corr = frame.header.corr,
9760                frame_type = ?frame.header.ty,
9761                "dropping late or unknown route.bind relay response"
9762            );
9763        }
9764        Ok(secondary_error.into_iter().collect())
9765    }
9766
9767    fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
9768        debug!(connection_id = connection_id.get(), "handling GOODBYE");
9769        // GOODBYE ends the connection's logical session even when its socket
9770        // stays open. Use disconnect teardown so verdicts, client notices and
9771        // scope authority are released at the same lifecycle boundary.
9772        self.cleanup_connection_with_end_reason(
9773            connection_id,
9774            RegistrationEndReason::ExplicitGoodbye,
9775        )
9776        .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
9777        Ok(Vec::new())
9778    }
9779}
9780
9781impl Default for ControlHandler {
9782    fn default() -> Self {
9783        Self::new(Arc::new(Registry::default()))
9784    }
9785}
9786
9787impl crate::supervise::SwapPromotionObserver for ControlHandler {
9788    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
9789        self.apply_registration_capabilities(registration);
9790    }
9791}
9792
9793fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
9794    CapabilityRequirementStatus {
9795        consumer: status.consumer,
9796        capability: status.capability,
9797        need: match status.need {
9798            subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
9799            subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
9800        },
9801        verdict: status.verdict.as_str().to_string(),
9802        episode_seq: status.episode_seq,
9803        config_satisfiable: status.config_satisfiable,
9804        runtime_available: status.runtime_available,
9805        detail: status.detail,
9806    }
9807}
9808
9809fn append_capability_problem_detail(
9810    detail: Option<String>,
9811    capability_detail: Option<String>,
9812) -> Option<String> {
9813    match (detail, capability_detail) {
9814        (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
9815        (Some(detail), None) => Some(detail),
9816        (None, Some(capability_detail)) => Some(capability_detail),
9817        (None, None) => None,
9818    }
9819}
9820
9821fn subc_ops() -> Vec<String> {
9822    SUBC_CONTROL_OPS
9823        .iter()
9824        .map(|op| (*op).to_string())
9825        .collect()
9826}
9827
9828fn module_subc_ops() -> Vec<String> {
9829    SUBC_CONTROL_OPS
9830        .iter()
9831        .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
9832        .map(|op| (*op).to_string())
9833        .collect()
9834}
9835
9836#[cfg(test)]
9837fn module_baseline_control_ops() -> Vec<String> {
9838    MODULE_BASELINE_CONTROL_OPS
9839        .iter()
9840        .map(|op| (*op).to_string())
9841        .collect()
9842}
9843
9844fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
9845    let mut seen = HashSet::new();
9846    let mut effective = Vec::new();
9847    for op in MODULE_BASELINE_CONTROL_OPS {
9848        if seen.insert((*op).to_string()) {
9849            effective.push((*op).to_string());
9850        }
9851    }
9852    for op in declared.unwrap_or_default() {
9853        if seen.insert(op.clone()) {
9854            effective.push(op);
9855        }
9856    }
9857    effective
9858}
9859
9860fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
9861    MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
9862}
9863
9864fn target_module_id(target: &RouteTarget) -> &str {
9865    match target {
9866        RouteTarget::ToolProvider { module_id }
9867        | RouteTarget::ManagementSurface { module_id }
9868        | RouteTarget::InternalService { module_id, .. } => module_id,
9869    }
9870}
9871
9872fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
9873    roles.iter().any(|role| match (target, role) {
9874        (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
9875        (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
9876        (
9877            RouteTarget::InternalService { service_id, .. },
9878            ProviderRole::InternalService {
9879                service_id: provided,
9880                ..
9881            },
9882        ) => service_id == provided,
9883        _ => false,
9884    })
9885}
9886
9887fn is_routable_role(role: &ProviderRole) -> bool {
9888    matches!(
9889        role,
9890        ProviderRole::ToolProvider { .. }
9891            | ProviderRole::ManagementSurface { .. }
9892            | ProviderRole::InternalService { .. }
9893    )
9894}
9895
9896#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9897enum ControlRequestBodyError {
9898    UnknownOp,
9899    InvalidBody,
9900}
9901
9902#[derive(Debug, Deserialize)]
9903struct ControlOpProbe {
9904    op: String,
9905}
9906
9907fn module_control_response_probe(body: &[u8]) -> Result<ControlOpProbe, serde_json::Error> {
9908    serde_json::from_slice(body).or_else(|original| {
9909        #[derive(Deserialize)]
9910        struct ConsentProbe {
9911            protocol: String,
9912        }
9913        match serde_json::from_slice::<ConsentProbe>(body) {
9914            Ok(probe) if probe.protocol == subc_protocol::consent::CONSENT_V1 => {
9915                Ok(ControlOpProbe { op: probe.protocol })
9916            }
9917            _ => Err(original),
9918        }
9919    })
9920}
9921
9922/// Channel-0 push ops this subc version understands. A push whose `op` is not in
9923/// this set is treated as a forward-compat unknown and ignored rather than errored.
9924const MODULE_PUSH_OPS: &[&str] = &["route.status"];
9925
9926fn is_known_module_push_op(body: &[u8]) -> bool {
9927    serde_json::from_slice::<ControlOpProbe>(body)
9928        .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
9929        .unwrap_or(false)
9930}
9931
9932fn is_known_module_request_op(body: &[u8]) -> bool {
9933    serde_json::from_slice::<ControlOpProbe>(body)
9934        .map(|probe| is_module_to_subc_op(&probe.op))
9935        .unwrap_or(false)
9936}
9937
9938fn is_module_to_subc_op(op: &str) -> bool {
9939    MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
9940}
9941
9942fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
9943    debug!(
9944        op = %op,
9945        connection_id = connection_id.get(),
9946        corr,
9947        "control dispatch"
9948    );
9949}
9950
9951fn log_slow_control_dispatch(
9952    dispatch_started_at: Option<StdInstant>,
9953    op: &'static str,
9954    connection_id: ConnectionId,
9955    corr: u64,
9956    route_open_timings: Option<&RouteOpenTimings>,
9957) {
9958    let Some(dispatch_started_at) = dispatch_started_at else {
9959        return;
9960    };
9961    // Use the same monotonic clock as the phase timers and relay deadline.
9962    let elapsed = Instant::from_std(dispatch_started_at).elapsed();
9963    if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
9964        if let Some(timings) = route_open_timings {
9965            warn!(
9966                op = %op,
9967                connection_id = connection_id.get(),
9968                corr,
9969                elapsed_ms = elapsed.as_millis() as u64,
9970                module_id = ?timings.module_id,
9971                principal = %timings.principal,
9972                admission_ms = timings.admission.as_millis() as u64,
9973                project_root_ms = timings.project_root.as_millis() as u64,
9974                egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
9975                bind_send_ms = timings.bind_send.as_millis() as u64,
9976                bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
9977                "slow control dispatch"
9978            );
9979            return;
9980        }
9981        warn!(
9982            op = %op,
9983            connection_id = connection_id.get(),
9984            corr,
9985            elapsed_ms = elapsed.as_millis() as u64,
9986            "slow control dispatch"
9987        );
9988    }
9989}
9990
9991/// A test stand-in for the OS caller lookup, given the recorded endpoints.
9992#[cfg(test)]
9993type TestCallerLookup =
9994    Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
9995
9996/// Every new client operation needs an explicit privilege decision here.
9997fn privileged_operation(request: &impl PrivilegedRequest) -> Option<PrivilegedOperation<'_>> {
9998    request.privileged_operation()
9999}
10000
10001trait PrivilegedRequest {
10002    fn privileged_operation(&self) -> Option<PrivilegedOperation<'_>>;
10003}
10004
10005impl PrivilegedRequest for ClientControlRequest {
10006    fn privileged_operation(&self) -> Option<PrivilegedOperation<'_>> {
10007        let request = self;
10008        match request {
10009        ClientControlRequest::SupervisorRestart { module_id, .. } => {
10010            Some(PrivilegedOperation::Restart { module_id })
10011        }
10012        ClientControlRequest::SupervisorSwap { module_id, .. } => {
10013            Some(PrivilegedOperation::Swap { module_id })
10014        }
10015        ClientControlRequest::SupervisorReload { module_id } => {
10016            Some(PrivilegedOperation::Reload { module_id })
10017        }
10018        ClientControlRequest::SupervisorRescan { preview } => {
10019            if *preview {
10020                None
10021            } else {
10022                Some(PrivilegedOperation::Rescan)
10023            }
10024        }
10025        // Only the well-formed apply asks for approval. A preview changes
10026        // nothing, and any other combination is refused before it touches
10027        // anything, so prompting the operator for it would ask about a request
10028        // that cannot run.
10029        ClientControlRequest::SupervisorRescanChecked {
10030            preview,
10031            candidate,
10032            expect,
10033        } => (!*preview && candidate.is_none() && expect.is_some())
10034            .then_some(PrivilegedOperation::Rescan),
10035        ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
10036            Some(PrivilegedOperation::SetEnabled {
10037                module_id,
10038                enabled: *enabled,
10039            })
10040        }
10041        ClientControlRequest::SupervisorReleaseReserved { module_id } => {
10042            Some(PrivilegedOperation::ReleaseReserved { module_id })
10043        }
10044        ClientControlRequest::ServerDescribe {}
10045        // The announcement uses the same client admission as supervisor.restart,
10046        // but cannot stop anything itself and needs no operator confirmation.
10047        | ClientControlRequest::DaemonShutdownIntent { .. }
10048        | ClientControlRequest::CatalogList { .. }
10049        | ClientControlRequest::RouteOpen { .. }
10050        | ClientControlRequest::RoutePoll { .. }
10051        | ClientControlRequest::SupervisorList {}
10052        | ClientControlRequest::SupervisorHealth {}
10053        | ClientControlRequest::SupervisorHealthProbe { .. }
10054        | ClientControlRequest::SupervisorRoutes { .. }
10055        | ClientControlRequest::SupervisorProvenance { .. }
10056        | ClientControlRequest::SupervisorStderrTail { .. }
10057        | ClientControlRequest::SupervisorTerminals { .. }
10058        | ClientControlRequest::SupervisorSpawnSnapshot {}
10059        | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
10060    }
10061    }
10062}
10063
10064fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
10065    match request {
10066        ClientControlRequest::DaemonShutdownIntent { .. } => ops::DAEMON_SHUTDOWN_INTENT,
10067        ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
10068        ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
10069        ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
10070        ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
10071        ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
10072        ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
10073        ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
10074        ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
10075        ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
10076        ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
10077        ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
10078        ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
10079        ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
10080        ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
10081        ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
10082        ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
10083        ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
10084        ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
10085        ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
10086        ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
10087    }
10088}
10089
10090/// The caller of one `route.open`, as its `route.open refused` line names it.
10091///
10092/// A connection id alone cannot name a caller after the fact: ids restart at
10093/// every daemon restart, so the same number belongs to different modules
10094/// before and after one. These three fields are what the refused line needs
10095/// to be attributable on its own, and they are rendered with the same helpers
10096/// as the `route.open accepted` line so both directions grep alike.
10097struct RouteOpenCaller {
10098    /// The attested principal (`direct`, `reserved:<module>`), or `unattested`
10099    /// when the request presented a consumer identity whose launch nonce did
10100    /// not verify. The claimed module id is never printed as a principal.
10101    principal: String,
10102    /// `none` for an unscoped open; the admitted stamp's label once scope
10103    /// admission issued one; before that, the requested selector's label.
10104    scope: String,
10105    /// The bind identity's harness, Debug-quoted and bounded like a scope ref,
10106    /// because it is a string the caller chose.
10107    harness: String,
10108}
10109
10110impl RouteOpenCaller {
10111    fn new(
10112        principal: Option<&Principal>,
10113        scope: Option<&ScopeSelector>,
10114        identity: &BindIdentity,
10115    ) -> Self {
10116        Self {
10117            principal: principal.map_or_else(|| "unattested".to_string(), principal_label),
10118            scope: scope.map_or_else(|| "none".to_string(), route_open_requested_scope_label),
10119            harness: bounded_log_text(&identity.harness),
10120        }
10121    }
10122
10123    /// For a refusal that has no parsed request at hand: names whatever the
10124    /// frame still says, and `unknown` where it says nothing parseable.
10125    fn unknown() -> Self {
10126        Self {
10127            principal: "unknown".to_string(),
10128            scope: "unknown".to_string(),
10129            harness: "unknown".to_string(),
10130        }
10131    }
10132}
10133
10134/// Longest client-chosen text (a scope ref, a harness) a route.open log line
10135/// prints before cutting it.
10136const MAX_LOGGED_TEXT_CHARS: usize = 64;
10137
10138/// `value` Debug-quoted, so control characters land escaped and cannot forge
10139/// a log line, and cut to [`MAX_LOGGED_TEXT_CHARS`] characters with a trailing
10140/// `…` inside the quotes so one request cannot flood the line.
10141fn bounded_log_text(value: &str) -> String {
10142    let mut bounded: String = value.chars().take(MAX_LOGGED_TEXT_CHARS).collect();
10143    if value.chars().nth(MAX_LOGGED_TEXT_CHARS).is_some() {
10144        bounded.push('…');
10145    }
10146    format!("{bounded:?}")
10147}
10148
10149/// The `scope=` field of the `route.open accepted` line, from the stamp that
10150/// admission issued: `<kind>/<owner>/<ref>@<epoch>`. The ref is Debug-quoted
10151/// so a ref holding control characters cannot forge a log line, and cut to 64
10152/// characters with a trailing `…` to bound the line. The bind proof and the
10153/// scope's attributes never appear.
10154fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
10155    use subc_protocol::scope::ScopeKind;
10156    let kind = match stamp.kind {
10157        ScopeKind::Head => "head",
10158        ScopeKind::Worker => "worker",
10159        ScopeKind::Ephemeral => "ephemeral",
10160        ScopeKind::Federated => "federated",
10161        _ => "unknown",
10162    };
10163    format!(
10164        "{kind}/{}/{}@{}",
10165        principal_label(&stamp.owner),
10166        bounded_log_text(&stamp.scope_ref),
10167        stamp.scope_epoch
10168    )
10169}
10170
10171/// The `scope=` field of a `route.open refused` line for an open refused
10172/// before scope admission issued a stamp: `requested/<owner>/<ref>@<epoch>`.
10173/// The kind is not known yet, so the literal `requested` stands in its place.
10174/// The owner is Debug-quoted as well as the ref, because unlike a stamp's
10175/// owner it has not been checked against the configured modules: a quoted
10176/// owner means the value was never validated. The bind proof never appears.
10177fn route_open_requested_scope_label(selector: &ScopeSelector) -> String {
10178    let epoch = selector
10179        .scope_epoch
10180        .map_or_else(|| "none".to_string(), |epoch| epoch.to_string());
10181    format!(
10182        "requested/{}/{}@{epoch}",
10183        bounded_log_text(&principal_label(&selector.owner)),
10184        bounded_log_text(&selector.scope_ref),
10185    )
10186}
10187
10188fn principal_label(principal: &Principal) -> String {
10189    match principal {
10190        Principal::Reserved { module_id } => format!("reserved:{module_id}"),
10191        Principal::Direct => "direct".to_string(),
10192        other => format!("{other:?}"),
10193    }
10194}
10195
10196fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
10197    match request {
10198        ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
10199        ModuleControlRequestFromModule::CatalogUpdateRoleDeclarations { .. } => {
10200            MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS
10201        }
10202        ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
10203        ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
10204        ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
10205        ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
10206        ModuleControlRequestFromModule::RouteAuthorize { .. } => {
10207            subc_protocol::scope::ROUTE_AUTHORIZE_OP
10208        }
10209        ModuleControlRequestFromModule::AgentCommands { .. } => "supervisor.agent_commands",
10210        _ => "unsupported",
10211    }
10212}
10213
10214fn parse_client_control_request(
10215    body: &[u8],
10216) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
10217    serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
10218        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
10219            Ok(probe)
10220                if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
10221                    || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
10222            {
10223                ControlRequestBodyError::InvalidBody
10224            }
10225            Ok(_) => ControlRequestBodyError::UnknownOp,
10226            Err(_) => ControlRequestBodyError::InvalidBody,
10227        };
10228        (err, classification)
10229    })
10230}
10231
10232fn parse_module_control_request_from_module(
10233    body: &[u8],
10234) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
10235    serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
10236        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
10237            Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
10238            Ok(_) => ControlRequestBodyError::UnknownOp,
10239            Err(_) => ControlRequestBodyError::InvalidBody,
10240        };
10241        (err, classification)
10242    })
10243}
10244
10245#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
10246enum ProviderRoleKind {
10247    ToolProvider,
10248    PipelineStage,
10249    ManagementSurface,
10250    InternalService,
10251}
10252
10253fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
10254    match role {
10255        ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
10256        ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
10257        ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
10258        ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
10259    }
10260}
10261
10262fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
10263    roles.iter().map(provider_role_kind).collect()
10264}
10265
10266/// Most refused scope records named individually in the log per sync; the
10267/// `refused` count on the accepted line is always complete.
10268const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
10269
10270/// Per-outcome counts of one accepted `scope.sync`, for its log line.
10271#[derive(Debug, Default, PartialEq, Eq)]
10272struct ScopeOutcomeCounts {
10273    created: usize,
10274    replaced: usize,
10275    updated: usize,
10276    unchanged: usize,
10277    refused: usize,
10278}
10279
10280impl ScopeOutcomeCounts {
10281    fn of(results: &[ScopeRecordResult]) -> Self {
10282        let mut counts = Self::default();
10283        for result in results {
10284            let slot = match result.outcome {
10285                ScopeRecordOutcome::Created => &mut counts.created,
10286                ScopeRecordOutcome::Replaced => &mut counts.replaced,
10287                ScopeRecordOutcome::Updated => &mut counts.updated,
10288                ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
10289                ScopeRecordOutcome::Refused => &mut counts.refused,
10290            };
10291            *slot += 1;
10292        }
10293        counts
10294    }
10295}
10296
10297#[cfg(test)]
10298mod scope_outcome_count_tests {
10299    use super::*;
10300
10301    fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
10302        ScopeRecordResult {
10303            scope_ref: "r".to_string(),
10304            scope_epoch: 1,
10305            outcome,
10306            code: None,
10307            message: None,
10308            version: None,
10309            parent_state: None,
10310        }
10311    }
10312
10313    /// Each outcome lands in its own count, so a refused record can never be
10314    /// hidden inside the total the log already printed.
10315    #[test]
10316    fn every_outcome_is_counted_in_its_own_field() {
10317        let results = [
10318            result(ScopeRecordOutcome::Created),
10319            result(ScopeRecordOutcome::Created),
10320            result(ScopeRecordOutcome::Replaced),
10321            result(ScopeRecordOutcome::Updated),
10322            result(ScopeRecordOutcome::Unchanged),
10323            result(ScopeRecordOutcome::Refused),
10324            result(ScopeRecordOutcome::Refused),
10325            result(ScopeRecordOutcome::Refused),
10326        ];
10327        assert_eq!(
10328            ScopeOutcomeCounts::of(&results),
10329            ScopeOutcomeCounts {
10330                created: 2,
10331                replaced: 1,
10332                updated: 1,
10333                unchanged: 1,
10334                refused: 3,
10335            }
10336        );
10337    }
10338}
10339
10340/// Return whether a catalog change can create a newly violating live route.
10341/// Removing an attested claim is intentionally excluded: it makes fewer routes
10342/// forbidden and therefore must leave the existing route census untouched.
10343fn capability_census_trigger(
10344    old: Option<&CapabilityDeclarations>,
10345    new: Option<&CapabilityDeclarations>,
10346) -> bool {
10347    let old_provides = old
10348        .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
10349        .unwrap_or_default();
10350    let old_denies = old
10351        .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
10352        .unwrap_or_default();
10353    let new = new.cloned().unwrap_or(CapabilityDeclarations {
10354        provides: Vec::new(),
10355        requires: Vec::new(),
10356        must_never_reach: Vec::new(),
10357    });
10358
10359    new.provides
10360        .iter()
10361        .any(|capability| !old_provides.contains(capability))
10362        || new
10363            .must_never_reach
10364            .iter()
10365            .any(|capability| !old_denies.contains(capability))
10366}
10367
10368fn catalog_update_frozen_field_message(
10369    registered: &ModuleManifest,
10370    provides: &[ProviderRole],
10371) -> Option<String> {
10372    let old_has_provides = !registered.provides.is_empty();
10373    let new_has_provides = !provides.is_empty();
10374    if old_has_provides != new_has_provides {
10375        return Some(format!(
10376            "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
10377            registered.module_id
10378        ));
10379    }
10380
10381    if provider_role_kind_set(&registered.provides) != provider_role_kind_set(provides) {
10382        return Some(format!(
10383            "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
10384            registered.module_id
10385        ));
10386    }
10387
10388    let registered_concurrency = manifest_concurrency(registered);
10389    let mut candidate = registered.clone();
10390    candidate.provides = provides.to_vec();
10391    let candidate_concurrency = manifest_concurrency(&candidate);
10392    if candidate_concurrency != registered_concurrency {
10393        return Some(format!(
10394            "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
10395            registered.module_id, registered_concurrency, candidate_concurrency
10396        ));
10397    }
10398
10399    // control_ops live beside the manifest in the HELLO body, not inside
10400    // ModuleManifest, so a provides-only catalog.update cannot change them.
10401    None
10402}
10403
10404fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
10405    manifest.provides.iter().any(is_routable_role)
10406}
10407
10408/// Returns the routable-provider concurrency subc should enforce for this manifest.
10409///
10410/// ToolProvider and ManagementSurface store their delivery concurrency directly.
10411/// InternalService has no role-specific concurrency field, so it retains the
10412/// existing ModuleManaged default for backward compatibility.
10413fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
10414    manifest
10415        .provides
10416        .iter()
10417        .find_map(|provider| match provider {
10418            ProviderRole::ToolProvider { concurrency, .. }
10419            | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
10420            ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
10421        })
10422        .unwrap_or(Concurrency::ModuleManaged)
10423}
10424
10425/// True when the manifest carries a ManagementSurface role whose concurrency
10426/// was RESOLVED BY SERDE DEFAULT rather than declared. Reads the raw HELLO
10427/// bytes because the typed manifest deliberately erases that distinction: the
10428/// default exists for wire compatibility, and this probe exists so the default
10429/// stays observable. Any parse irregularity returns false -- the caller only
10430/// logs, and a malformed body already failed registration upstream.
10431fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
10432    let has_management_surface = manifest
10433        .provides
10434        .iter()
10435        .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
10436    if !has_management_surface {
10437        return false;
10438    }
10439    let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
10440        return false;
10441    };
10442    let Some(provides) = raw
10443        .get("manifest")
10444        .and_then(|manifest| manifest.get("provides"))
10445        .and_then(serde_json::Value::as_array)
10446    else {
10447        return false;
10448    };
10449    // ProviderRole is internally tagged (`tag = "role"`), so the wire shape is
10450    // flat: {"role": "management_surface", ..., "concurrency": ...} -- verified
10451    // against the management_surface_manifest_without_concurrency golden, not
10452    // recalled (the externally-tagged guess was this function's first bug).
10453    provides.iter().any(|role| {
10454        role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
10455            && role.get("concurrency").is_none()
10456    })
10457}
10458
10459fn negotiate_version(peer_version: u8) -> Result<u8, String> {
10460    if peer_version != PROTOCOL_VERSION {
10461        return Err(format!(
10462            "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
10463        ));
10464    }
10465    Ok(PROTOCOL_VERSION)
10466}
10467
10468fn pong(frame: &Frame) -> Result<Frame, RouterError> {
10469    Frame::build_with_version(
10470        response_version(frame),
10471        FrameType::Pong,
10472        frame.header.flags,
10473        0,
10474        0,
10475        frame.header.corr,
10476        Vec::new(),
10477    )
10478    .map_err(RouterError::FrameBuild)
10479}
10480
10481pub(crate) fn control_error_frame(
10482    frame: &Frame,
10483    code: &'static str,
10484    message: impl Into<String>,
10485) -> Result<Frame, RouterError> {
10486    control_error_body_frame(
10487        frame,
10488        ErrorBody {
10489            code: code.to_string(),
10490            message: message.into(),
10491            detail: None,
10492        },
10493    )
10494}
10495
10496/// A channel-0 ERROR for a refusal the daemon decided itself: every
10497/// route.open refusal, the drain `module_reloading` and scope-change/end
10498/// bodies, spawn-subscription and supervisor refusals. Carries the
10499/// daemon-origin bit, which is what lets a client treat `module_reloading`
10500/// as proof that its request never reached a module. Response, push,
10501/// goodbye and relay-request frames keep the plain control flags in their
10502/// own builders.
10503pub(crate) fn control_error_body_frame(
10504    frame: &Frame,
10505    error: ErrorBody,
10506) -> Result<Frame, RouterError> {
10507    control_error_frame_with_flags(frame, error, daemon_error_flags())
10508}
10509
10510/// A channel-0 ERROR relaying a module's own rejection body unchanged: a
10511/// refused route.bind answered as a route.open refusal, or a refused
10512/// module-control RPC. Plain control flags — the code and message are
10513/// module-chosen, so the frame must not claim daemon authorship, even when
10514/// the module picks a daemon code like `module_reloading`.
10515fn control_relayed_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
10516    control_error_frame_with_flags(frame, error, control_flags())
10517}
10518
10519fn control_error_frame_with_flags(
10520    frame: &Frame,
10521    error: ErrorBody,
10522    flags: Flags,
10523) -> Result<Frame, RouterError> {
10524    let body = serde_json::to_vec(&error).map_err(|err| {
10525        RouterError::backend(
10526            0,
10527            frame.header.corr,
10528            format!("failed to encode control ERROR: {err}"),
10529        )
10530    })?;
10531
10532    Frame::build_with_version(
10533        response_version(frame),
10534        FrameType::Error,
10535        flags,
10536        0,
10537        0,
10538        frame.header.corr,
10539        body,
10540    )
10541    .map_err(RouterError::FrameBuild)
10542}
10543
10544pub(crate) fn control_response_body_frame<T: Serialize>(
10545    frame: &Frame,
10546    reply: &T,
10547    label: &'static str,
10548) -> Result<Frame, RouterError> {
10549    let body = serde_json::to_vec(reply).map_err(|err| {
10550        RouterError::backend(
10551            0,
10552            frame.header.corr,
10553            format!("failed to encode {label}: {err}"),
10554        )
10555    })?;
10556
10557    Frame::build_with_version(
10558        response_version(frame),
10559        FrameType::Response,
10560        control_flags(),
10561        0,
10562        0,
10563        frame.header.corr,
10564        body,
10565    )
10566    .map_err(RouterError::FrameBuild)
10567}
10568
10569/// Map a forwarding failure to the wire code a client sees.
10570///
10571/// The code is not a label: clients BRANCH on it. Both SDKs decide "retry in
10572/// place" with `subc_protocol::error_codes::is_retryable_route_open`, so a code
10573/// chosen here decides whether a caller retries or gives up.
10574///
10575/// That makes attribution the load-bearing property, not merely having a code. A
10576/// permanent fault published as a retryable one produces a fleet-wide retry storm
10577/// against something that can never recover; a transient fault published as
10578/// permanent gives up on work that would have succeeded. Both look correct in a
10579/// log, which is why `retryability_of_forwarding_codes_matches_the_failure` pins
10580/// the mapping per variant rather than merely asserting that some code exists.
10581///
10582/// That fence partitions by RETRYABILITY, which is coarser than identity: swapping
10583/// two codes on the same side of the boundary passes it. Measured rather than
10584/// assumed — `NoModuleConnection` re-pointed at `module_reloading` is caught only
10585/// by `supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up`,
10586/// a test named for something else that happens to assert the string.
10587///
10588/// That accidental coverage is deliberately left alone rather than promoted to a
10589/// named test, because it guards a property this function does not promise.
10590/// Checked at source: every consumer branches on the RETRYABLE SET and none on a
10591/// specific code within a class, so identity is free to change and only the
10592/// partition is a contract. Splitting it out would assert a guarantee nothing
10593/// depends on — and a suite that promises more than the code does is the harder
10594/// thing to correct later, because the next reader cannot tell which assertions
10595/// are load-bearing.
10596///
10597/// Pin identity here the moment a consumer branches on a specific code.
10598/// The spawn a HELLO would register as and the provider declarations it
10599/// requests, carried from the HELLO's checks to the point the registry
10600/// accepts it.
10601struct HelloAuthority {
10602    record: Arc<crate::spawn_authority::SpawnAuthorityRecord>,
10603    requested: ProviderDeclarations,
10604    /// Whether admitting this HELLO froze the spawn's requested declarations,
10605    /// so a refusal after admission undoes that freeze and no other.
10606    froze: std::sync::atomic::AtomicBool,
10607}
10608
10609fn forwarding_error_code(err: &ForwardingError) -> &'static str {
10610    match err {
10611        ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
10612        ForwardingError::NoModuleConnection => "target_unavailable",
10613        ForwardingError::ModuleReloading { .. } => "module_reloading",
10614        ForwardingError::ClientRouteChannelExhausted { .. }
10615        | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
10616        ForwardingError::StaleModuleEndpoint
10617        | ForwardingError::UnknownReservation { .. }
10618        | ForwardingError::ConnectionClosing { .. }
10619        | ForwardingError::ClientEgressClosed { .. }
10620        | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
10621        // Only a swap candidate's registration can produce this, and it means
10622        // exactly what a second active HELLO for a live id means.
10623        ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
10624        ForwardingError::RelayCorrelationExhausted
10625        | ForwardingError::RouteOpenBuild(_)
10626        | ForwardingError::Poisoned => "forwarding_error",
10627    }
10628}
10629
10630fn response_version(frame: &Frame) -> u8 {
10631    if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
10632        frame.header.ver
10633    } else {
10634        PROTOCOL_VERSION
10635    }
10636}
10637
10638fn control_flags() -> Flags {
10639    Flags::new(false, Priority::Passive, false)
10640}
10641
10642/// GOODBYE for a route.bind the daemon gave up on after reserving the module's
10643/// channel. The target is the module (a client never saw the route), so this
10644/// takes the module path: delivered late rather than dropped when the module's
10645/// queue is momentarily full, and never closing its connection.
10646fn send_goodbye_target_best_effort(
10647    counters: &DaemonCounters,
10648    target: &GoodbyeTarget,
10649    context: &'static str,
10650) {
10651    let Ok(frame) = Frame::build_with_version(
10652        target.negotiated_ver,
10653        FrameType::Goodbye,
10654        control_flags(),
10655        target.channel,
10656        target.epoch,
10657        0,
10658        Vec::new(),
10659    ) else {
10660        return;
10661    };
10662    crate::forwarding::send_module_route_goodbye(
10663        counters,
10664        &target.sink,
10665        frame,
10666        target.module_id.as_deref(),
10667        context,
10668    );
10669}
10670
10671fn route_close_reason_for_opener(
10672    reason: RouteCloseReason,
10673    accepts_close_reasons: Option<&[String]>,
10674) -> RouteCloseReason {
10675    if reason == RouteCloseReason::AuthorityChanged
10676        && !accepts_close_reasons
10677            .is_some_and(|reasons| reasons.iter().any(|reason| reason == "authority_changed"))
10678    {
10679        // Older readers forbid reopening for unknown reasons. Reload already
10680        // permits reopening and, like an authority change, requires a new bind.
10681        RouteCloseReason::Reload
10682    } else {
10683        reason
10684    }
10685}
10686
10687pub(crate) fn send_route_control_pushes(
10688    forwarding: &ForwardingTable,
10689    routes: Vec<EndpointRoute>,
10690    push: ClientControlPush,
10691) {
10692    let reason = match &push {
10693        ClientControlPush::RouteClosing { reason, .. }
10694        | ClientControlPush::RouteClosed { reason, .. } => *reason,
10695    };
10696    let mut targets: Vec<(GoodbyeTarget, RouteCloseReason, Vec<u16>)> = Vec::new();
10697    for route in routes {
10698        let recipient_reason =
10699            route_close_reason_for_opener(reason, route.accepts_close_reasons.as_deref());
10700        let target = route.goodbye_target;
10701        if let Some((existing, _, channels)) = targets.iter_mut().find(|(existing, reason, _)| {
10702            existing.connection_id == target.connection_id && *reason == recipient_reason
10703        }) {
10704            debug_assert_eq!(
10705                existing.negotiated_ver, target.negotiated_ver,
10706                "one connection cannot negotiate multiple frame versions"
10707            );
10708            if !channels.contains(&target.channel) {
10709                channels.push(target.channel);
10710            }
10711            continue;
10712        }
10713        let channel = target.channel;
10714        targets.push((target, recipient_reason, vec![channel]));
10715    }
10716    for (target, recipient_reason, mut channels) in targets {
10717        channels.sort_unstable();
10718        let mut push = push.clone();
10719        match &mut push {
10720            ClientControlPush::RouteClosing {
10721                channels: covered,
10722                reason,
10723                ..
10724            }
10725            | ClientControlPush::RouteClosed {
10726                channels: covered,
10727                reason,
10728                ..
10729            } => {
10730                *covered = channels;
10731                *reason = recipient_reason;
10732            }
10733        }
10734        let body = match serde_json::to_vec(&push) {
10735            Ok(body) => body,
10736            Err(err) => {
10737                warn!(error = %err, "failed to serialize route lifecycle control PUSH");
10738                continue;
10739            }
10740        };
10741        let frame = match Frame::build_with_version(
10742            target.negotiated_ver,
10743            FrameType::Push,
10744            control_flags(),
10745            0,
10746            0,
10747            0,
10748            body.clone(),
10749        ) {
10750            Ok(frame) => frame,
10751            Err(err) => {
10752                warn!(
10753                    route_channel = target.channel,
10754                    error = %err,
10755                    "failed to build route lifecycle control PUSH frame"
10756                );
10757                continue;
10758            }
10759        };
10760        if let Err(err) = target.sink.try_send(frame) {
10761            if target.close_on_delivery_failure() {
10762                warn!(
10763                    target_connection_id = target.connection_id.get(),
10764                    route_channel = target.channel,
10765                    error = %err,
10766                    "route lifecycle control PUSH was not delivered to client; closing target connection"
10767                );
10768                let _ = forwarding.escalate_client_delivery_failure(
10769                    target.connection_id,
10770                    target.channel,
10771                    target.epoch,
10772                    CloseReason::new(
10773                        "route_lifecycle_push_delivery_failed",
10774                        format!(
10775                            "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
10776                            target.channel
10777                        ),
10778                    ),
10779                    crate::forwarding::UndeliveredFrame {
10780                        module_id: target.module_id.as_deref(),
10781                        sink: &target.sink,
10782                    },
10783                );
10784            }
10785        }
10786    }
10787}
10788
10789#[cfg(test)]
10790mod tests {
10791    include!("control/agent_commands_tests.rs");
10792    include!("control/resource_revision_tests.rs");
10793    include!("control/route_open_recovery_tests.rs");
10794    #[tokio::test]
10795    async fn route_busy_wait_attribution_resolves_targets_instead_of_trusting_selectors() {
10796        let registry = Arc::new(Registry::default());
10797        let handler = ControlHandler::new(registry);
10798        assert_eq!(
10799            handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
10800            None
10801        );
10802        let (ctx, mut rx) = route_ctx(ConnectionId::new(901));
10803        hello_via_sink(
10804            &handler,
10805            &ctx,
10806            &mut rx,
10807            hello_frame("daemon", PROTOCOL_VERSION, 1),
10808        )
10809        .await;
10810        assert_eq!(
10811            handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
10812            Some("daemon".into())
10813        );
10814        assert_eq!(
10815            handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("missing")),
10816            None
10817        );
10818        assert_eq!(
10819            handler.resolve_reader_wait_target(ops::SUPERVISOR_TERMINALS, Some("daemon")),
10820            None,
10821            "a registered module is not necessarily supervised"
10822        );
10823        assert_eq!(
10824            handler.resolve_reader_wait_target(ops::SUPERVISOR_PROVENANCE, None),
10825            None,
10826            "aggregate provenance has no single target"
10827        );
10828        assert_eq!(
10829            handler.resolve_reader_wait_target(ops::APP_OPEN, Some("daemon")),
10830            None
10831        );
10832    }
10833
10834    mod peer_manifest {
10835        include!("peer_manifest_tests.rs");
10836    }
10837    mod catalog_declarations {
10838        include!("control/catalog_tests.rs");
10839    }
10840    mod activity_read {
10841        include!("activity_read_tests.rs");
10842    }
10843
10844    mod lifecycle_activity {
10845        include!("control_lifecycle_tests.rs");
10846    }
10847    #[cfg(unix)]
10848    #[tokio::test]
10849    async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
10850        let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
10851        let path = dir.join("subc.jsonc");
10852        std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
10853            "program":"/bin/sleep","args":["60"],"protocol":"none",
10854            "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
10855        }}}).to_string()).unwrap();
10856        let mut configured = crate::daemon_config::load(&path)
10857            .unwrap()
10858            .unwrap()
10859            .modules
10860            .pop()
10861            .unwrap();
10862        let registry = std::sync::Arc::new(crate::Registry::default());
10863        let handle = crate::SupervisorHandle::new();
10864        let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
10865            .with_handle(handle.clone());
10866        let module = supervisor
10867            .supervise_configured_with_health(
10868                configured.module_spec(),
10869                true,
10870                configured.health.clone(),
10871                None,
10872                configured.restart,
10873            )
10874            .unwrap();
10875        let handler = super::ControlHandler::new(registry).with_supervisor(handle);
10876        let before = module.status().unwrap().pid;
10877        configured.health.http = Some("http://127.0.0.1:1/healthz".into());
10878        configured.health.cadence = std::time::Duration::from_secs(3600);
10879        let health_only = handler
10880            .reconcile_supervised_modules(
10881                &supervisor,
10882                vec![configured.clone()],
10883                false,
10884                Principal::Direct,
10885            )
10886            .await
10887            .unwrap();
10888        assert!(
10889            health_only.0.changed_pending_reload.is_empty(),
10890            "health policy is already applied live"
10891        );
10892        assert_eq!(module.status().unwrap().pid, before);
10893        assert_eq!(
10894            module.configuration().unwrap().1.http,
10895            configured.health.http
10896        );
10897        configured.args = vec!["61".into()];
10898        let launch = handler
10899            .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
10900            .await
10901            .unwrap();
10902        assert_eq!(launch.0.changed_pending_reload, ["stock"]);
10903        assert_eq!(
10904            module.status().unwrap().pid,
10905            before,
10906            "a launch edit is stored until reload"
10907        );
10908        module.drain().await.unwrap();
10909    }
10910    use cortexkit_test_support::ScratchDir;
10911    use std::{
10912        collections::BTreeMap,
10913        fmt,
10914        path::PathBuf,
10915        sync::{Arc, Mutex},
10916        time::Duration,
10917    };
10918
10919    use serde_json::{json, Value};
10920    use subc_protocol::{
10921        manifest::{
10922            Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
10923            ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
10924        },
10925        session::HealthStatus,
10926        FrameType,
10927    };
10928
10929    use super::*;
10930    use crate::{
10931        forwarding::{DataRoute, DataRouteState},
10932        registry::ChannelState,
10933        router::FrameSink,
10934        stderr_tail::DEFAULT_MAX_LINE_BYTES,
10935        supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
10936        RouteCtx, Router,
10937    };
10938    use tokio::{
10939        sync::mpsc,
10940        time::{sleep, Instant},
10941    };
10942    use tracing::{
10943        field::{Field, Visit},
10944        Event, Subscriber,
10945    };
10946    use tracing_subscriber::{layer::Context, prelude::*, Layer};
10947
10948    /// Locates the `fake-aft-stub` binary from a `src/lib.rs` unit test.
10949    ///
10950    /// `CARGO_BIN_EXE_*` (compile-time `env!` and runtime `std::env::var` alike)
10951    /// is only populated for `tests/*.rs` integration test binaries -- this file
10952    /// compiles as part of the library target, which gets neither. The binary
10953    /// lives in the profile folder this test was built into,
10954    /// `<target-dir>/<profile>/fake-aft-stub`, which `cargo_profile_bin` finds
10955    /// from the test's own path.
10956    ///
10957    /// THE BINARY IS NOT ALWAYS THERE, and the existence check below is why.
10958    /// `cargo test -p subc-core` builds every target including `[[bin]]`, so the
10959    /// stub is on disk; `cargo test -p subc-core --lib` builds ONLY the library
10960    /// test and leaves the stub unbuilt. A bare spawn then fails with a raw
10961    /// `NotFound`, which reads as a broken test rather than an unbuilt
10962    /// dependency -- so state the cause and the remedy instead. Deliberately a
10963    /// panic and not a silent skip: a test that quietly passes when it could not
10964    /// run is worse than one that fails, because it reports health it never
10965    /// verified.
10966    fn fake_aft_stub_path() -> PathBuf {
10967        let path = cortexkit_test_support::cargo_profile_bin("fake-aft-stub");
10968        assert!(
10969            path.exists(),
10970            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
10971             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
10972            path.display()
10973        );
10974        path
10975    }
10976
10977    /// Whether clients retry `code` in place: the predicate itself, never a copy
10978    /// of its set. A copied list breaks silently when a code is added to or
10979    /// removed from the real one, and a stale copy here would let exactly the
10980    /// failure this test exists to catch pass.
10981    fn client_retries(code: &str) -> bool {
10982        subc_protocol::error_codes::is_retryable_route_open(code)
10983    }
10984
10985    /// A code is not a label — clients branch on it, so publishing the wrong KIND
10986    /// of failure is worse than publishing none. A permanent fault dressed as
10987    /// retryable makes every client in the fleet retry forever against something
10988    /// that cannot recover; a transient fault dressed as permanent abandons work
10989    /// that would have succeeded.
10990    ///
10991    /// Asserting "a code exists" cannot catch either, because the string is free
10992    /// to say anything. This enumerates every variant and pins which side of the
10993    /// retry boundary it lands on, so a new variant must be classified here
10994    /// deliberately rather than inheriting whichever arm it was appended to.
10995    #[test]
10996    fn retryability_of_forwarding_codes_matches_the_failure() {
10997        // Transient by nature: the target is booting, reloading, or its endpoint
10998        // was swapped mid-flight. Retrying is how these resolve.
10999        let transient = [
11000            ForwardingError::NoModuleConnection,
11001            ForwardingError::ModuleReloading {
11002                module_id: "m".into(),
11003            },
11004            ForwardingError::StaleModuleEndpoint,
11005            ForwardingError::UnknownReservation {
11006                client_channel: 1,
11007                module_channel: 1,
11008            },
11009            ForwardingError::ConnectionClosing {
11010                connection_id: ConnectionId::new(1),
11011            },
11012            ForwardingError::ClientEgressClosed {
11013                connection_id: ConnectionId::new(1),
11014            },
11015            ForwardingError::ModuleEgressUnavailable {
11016                connection_id: ConnectionId::new(1),
11017            },
11018        ];
11019        for err in transient {
11020            let code = forwarding_error_code(&err);
11021            assert!(
11022                client_retries(code),
11023                "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
11024            );
11025        }
11026
11027        // Not fixed by retrying. Channel and correlation exhaustion need the
11028        // caller to close routes, and a poisoned lock is a daemon that cannot
11029        // recover at all — the worst thing to advertise as retryable, since every
11030        // client would storm a daemon that will never answer.
11031        let permanent = [
11032            ForwardingError::ConnectionRoleConflict {
11033                connection_id: ConnectionId::new(1),
11034            },
11035            ForwardingError::ClientRouteChannelExhausted {
11036                connection_id: ConnectionId::new(1),
11037            },
11038            ForwardingError::ModuleRouteChannelExhausted {
11039                endpoint: ModuleEndpointId {
11040                    connection_id: ConnectionId::new(1),
11041                    generation: 1,
11042                },
11043            },
11044            ForwardingError::RelayCorrelationExhausted,
11045            ForwardingError::RouteOpenBuild("x".into()),
11046            ForwardingError::Poisoned,
11047        ];
11048        for err in permanent {
11049            let code = forwarding_error_code(&err);
11050            assert!(
11051                !client_retries(code),
11052                "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
11053            );
11054        }
11055    }
11056
11057    /// The principal is the daemon's answer to "who is calling", and modules
11058    /// branch on it: aft gates bash on it, cerebellum gates browser control,
11059    /// plexus gates connector invocation. So a stamp is an authorization input in
11060    /// another process, not a label — and both possible answers SUCCEED, which is
11061    /// what makes a wrong one quiet. An unattested caller stamped `Reserved` hands
11062    /// first-party capability to something that never proved it; a supervised one
11063    /// stamped `Direct` silently strips a module of capability it is entitled to.
11064    ///
11065    /// Neither shows up in a test that only checks the bind succeeded. Before this
11066    /// test the only coverage was accidental —
11067    /// `route_open_round_trip_via_tagged_shape_forwards_through_stub` asserts the
11068    /// stamped principal on its way past, so narrowing that wire-shape test to its
11069    /// stated subject would have deleted the last assertion on this value. It
11070    /// still asserts the stamp, which is now redundancy rather than the only
11071    /// guard: both fail under the same mutation, and this one names the reason.
11072    /// SCOPE: this handler's supervisor has spawned nothing, so
11073    /// `spawned_consumer_authorized` can only ever return false and the GRANT arm
11074    /// is unreachable here. Both assertions below are refusals, and a mutant that
11075    /// refuses everything would satisfy them.
11076    ///
11077    /// The grant side is covered where a real nonce exists: `tests/forwarding.rs`
11078    /// spawns a supervised consumer, reads its live nonce, and asserts the module
11079    /// observed `principal.kind == "reserved"` carrying that module_id — verified
11080    /// at source rather than assumed, since a citation is a claim about another
11081    /// file and ages like one. Recorded because a harness that structurally
11082    /// cannot reach an arm reports "none" for that arm identically to one that
11083    /// covers it and found nothing.
11084    #[tokio::test]
11085    async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
11086        let handler = ControlHandler::default();
11087        let frame =
11088            Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
11089
11090        // Absent consumer_identity is the ordinary case: a human at a terminal, or
11091        // any process holding the connection file. Nothing was proved, so nothing
11092        // may be granted beyond the unattested floor.
11093        let stamped = handler
11094            .route_open_principal(ConnectionId::new(1), &frame, None)
11095            .unwrap()
11096            .unwrap();
11097        assert_eq!(
11098            stamped,
11099            Principal::Direct,
11100            "a caller that proved nothing must not be stamped as a supervised module"
11101        );
11102
11103        // A claimed module_id with a nonce no supervised child was given is a
11104        // forgery attempt, not a weaker caller: it must be REFUSED rather than
11105        // quietly demoted to Direct, or an impersonation attempt looks identical
11106        // to an ordinary unattested connection.
11107        let forged = handler
11108            .route_open_principal(
11109                ConnectionId::new(1),
11110                &frame,
11111                Some(ConsumerIdentity {
11112                    module_id: "aft".to_string(),
11113                    launch_nonce: "not-a-real-nonce".to_string(),
11114                }),
11115            )
11116            .unwrap();
11117        let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
11118        assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
11119    }
11120
11121    /// The test above hands `route_open_principal` an identity it built itself,
11122    /// which proves the stamping rule and nothing about where the identity comes
11123    /// from. The real producer is a wire body, and the two are joined by a serde
11124    /// field name that nothing else asserts.
11125    ///
11126    /// That join fails quietly in one specific way: an unrecognised key is simply
11127    /// absent after parsing, so a renamed or misspelled `consumer_identity`
11128    /// yields `None` and every supervised module silently drops to `Direct`.
11129    /// Capability-wise that is the safe direction, but it surfaces far from its
11130    /// cause — as a module mysteriously refused bash — and it would pass every
11131    /// test that builds its own input.
11132    ///
11133    /// Deliberately NOT closed with `deny_unknown_fields`: refusing unknown keys
11134    /// would break every client the moment the daemon gains a field, trading a
11135    /// quiet demotion for a hard refusal on additive change. Asserting the join
11136    /// instead means a rename breaks a test here rather than the fleet.
11137    #[test]
11138    fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
11139        let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
11140        let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
11141        let ClientControlRequest::RouteOpen {
11142            consumer_identity, ..
11143        } = parsed
11144        else {
11145            panic!("route.open body must parse as RouteOpen");
11146        };
11147        assert_eq!(
11148            consumer_identity,
11149            Some(ConsumerIdentity {
11150                module_id: "aft".to_string(),
11151                launch_nonce: "n".to_string(),
11152            }),
11153            "the wire field name must reach the value route_open_principal reads"
11154        );
11155    }
11156
11157    fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
11158        ModuleManifest::builder(module_id, "0.1.0")
11159            .protocol_ver(protocol_ver)
11160            .provides(vec![ProviderRole::ToolProvider {
11161                tools: vec![Tool {
11162                    name: "read".to_string(),
11163                    description: None,
11164                    execution_mode: ExecutionMode::Pure,
11165                    schema: json!({"type": "object"}),
11166                }],
11167                identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
11168                concurrency: Concurrency::ModuleManaged,
11169                emits_push: true,
11170                sub_supervises: true,
11171            }])
11172            .build()
11173    }
11174
11175    fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
11176        hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
11177    }
11178
11179    fn hello_frame_with_control_ops(
11180        module_id: &str,
11181        protocol_ver: u8,
11182        corr: u64,
11183        control_ops: Option<Vec<String>>,
11184    ) -> Frame {
11185        hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
11186    }
11187
11188    fn hello_frame_with_nonce(
11189        module_id: &str,
11190        protocol_ver: u8,
11191        corr: u64,
11192        launch_nonce: Option<&str>,
11193    ) -> Frame {
11194        hello_frame_full(
11195            module_id,
11196            protocol_ver,
11197            corr,
11198            None,
11199            launch_nonce.map(ToOwned::to_owned),
11200        )
11201    }
11202
11203    fn hello_frame_full(
11204        module_id: &str,
11205        protocol_ver: u8,
11206        corr: u64,
11207        control_ops: Option<Vec<String>>,
11208        launch_nonce: Option<String>,
11209    ) -> Frame {
11210        let body = serde_json::to_vec(&ModuleHelloBody {
11211            manifest: manifest(module_id, protocol_ver),
11212            protocol_ver,
11213            control_ops,
11214            launch_nonce,
11215        })
11216        .unwrap();
11217        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
11218    }
11219
11220    fn hello_frame_with_rich_ui(
11221        module_id: &str,
11222        protocol_ver: u8,
11223        corr: u64,
11224        launch_nonce: Option<&str>,
11225        rich_ui: Value,
11226    ) -> Frame {
11227        let mut body = serde_json::to_value(ModuleHelloBody {
11228            manifest: manifest(module_id, protocol_ver),
11229            protocol_ver,
11230            control_ops: None,
11231            launch_nonce: launch_nonce.map(str::to_string),
11232        })
11233        .unwrap();
11234        body["manifest"]["rich_ui"] = rich_ui;
11235        Frame::build(
11236            FrameType::Hello,
11237            control_flags(),
11238            0,
11239            0,
11240            corr,
11241            serde_json::to_vec(&body).unwrap(),
11242        )
11243        .unwrap()
11244    }
11245
11246    fn non_routable_hello_frame_with_control_ops(
11247        module_id: &str,
11248        corr: u64,
11249        control_ops: Option<Vec<String>>,
11250    ) -> Frame {
11251        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
11252        manifest.provides.clear();
11253        let body = serde_json::to_vec(&ModuleHelloBody {
11254            manifest,
11255            protocol_ver: PROTOCOL_VERSION,
11256            control_ops,
11257            launch_nonce: None,
11258        })
11259        .unwrap();
11260        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
11261    }
11262
11263    fn capability_grammar_hello_frame(
11264        capabilities: Value,
11265        runtime_computed: Option<Value>,
11266        corr: u64,
11267    ) -> Frame {
11268        let mut body = serde_json::to_value(ModuleHelloBody {
11269            manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
11270            protocol_ver: PROTOCOL_VERSION,
11271            control_ops: None,
11272            launch_nonce: None,
11273        })
11274        .expect("HELLO body serializes");
11275        body["manifest"]["capabilities"] = capabilities;
11276        if let Some(runtime_computed) = runtime_computed {
11277            body["runtime_computed"] = runtime_computed;
11278        }
11279        Frame::build(
11280            FrameType::Hello,
11281            control_flags(),
11282            0,
11283            0,
11284            corr,
11285            serde_json::to_vec(&body).expect("HELLO body reserializes"),
11286        )
11287        .expect("HELLO frame builds")
11288    }
11289
11290    fn channel_request(channel: u16, corr: u64) -> Frame {
11291        Frame::build(
11292            FrameType::Request,
11293            Flags::new(true, Priority::Interactive, false),
11294            channel,
11295            0,
11296            corr,
11297            b"opaque".to_vec(),
11298        )
11299        .unwrap()
11300    }
11301
11302    fn route_ctx(
11303        connection_id: ConnectionId,
11304    ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
11305        let (tx, rx) = mpsc::channel(8);
11306        (
11307            RouteCtx {
11308                connection_id,
11309                egress: FrameSink::new(tx),
11310            },
11311            rx,
11312        )
11313    }
11314
11315    fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
11316        serde_json::from_slice(&frame.body).unwrap()
11317    }
11318
11319    /// Register a module over a connection that has a sink and return the
11320    /// HELLO_ACK the module reads. A successful HELLO queues its ack on the
11321    /// module's own sink rather than returning it as a reply, so the ack is
11322    /// taken off `rx` here and whatever the test reads next is what followed it.
11323    async fn hello_via_sink(
11324        handler: &ControlHandler,
11325        ctx: &RouteCtx,
11326        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
11327        hello: Frame,
11328    ) -> Frame {
11329        let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
11330        assert!(
11331            replies.is_empty(),
11332            "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
11333        );
11334        let ack = rx
11335            .try_recv()
11336            .expect("HELLO_ACK is queued on the module sink")
11337            .frame;
11338        assert_eq!(ack.header.ty, FrameType::HelloAck);
11339        ack
11340    }
11341
11342    fn parse_error(frame: &Frame) -> Value {
11343        serde_json::from_slice(&frame.body).unwrap()
11344    }
11345
11346    fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
11347        serde_json::from_slice(&frame.body).unwrap()
11348    }
11349
11350    fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
11351        let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
11352            route_channel,
11353            route_epoch: 0,
11354            kind,
11355        })
11356        .unwrap();
11357        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11358    }
11359
11360    fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
11361        let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
11362            module_id: module_id.to_string(),
11363        })
11364        .unwrap();
11365        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11366    }
11367
11368    fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
11369        route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
11370    }
11371
11372    // Exercise retired opener fields without putting them back in BindIdentity.
11373    fn legacy_route_open_frame(corr: u64, module_id: &str, root: ScratchDir) -> Frame {
11374        let path = ProjectRootId::from_path_allowing_missing(root.path()).unwrap();
11375        let mut frame = route_open_frame(corr, module_id, root);
11376        let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
11377        body["identity"]["project_root"] = serde_json::json!(path.as_path());
11378        frame.body = serde_json::to_vec(&body).unwrap();
11379        frame
11380    }
11381
11382    fn route_open_frame_with_consumer_capabilities(
11383        corr: u64,
11384        module_id: &str,
11385        _project_root: ScratchDir,
11386        consumer_capabilities: Option<Vec<String>>,
11387    ) -> Frame {
11388        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
11389            busy_refusals: None,
11390            target: RouteTarget::ToolProvider {
11391                module_id: module_id.to_string(),
11392            },
11393            identity: BindIdentity::new("unit".to_string(), "session".to_string()),
11394            consumer_identity: None,
11395            consumer_capabilities,
11396            role_versions: None,
11397            admission_facts: None,
11398            scope: None,
11399        })
11400        .unwrap();
11401        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11402    }
11403
11404    fn route_open_frame_with_role_versions(
11405        corr: u64,
11406        module_id: &str,
11407        _project_root: ScratchDir,
11408        role_versions: Option<BTreeMap<String, String>>,
11409    ) -> Frame {
11410        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
11411            busy_refusals: None,
11412            target: RouteTarget::ToolProvider {
11413                module_id: module_id.to_string(),
11414            },
11415            identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
11416            consumer_identity: None,
11417            consumer_capabilities: None,
11418            role_versions,
11419            admission_facts: None,
11420            scope: None,
11421        })
11422        .unwrap();
11423        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11424    }
11425
11426    fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
11427        entries
11428            .iter()
11429            .map(|(role, version)| (role.to_string(), version.to_string()))
11430            .collect()
11431    }
11432
11433    fn route_open_frame_with_admission_facts(
11434        corr: u64,
11435        module_id: &str,
11436        _project_root: ScratchDir,
11437        consumer_identity: Option<subc_control::ConsumerIdentity>,
11438        facts: Option<Value>,
11439    ) -> Frame {
11440        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
11441            busy_refusals: None,
11442            target: RouteTarget::ToolProvider {
11443                module_id: module_id.to_string(),
11444            },
11445            identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
11446            consumer_identity,
11447            consumer_capabilities: None,
11448            role_versions: None,
11449            admission_facts: facts,
11450            scope: None,
11451        })
11452        .unwrap();
11453        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11454    }
11455
11456    #[derive(Clone, Default)]
11457    struct EventCapture {
11458        events: Arc<Mutex<Vec<CapturedEvent>>>,
11459    }
11460
11461    #[derive(Clone, Debug)]
11462    struct CapturedEvent {
11463        target: String,
11464        level: tracing::Level,
11465        fields: BTreeMap<String, String>,
11466    }
11467
11468    impl EventCapture {
11469        fn events(&self) -> Vec<CapturedEvent> {
11470            self.events.lock().unwrap().clone()
11471        }
11472    }
11473
11474    impl<S> Layer<S> for EventCapture
11475    where
11476        S: Subscriber,
11477    {
11478        fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
11479            let mut visitor = EventFieldVisitor::default();
11480            event.record(&mut visitor);
11481            self.events.lock().unwrap().push(CapturedEvent {
11482                target: event.metadata().target().to_string(),
11483                level: *event.metadata().level(),
11484                fields: visitor.fields,
11485            });
11486        }
11487    }
11488
11489    #[derive(Default)]
11490    struct EventFieldVisitor {
11491        fields: BTreeMap<String, String>,
11492    }
11493
11494    impl Visit for EventFieldVisitor {
11495        fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
11496            self.fields
11497                .insert(field.name().to_string(), format!("{value:?}"));
11498        }
11499    }
11500
11501    fn health_response(corr: u64, status: HealthStatus) -> Frame {
11502        let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
11503            status,
11504            detail: Some("warming".to_string()),
11505            metrics: Some(json!({"queue_depth": 3})),
11506        })
11507        .unwrap();
11508        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
11509    }
11510
11511    fn route_bind_ack(corr: u64) -> Frame {
11512        let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
11513        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
11514    }
11515
11516    fn unique_project_root(label: &str) -> ScratchDir {
11517        ScratchDir::new(label)
11518    }
11519
11520    fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
11521        match parse_route_poll(frame) {
11522            ClientControlResponse::RoutePoll {
11523                status: None,
11524                live: Some(live),
11525                ..
11526            } => assert_eq!(live, expected_live),
11527            other => panic!("unexpected route.poll response: {other:?}"),
11528        }
11529    }
11530
11531    fn bind_liveness_route(
11532        registry: &Registry,
11533        forwarding: &ForwardingTable,
11534        module_id: &str,
11535    ) -> (RouteCtx, u16, u32) {
11536        let module_connection = ConnectionId::new(101);
11537        let client_connection = ConnectionId::new(202);
11538        let registration = registry
11539            .register_with_control_ops(
11540                manifest(module_id, PROTOCOL_VERSION),
11541                PROTOCOL_VERSION,
11542                module_connection,
11543                module_baseline_control_ops(),
11544            )
11545            .unwrap();
11546        let (module_tx, _module_rx) = mpsc::channel(8);
11547        let endpoint = forwarding
11548            .register_module_connection(
11549                module_connection,
11550                module_id.to_string(),
11551                PROTOCOL_VERSION,
11552                manifest_concurrency(&registration.manifest),
11553                FrameSink::new(module_tx),
11554            )
11555            .unwrap();
11556        let (client_ctx, _client_rx) = route_ctx(client_connection);
11557        let pending = forwarding
11558            .begin_route_bind_relay_for_test(
11559                client_connection,
11560                client_ctx.egress.clone(),
11561                1,
11562                module_id,
11563            )
11564            .unwrap();
11565        assert_eq!(pending.endpoint, endpoint);
11566        let route_channel = pending.client_channel;
11567        let route_epoch = pending.client_epoch;
11568        forwarding
11569            .complete_pending_relay(
11570                module_connection,
11571                pending.corr,
11572                RouteBindRelayOutcome::Accepted,
11573            )
11574            .unwrap();
11575        (client_ctx, route_channel, route_epoch)
11576    }
11577
11578    struct FakeProcessLiveness {
11579        live: Option<bool>,
11580    }
11581
11582    impl ModuleProcessLiveness for FakeProcessLiveness {
11583        fn process_live(&self, _module_id: &str) -> Option<bool> {
11584            self.live
11585        }
11586    }
11587
11588    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11589    async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
11590    {
11591        let registry = Arc::new(Registry::default());
11592        let supervisor_handle = SupervisorHandle::new();
11593        let supervisor = Supervisor::new_for_test(
11594            Arc::clone(&registry),
11595            RestartPolicy::new(1, Duration::from_millis(10)),
11596        )
11597        .with_handle(supervisor_handle.clone());
11598        let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
11599        let module = supervisor
11600            .spawn(ModuleSpec {
11601                script: None,
11602                carry_containment: false,
11603                module_id: "stderr-tail-wire".to_string(),
11604                program: fake_aft_stub_path(),
11605                args: Vec::new(),
11606                env: vec![
11607                    ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
11608                    ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
11609                ],
11610                reserved: false,
11611                reserved_prefixes: Vec::new(),
11612                protocol: ModuleProtocol::Subc,
11613                overlap: Default::default(),
11614            })
11615            .unwrap();
11616
11617        let deadline = Instant::now() + Duration::from_secs(5);
11618        loop {
11619            let tail = module.stderr_tail(None, None);
11620            if tail
11621                .entries
11622                .iter()
11623                .any(|entry| matches!(entry, TailEntry::ProcessStart))
11624                && tail.entries.iter().any(|entry| {
11625                    matches!(
11626                        entry,
11627                        TailEntry::Line {
11628                            truncated: true,
11629                            ..
11630                        }
11631                    )
11632                })
11633            {
11634                break;
11635            }
11636            assert!(
11637                Instant::now() < deadline,
11638                "module did not produce a truncated line and restart boundary: {tail:?}"
11639            );
11640            sleep(Duration::from_millis(10)).await;
11641        }
11642
11643        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
11644        let request = ClientControlRequest::SupervisorStderrTail {
11645            module_id: "stderr-tail-wire".to_string(),
11646            max_lines: None,
11647            max_bytes: None,
11648        };
11649        let frame = Frame::build(
11650            FrameType::Request,
11651            control_flags(),
11652            0,
11653            0,
11654            1,
11655            serde_json::to_vec(&request).unwrap(),
11656        )
11657        .unwrap();
11658        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
11659        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
11660        let ClientControlResponse::SupervisorStderrTail { tail, .. } =
11661            serde_json::from_slice(&responses[0].body).unwrap()
11662        else {
11663            panic!("expected supervisor.stderr_tail response");
11664        };
11665
11666        assert!(
11667            tail.entries
11668                .iter()
11669                .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
11670            "the control response lost the restart boundary"
11671        );
11672        let Some(StderrTailEntry::Line {
11673            text,
11674            truncated,
11675            at_ms,
11676        }) = tail.entries.iter().find(|entry| {
11677            matches!(
11678                entry,
11679                StderrTailEntry::Line {
11680                    truncated: true,
11681                    ..
11682                }
11683            )
11684        })
11685        else {
11686            panic!("the control response lost the truncated line");
11687        };
11688        assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
11689        assert!(*truncated);
11690        assert!(
11691            at_ms.is_some(),
11692            "the control response lost the line's capture time"
11693        );
11694    }
11695
11696    /// `supervisor.terminals` reads journal files. On a single-worker runtime a
11697    /// read done on the worker thread would stall every other task until it
11698    /// finished; the read must run off the worker so this test's own task keeps
11699    /// running while the read is paused.
11700    #[tokio::test(flavor = "current_thread")]
11701    async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
11702        let dir = ScratchDir::new("terminals-off-worker");
11703        let journal_path = dir.join("terminals.jsonl");
11704        let registry = Arc::new(Registry::default());
11705        let supervisor_handle = SupervisorHandle::new();
11706        let supervisor =
11707            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
11708                .with_handle(supervisor_handle.clone())
11709                .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
11710        let module = supervisor
11711            .spawn(ModuleSpec {
11712                script: None,
11713                carry_containment: false,
11714                module_id: "terminal-off-worker".to_string(),
11715                program: fake_aft_stub_path(),
11716                args: Vec::new(),
11717                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11718                reserved: false,
11719                reserved_prefixes: Vec::new(),
11720                protocol: ModuleProtocol::Subc,
11721                overlap: Default::default(),
11722            })
11723            .unwrap();
11724        let deadline = Instant::now() + Duration::from_secs(5);
11725        while module.terminal_history().entries.len() != 2 {
11726            assert!(Instant::now() < deadline, "module did not record two exits");
11727            sleep(Duration::from_millis(10)).await;
11728        }
11729
11730        let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
11731        let handler =
11732            Arc::new(ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle));
11733        let frame = Frame::build(
11734            FrameType::Request,
11735            control_flags(),
11736            0,
11737            0,
11738            1,
11739            serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
11740                module_id: "terminal-off-worker".to_string(),
11741            })
11742            .unwrap(),
11743        )
11744        .unwrap();
11745        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
11746        let spawned_at = std::time::Instant::now();
11747        let read = tokio::spawn({
11748            let handler = Arc::clone(&handler);
11749            async move { handler.handle_control_frame(&ctx, frame).await }
11750        });
11751        // Waiting for the pause from a blocking thread keeps this task pending,
11752        // so the runtime's single worker is free to run the read task.
11753        tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
11754            .await
11755            .unwrap()
11756            .expect("the history read reached its pause");
11757        let elapsed = spawned_at.elapsed();
11758        assert!(
11759            elapsed < Duration::from_secs(2) && !read.is_finished(),
11760            "this task could not run while the history read was paused \
11761             (resumed after {elapsed:?}, read finished: {})",
11762            read.is_finished()
11763        );
11764
11765        drop(release);
11766        let responses = read.await.unwrap().unwrap();
11767        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
11768        let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
11769            panic!("expected supervisor.terminals response");
11770        };
11771        assert_eq!(terminals.entries.len(), 2);
11772        assert_eq!(terminals.journal_skipped_lines, 0);
11773        assert_eq!(terminals.journal_read_errors, 0);
11774    }
11775
11776    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11777    async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
11778        let registry = Arc::new(Registry::default());
11779        let supervisor_handle = SupervisorHandle::new();
11780        let supervisor =
11781            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
11782                .with_handle(supervisor_handle.clone());
11783        let module = supervisor
11784            .spawn(ModuleSpec {
11785                script: None,
11786                carry_containment: false,
11787                module_id: "terminal-golden".to_string(),
11788                program: fake_aft_stub_path(),
11789                args: Vec::new(),
11790                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11791                reserved: false,
11792                reserved_prefixes: Vec::new(),
11793                protocol: ModuleProtocol::Subc,
11794                overlap: Default::default(),
11795            })
11796            .unwrap();
11797
11798        let deadline = Instant::now() + Duration::from_secs(5);
11799        while module.terminal_history().entries.len() != 2 {
11800            assert!(
11801                Instant::now() < deadline,
11802                "module did not retain two terminal exits: {:?}",
11803                module.terminal_history()
11804            );
11805            sleep(Duration::from_millis(10)).await;
11806        }
11807
11808        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
11809        let request = ClientControlRequest::SupervisorTerminals {
11810            module_id: "terminal-golden".to_string(),
11811        };
11812        let frame = Frame::build(
11813            FrameType::Request,
11814            control_flags(),
11815            0,
11816            0,
11817            1,
11818            serde_json::to_vec(&request).unwrap(),
11819        )
11820        .unwrap();
11821        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
11822        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
11823        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
11824        let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
11825            panic!("expected supervisor.terminals response");
11826        };
11827        assert_eq!(terminals.entries.len(), 2);
11828        assert_eq!(terminals.dropped, 0);
11829
11830        let mut rendered = serde_json::to_value(response).unwrap();
11831        // Wall-clock fields are the observation contract, but not stable fixture
11832        // bytes; normalize only them after the real handler has shaped the response.
11833        rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
11834        for (index, entry) in rendered["entries"]
11835            .as_array_mut()
11836            .expect("terminal response entries array")
11837            .iter_mut()
11838            .enumerate()
11839        {
11840            entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
11841        }
11842
11843        let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11844            .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
11845        let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
11846        if std::env::var_os("UPDATE_GOLDEN").is_some() {
11847            std::fs::write(&golden_path, &serialized).unwrap();
11848        }
11849        let expected: Value =
11850            serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
11851        assert_eq!(rendered, expected);
11852    }
11853
11854    #[test]
11855    fn hello_registers_manifest_and_returns_ack() {
11856        let registry = Arc::new(Registry::default());
11857        let handler = ControlHandler::new(Arc::clone(&registry));
11858        let conn = ConnectionId::new(1);
11859
11860        let responses = handler
11861            .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
11862            .unwrap();
11863
11864        assert_eq!(responses.len(), 1);
11865        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
11866        assert_eq!(responses[0].header.channel, 0);
11867        assert_eq!(responses[0].header.corr, 7);
11868        let ack = parse_ack(&responses[0]);
11869        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
11870        assert!(ack
11871            .subc_capabilities
11872            .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
11873        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
11874        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
11875        assert!(ack
11876            .subc_ops
11877            .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
11878        assert!(ack
11879            .subc_ops
11880            .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
11881        assert!(ack
11882            .subc_ops
11883            .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS.to_string()));
11884
11885        let registration = registry.get_module("aft").unwrap().unwrap();
11886        assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
11887        assert_eq!(registration.state, ChannelState::Active);
11888        assert_eq!(registration.connection_id, conn);
11889        assert_eq!(registration.control_ops, module_baseline_control_ops());
11890        assert!(registration.manifest.display_name.is_none());
11891    }
11892
11893    #[test]
11894    fn hello_display_name_refuses_invalid_manifest_by_field() {
11895        let registry = Arc::new(Registry::default());
11896        let handler = ControlHandler::new(Arc::clone(&registry));
11897        for (index, invalid) in [
11898            json!(""),
11899            json!("a".repeat(41)),
11900            json!(" Routing"),
11901            json!("a\nb"),
11902            json!("a\u{202e}b"),
11903            json!(7),
11904        ]
11905        .into_iter()
11906        .enumerate()
11907        {
11908            let module_id = format!("bad-display-name-{index}");
11909            let mut body = serde_json::to_value(ModuleHelloBody {
11910                manifest: manifest(&module_id, PROTOCOL_VERSION),
11911                protocol_ver: PROTOCOL_VERSION,
11912                control_ops: None,
11913                launch_nonce: None,
11914            })
11915            .unwrap();
11916            body["manifest"]["display_name"] = invalid;
11917            let hello = Frame::build(
11918                FrameType::Hello,
11919                control_flags(),
11920                0,
11921                0,
11922                7,
11923                serde_json::to_vec(&body).unwrap(),
11924            )
11925            .unwrap();
11926            let response = handler
11927                .handle_control(ConnectionId::new(100 + index as u64), hello)
11928                .unwrap();
11929            assert_eq!(response[0].header.ty, FrameType::Error);
11930            let error = parse_error(&response[0]);
11931            assert_eq!(error["code"], "invalid_manifest");
11932            assert!(error["message"].as_str().unwrap().contains("display_name"));
11933            assert!(registry.get_module(&module_id).unwrap().is_none());
11934        }
11935    }
11936
11937    #[test]
11938    fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
11939        let invalid_identifiers = [
11940            ("case_change", "credentials-Provider/v1"),
11941            ("leading_zero", "credentials-provider/v01"),
11942            ("trailing_hyphen", "credentials-provider-/v1"),
11943            ("consecutive_hyphens", "credentials--provider/v1"),
11944            ("uppercase", "Credentials-provider/v1"),
11945            ("missing_v", "credentials-provider/1"),
11946            ("whitespace", "credentials provider/v1"),
11947            ("zero_version", "credentials-provider/v0"),
11948            ("out_of_range_version", "credentials-provider/v4294967296"),
11949            (
11950                "overlength_name",
11951                "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
11952            ),
11953        ];
11954        let mut cases = invalid_identifiers
11955            .into_iter()
11956            .map(|(name, identifier)| {
11957                (
11958                    format!("identifier_{name}"),
11959                    "capabilities.provides[0]".to_string(),
11960                    identifier.to_string(),
11961                    json!({ "provides": [identifier] }),
11962                    None,
11963                )
11964            })
11965            .collect::<Vec<_>>();
11966        cases.extend([
11967            (
11968                "unknown_need".to_string(),
11969                "capabilities.requires[0].need".to_string(),
11970                "deferred".to_string(),
11971                json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
11972                None,
11973            ),
11974            (
11975                "duplicate_provides".to_string(),
11976                "capabilities.provides[1]".to_string(),
11977                "credentials-provider/v1".to_string(),
11978                json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
11979                None,
11980            ),
11981            (
11982                "duplicate_must_never_reach".to_string(),
11983                "capabilities.must_never_reach[1]".to_string(),
11984                "credentials-provider/v1".to_string(),
11985                json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
11986                None,
11987            ),
11988            (
11989                "duplicate_requires_same_need".to_string(),
11990                "capabilities.requires[1]".to_string(),
11991                "credentials-provider/v1".to_string(),
11992                json!({ "requires": [
11993                    { "capability": "credentials-provider/v1", "need": "required" },
11994                    { "capability": "credentials-provider/v1", "need": "required" }
11995                ] }),
11996                None,
11997            ),
11998            (
11999                "duplicate_requires_conflicting_need".to_string(),
12000                "capabilities.requires[1]".to_string(),
12001                "credentials-provider/v1".to_string(),
12002                json!({ "requires": [
12003                    { "capability": "credentials-provider/v1", "need": "required" },
12004                    { "capability": "credentials-provider/v1", "need": "optional" }
12005                ] }),
12006                None,
12007            ),
12008            (
12009                "capabilities_root_pointer".to_string(),
12010                "runtime_computed[0]".to_string(),
12011                "/capabilities".to_string(),
12012                json!({}),
12013                Some(json!(["/capabilities"])),
12014            ),
12015            (
12016                "capabilities_descendant_pointer".to_string(),
12017                "runtime_computed[0]".to_string(),
12018                "/capabilities/provides".to_string(),
12019                json!({}),
12020                Some(json!(["/capabilities/provides"])),
12021            ),
12022            (
12023                "malformed_pointer_without_leading_slash".to_string(),
12024                "runtime_computed[0]".to_string(),
12025                "capabilities".to_string(),
12026                json!({}),
12027                Some(json!(["capabilities"])),
12028            ),
12029            (
12030                "malformed_pointer_escape".to_string(),
12031                "runtime_computed[0]".to_string(),
12032                "/roles/~2/tools".to_string(),
12033                json!({}),
12034                Some(json!(["/roles/~2/tools"])),
12035            ),
12036            (
12037                "unknown_capabilities_field".to_string(),
12038                "capabilities.future".to_string(),
12039                "<array>".to_string(),
12040                json!({ "future": [] }),
12041                None,
12042            ),
12043        ]);
12044
12045        for (index, (name, field, value, capabilities, runtime_computed)) in
12046            cases.into_iter().enumerate()
12047        {
12048            let registry = Arc::new(Registry::default());
12049            let handler = ControlHandler::new(Arc::clone(&registry));
12050            let response = handler
12051                .handle_control(
12052                    ConnectionId::new((index + 1) as u64),
12053                    capability_grammar_hello_frame(
12054                        capabilities,
12055                        runtime_computed,
12056                        index as u64 + 1,
12057                    ),
12058                )
12059                .expect("invalid HELLO returns a refusal");
12060
12061            assert_eq!(response.len(), 1, "{name} must emit one refusal");
12062            let error = parse_error(&response[0]);
12063            assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
12064            let message = error["message"]
12065                .as_str()
12066                .expect("error message is a string");
12067            assert!(
12068                message.contains(&field),
12069                "{name}: field missing from {message}"
12070            );
12071            assert!(
12072                message.contains(&value),
12073                "{name}: value missing from {message}"
12074            );
12075            assert_eq!(
12076                registry
12077                    .active_registration_count()
12078                    .expect("registry reads"),
12079                0,
12080                "{name}: refused HELLO must not create a catalog entry"
12081            );
12082        }
12083    }
12084
12085    #[test]
12086    fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
12087        let registry = Arc::new(Registry::default());
12088        let handler = ControlHandler::new(Arc::clone(&registry));
12089        let capabilities = json!({
12090            "provides": ["credentials-provider/v1"],
12091            "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
12092            "must_never_reach": ["federation-transport/v1"]
12093        });
12094        let response = handler
12095            .handle_control(
12096                ConnectionId::new(99),
12097                capability_grammar_hello_frame(
12098                    capabilities.clone(),
12099                    Some(json!(["/roles/0/tools"])),
12100                    99,
12101                ),
12102            )
12103            .expect("valid HELLO registers");
12104        assert_eq!(response[0].header.ty, FrameType::HelloAck);
12105
12106        let request = Frame::build(
12107            FrameType::Request,
12108            control_flags(),
12109            0,
12110            0,
12111            100,
12112            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
12113                .expect("catalog request serializes"),
12114        )
12115        .expect("catalog request frame builds");
12116        let response = handler
12117            .handle_catalog_list(request, None)
12118            .expect("catalog list succeeds");
12119        let ClientControlResponse::CatalogList { modules, .. } =
12120            serde_json::from_slice(&response[0].body).expect("catalog response decodes")
12121        else {
12122            panic!("catalog request must return catalog.list");
12123        };
12124        assert_eq!(modules.len(), 1);
12125        assert_eq!(
12126            serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
12127            capabilities
12128        );
12129    }
12130
12131    #[test]
12132    fn catalog_list_mirrors_management_operation_description() {
12133        let registry = Arc::new(Registry::default());
12134        let handler = ControlHandler::new(Arc::clone(&registry));
12135        let description = "List managed records and return their identifiers and metadata.";
12136        let mut manifest = manifest("described-management", PROTOCOL_VERSION);
12137        manifest.provides = vec![ProviderRole::ManagementSurface {
12138            operations: vec![ManagementOperation {
12139                name: "records.list".to_string(),
12140                kind: ManagementOperationKind::Query,
12141                description: Some(description.to_string()),
12142            }],
12143            config_schema: json!({"type": "object"}),
12144            observability: vec![ObservabilitySurface {
12145                name: "records.stats".to_string(),
12146                kind: ObservabilityKind::Snapshot,
12147            }],
12148            identity_scope: vec![IdentityScope::Project],
12149            concurrency: Concurrency::ModuleManaged,
12150        }];
12151        registry
12152            .register_with_control_ops(
12153                manifest,
12154                PROTOCOL_VERSION,
12155                ConnectionId::new(99),
12156                Vec::new(),
12157            )
12158            .expect("described management manifest registers");
12159
12160        let request = Frame::build(
12161            FrameType::Request,
12162            control_flags(),
12163            0,
12164            0,
12165            100,
12166            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
12167                .expect("catalog request serializes"),
12168        )
12169        .expect("catalog request frame builds");
12170        let response = handler
12171            .handle_catalog_list(request, None)
12172            .expect("catalog list succeeds");
12173        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
12174        assert_eq!(
12175            body["modules"][0]["roles"][0]["operations"][0]["description"], description,
12176            "catalog.list must preserve the declared operation description verbatim"
12177        );
12178    }
12179
12180    #[test]
12181    fn catalog_list_mirrors_registered_operation_delivery() {
12182        let registry = Arc::new(Registry::default());
12183        let handler = ControlHandler::new(Arc::clone(&registry));
12184        let delivery =
12185            json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
12186        let mut body = serde_json::to_value(ModuleHelloBody {
12187            manifest: manifest("watch-management", PROTOCOL_VERSION),
12188            protocol_ver: PROTOCOL_VERSION,
12189            control_ops: None,
12190            launch_nonce: None,
12191        })
12192        .unwrap();
12193        body["manifest"]["provides"] = json!([{
12194            "role": "management_surface",
12195            "operations": [
12196                {"name": "status", "kind": "query"},
12197                {"name": "future", "kind": "query"},
12198                {"name": "reset", "kind": "mutate"}
12199            ],
12200            "config_schema": {}, "observability": [], "identity_scope": []
12201        }]);
12202        body["manifest"]["operation_delivery"] = delivery.clone();
12203        let hello = Frame::build(
12204            FrameType::Hello,
12205            control_flags(),
12206            0,
12207            0,
12208            99,
12209            serde_json::to_vec(&body).unwrap(),
12210        )
12211        .unwrap();
12212        let response = handler
12213            .handle_control(ConnectionId::new(99), hello)
12214            .unwrap();
12215        assert_eq!(response[0].header.ty, FrameType::HelloAck);
12216
12217        let (_, registrations) = registry.list_modules().unwrap();
12218        let registered = &registrations[0].manifest;
12219        assert!(registered.is_watch_operation("status"));
12220        assert!(!registered.is_watch_operation("future"));
12221        assert!(!registered.is_watch_operation("reset"));
12222        assert!(!registered.is_watch_operation("missing"));
12223
12224        let request = Frame::build(
12225            FrameType::Request,
12226            control_flags(),
12227            0,
12228            0,
12229            100,
12230            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
12231        )
12232        .unwrap();
12233        let response = handler.handle_catalog_list(request, None).unwrap();
12234        let body: Value = serde_json::from_slice(&response[0].body).unwrap();
12235        assert_eq!(body["modules"][0]["operation_delivery"], delivery);
12236        let ClientControlResponse::CatalogList { modules, .. } =
12237            serde_json::from_slice(&response[0].body).unwrap()
12238        else {
12239            panic!("expected catalog.list");
12240        };
12241        assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
12242    }
12243
12244    #[test]
12245    fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
12246        let registry = Arc::new(Registry::default());
12247        let supervisor = SupervisorHandle::new();
12248        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
12249        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
12250        let descriptor = json!({
12251            "schema": "ckios.richui.manifest/1",
12252            "manifest_digest": format!("sha256:{}", "a".repeat(64)),
12253            "bytes": 4096,
12254            "chunks": 4
12255        });
12256        let connection = ConnectionId::new(201);
12257        let hello = handler
12258            .handle_control(
12259                connection,
12260                hello_frame_with_rich_ui(
12261                    "vault",
12262                    PROTOCOL_VERSION,
12263                    201,
12264                    Some("the-real-nonce"),
12265                    descriptor.clone(),
12266                ),
12267            )
12268            .unwrap();
12269        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
12270
12271        let request =
12272            Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
12273        let provides = registry
12274            .get_module_by_connection(connection)
12275            .unwrap()
12276            .unwrap()
12277            .manifest
12278            .provides;
12279        handler
12280            .handle_catalog_update(connection, request, provides, None, Some(false))
12281            .unwrap();
12282
12283        let request = Frame::build(
12284            FrameType::Request,
12285            control_flags(),
12286            0,
12287            0,
12288            203,
12289            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
12290        )
12291        .unwrap();
12292        let response = handler.handle_catalog_list(request, None).unwrap();
12293        let ClientControlResponse::CatalogList { modules, .. } =
12294            serde_json::from_slice(&response[0].body).unwrap()
12295        else {
12296            panic!("expected catalog.list");
12297        };
12298        assert_eq!(
12299            serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
12300            descriptor
12301        );
12302        assert!(modules[0].nonce_verified);
12303    }
12304
12305    fn hello_with_role_declarations(declarations: Value) -> Frame {
12306        let mut body = serde_json::to_value(ModuleHelloBody {
12307            manifest: manifest("role-test", PROTOCOL_VERSION),
12308            protocol_ver: PROTOCOL_VERSION,
12309            control_ops: None,
12310            launch_nonce: None,
12311        })
12312        .unwrap();
12313        body["manifest"]["role_declarations"] = declarations;
12314        Frame::build(
12315            FrameType::Hello,
12316            control_flags(),
12317            0,
12318            0,
12319            301,
12320            serde_json::to_vec(&body).unwrap(),
12321        )
12322        .unwrap()
12323    }
12324
12325    fn update_role_declarations_frame(declarations: Value) -> Frame {
12326        Frame::build(
12327            FrameType::Request,
12328            control_flags(),
12329            0,
12330            0,
12331            303,
12332            serde_json::to_vec(&json!({
12333                "op": "catalog.update_role_declarations",
12334                "role_declarations": declarations,
12335            }))
12336            .unwrap(),
12337        )
12338        .unwrap()
12339    }
12340
12341    #[tokio::test]
12342    async fn role_declarations_update_replaces_then_clears_catalog() {
12343        let registry = Arc::new(Registry::default());
12344        let handler = ControlHandler::new(Arc::clone(&registry));
12345        let (ctx, mut rx) = route_ctx(ConnectionId::new(301));
12346        let initial = json!({"media-provider/v1": {"engines": ["old"]}, "old/v1": {}});
12347        hello_via_sink(
12348            &handler,
12349            &ctx,
12350            &mut rx,
12351            hello_with_role_declarations(initial.clone()),
12352        )
12353        .await;
12354        assert_eq!(
12355            role_declarations_catalog(&handler).0["role_declarations"],
12356            initial
12357        );
12358        let before = registry.get_module("role-test").unwrap().unwrap();
12359        for declarations in [
12360            json!({"media-provider/v1": {"engines": ["new"]}}),
12361            json!({}),
12362        ] {
12363            let response = handler
12364                .handle_control_frame(&ctx, update_role_declarations_frame(declarations.clone()))
12365                .await
12366                .unwrap();
12367            assert_eq!(response[0].header.ty, FrameType::Response);
12368            assert_eq!(
12369                serde_json::from_slice::<Value>(&response[0].body).unwrap(),
12370                json!({"op": "catalog.update_role_declarations"})
12371            );
12372            let (raw, entry) = role_declarations_catalog(&handler);
12373            assert_eq!(
12374                serde_json::to_value(entry.role_declarations).unwrap(),
12375                declarations
12376            );
12377            if declarations.as_object().unwrap().is_empty() {
12378                assert!(raw.get("role_declarations").is_none());
12379            } else {
12380                assert_eq!(raw["role_declarations"], declarations);
12381            }
12382            let after = registry.get_module("role-test").unwrap().unwrap();
12383            let mut expected = before.manifest.clone();
12384            expected.role_declarations = serde_json::from_value(declarations).unwrap();
12385            assert_eq!(after.manifest, expected);
12386            assert_eq!(after.ready, before.ready);
12387        }
12388    }
12389
12390    #[tokio::test]
12391    async fn role_declarations_update_refuses_invalid_maps_without_changing_catalog() {
12392        let registry = Arc::new(Registry::default());
12393        let handler = ControlHandler::new(registry);
12394        let (ctx, mut rx) = route_ctx(ConnectionId::new(301));
12395        let initial = json!({"media-provider/v1": {"engines": ["old"]}});
12396        hello_via_sink(
12397            &handler,
12398            &ctx,
12399            &mut rx,
12400            hello_with_role_declarations(initial.clone()),
12401        )
12402        .await;
12403        for (invalid, reason) in [
12404            (json!(null), "role_declarations must be an object"),
12405            (json!([]), "role_declarations must be an object"),
12406            (
12407                json!({"bad key": {}}),
12408                "role_declarations key must use capability-name grammar",
12409            ),
12410            (
12411                json!({"media-provider/v1": []}),
12412                "role_declarations values must be objects",
12413            ),
12414            (
12415                json!({"media-provider/v1": {"padding": "x".repeat(16384)}}),
12416                "role_declarations exceeds 16384 compact JSON bytes",
12417            ),
12418        ] {
12419            let response = handler
12420                .handle_control_frame(&ctx, update_role_declarations_frame(invalid))
12421                .await
12422                .unwrap();
12423            let error = parse_error(&response[0]);
12424            assert_eq!(error["code"], "invalid_role_declarations");
12425            assert_eq!(error["message"], reason);
12426            assert_eq!(
12427                role_declarations_catalog(&handler).0["role_declarations"],
12428                initial
12429            );
12430        }
12431    }
12432
12433    #[tokio::test]
12434    async fn role_declarations_update_is_owned_by_the_registered_connection() {
12435        let registry = Arc::new(Registry::default());
12436        let handler = ControlHandler::new(Arc::clone(&registry));
12437        let (owner, mut owner_rx) = route_ctx(ConnectionId::new(301));
12438        let initial = json!({"media-provider/v1": {"engines": ["old"]}});
12439        hello_via_sink(
12440            &handler,
12441            &owner,
12442            &mut owner_rx,
12443            hello_with_role_declarations(initial.clone()),
12444        )
12445        .await;
12446        let (other, mut other_rx) = route_ctx(ConnectionId::new(302));
12447        let replacement = json!({"media-provider/v1": {"engines": ["other"]}});
12448        let response = handler
12449            .handle_control_frame(&other, update_role_declarations_frame(replacement.clone()))
12450            .await
12451            .unwrap();
12452        assert_eq!(parse_error(&response[0])["code"], "not_registered");
12453        hello_via_sink(
12454            &handler,
12455            &other,
12456            &mut other_rx,
12457            hello_frame("other", PROTOCOL_VERSION, 304),
12458        )
12459        .await;
12460        let response = handler
12461            .handle_control_frame(&other, update_role_declarations_frame(replacement.clone()))
12462            .await
12463            .unwrap();
12464        assert_eq!(response[0].header.ty, FrameType::Response);
12465        assert_eq!(
12466            serde_json::to_value(
12467                registry
12468                    .get_module("role-test")
12469                    .unwrap()
12470                    .unwrap()
12471                    .manifest
12472                    .role_declarations
12473            )
12474            .unwrap(),
12475            initial
12476        );
12477        assert_eq!(
12478            serde_json::to_value(
12479                registry
12480                    .get_module("other")
12481                    .unwrap()
12482                    .unwrap()
12483                    .manifest
12484                    .role_declarations
12485            )
12486            .unwrap(),
12487            replacement
12488        );
12489    }
12490
12491    fn role_declarations_catalog(handler: &ControlHandler) -> (Value, CatalogEntry) {
12492        let request = Frame::build(
12493            FrameType::Request,
12494            control_flags(),
12495            0,
12496            0,
12497            302,
12498            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
12499        )
12500        .unwrap();
12501        let response = handler.handle_catalog_list(request, None).unwrap();
12502        let raw: Value = serde_json::from_slice(&response[0].body).unwrap();
12503        let ClientControlResponse::CatalogList { mut modules, .. } =
12504            serde_json::from_slice(&response[0].body).unwrap()
12505        else {
12506            panic!("expected catalog.list");
12507        };
12508        (raw["modules"][0].clone(), modules.remove(0))
12509    }
12510
12511    #[test]
12512    fn catalog_list_mirrors_role_declarations_through_hello_and_catalog_update() {
12513        let registry = Arc::new(Registry::default());
12514        let handler = ControlHandler::new(Arc::clone(&registry));
12515        let declarations = json!({
12516            "routing/v1": {"consumers": [{"name": "chat", "default_demand": 2}]},
12517            "future/v2": {"unknown": [null, true, {"nested": "λ"}]}
12518        });
12519        let connection = ConnectionId::new(301);
12520        let response = handler
12521            .handle_control(
12522                connection,
12523                hello_with_role_declarations(declarations.clone()),
12524            )
12525            .unwrap();
12526        assert_eq!(response[0].header.ty, FrameType::HelloAck);
12527        let registered = registry
12528            .get_module_by_connection(connection)
12529            .unwrap()
12530            .unwrap()
12531            .manifest;
12532        // Compared as JSON values, not bytes: object key order is not part of
12533        // the relay's promise, and workspace builds unify serde_json's
12534        // preserve_order feature, which changes the order a literal keeps.
12535        assert_eq!(
12536            serde_json::to_value(&registered.role_declarations).unwrap(),
12537            declarations
12538        );
12539        let (raw, entry) = role_declarations_catalog(&handler);
12540        assert_eq!(raw["role_declarations"], declarations);
12541        assert_eq!(entry.role_declarations, registered.role_declarations);
12542
12543        let request =
12544            Frame::build(FrameType::Request, control_flags(), 0, 0, 303, Vec::new()).unwrap();
12545        handler
12546            .handle_catalog_update(connection, request, registered.provides, None, Some(false))
12547            .unwrap();
12548        let (raw, entry) = role_declarations_catalog(&handler);
12549        assert_eq!(raw["role_declarations"], declarations);
12550        assert_eq!(entry.role_declarations, registered.role_declarations);
12551    }
12552
12553    fn assert_hello_drops_role_declarations_with_warning(declarations: Value, reason: &str) {
12554        let registry = Arc::new(Registry::default());
12555        let handler = ControlHandler::new(Arc::clone(&registry));
12556        let capture = EventCapture::default();
12557        let _subscriber =
12558            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12559        let response = handler
12560            .handle_control(
12561                ConnectionId::new(301),
12562                hello_with_role_declarations(declarations),
12563            )
12564            .unwrap();
12565        assert_eq!(response[0].header.ty, FrameType::HelloAck);
12566        assert!(registry
12567            .get_module_by_connection(ConnectionId::new(301))
12568            .unwrap()
12569            .unwrap()
12570            .manifest
12571            .role_declarations
12572            .is_empty());
12573        let (raw, entry) = role_declarations_catalog(&handler);
12574        assert!(raw.get("role_declarations").is_none());
12575        assert!(entry.role_declarations.is_empty());
12576        let warnings: Vec<_> = capture
12577            .events()
12578            .into_iter()
12579            .filter(|event| {
12580                event.fields.get("message").map(String::as_str)
12581                    == Some("dropping whole role_declarations map from HELLO")
12582            })
12583            .collect();
12584        assert_eq!(warnings.len(), 1);
12585        assert_eq!(warnings[0].level, tracing::Level::WARN);
12586        assert_eq!(warnings[0].fields["module_id"], "\"role-test\"");
12587        assert_eq!(warnings[0].fields["reason"], reason);
12588    }
12589
12590    #[test]
12591    fn hello_non_object_role_declarations_drop_whole_map_and_warn() {
12592        for value in [
12593            json!(null),
12594            json!(false),
12595            json!(0),
12596            json!("opaque"),
12597            json!([]),
12598        ] {
12599            assert_hello_drops_role_declarations_with_warning(
12600                json!({"routing/v1": {}, "broken/v1": value}),
12601                "role_declarations values must be objects",
12602            );
12603        }
12604        assert_hello_drops_role_declarations_with_warning(
12605            json!([]),
12606            "role_declarations must be an object",
12607        );
12608    }
12609
12610    #[test]
12611    fn hello_bad_role_declaration_key_drops_whole_map_and_warns() {
12612        assert_hello_drops_role_declarations_with_warning(
12613            json!({"routing/v1": {}, "bad key": {}}),
12614            "role_declarations key must use capability-name grammar",
12615        );
12616    }
12617
12618    #[test]
12619    fn hello_oversized_role_declarations_drop_whole_map_and_warn() {
12620        assert_hello_drops_role_declarations_with_warning(
12621            json!({"routing/v1": {}, "overflow/v1": {"padding": "x".repeat(16_384)}}),
12622            "role_declarations exceeds 16384 compact JSON bytes",
12623        );
12624    }
12625
12626    fn hello_with_flow_operations(capabilities: Value, flow_operations: Option<Value>) -> Frame {
12627        let mut body = serde_json::to_value(ModuleHelloBody {
12628            manifest: manifest("flow-ops", PROTOCOL_VERSION),
12629            protocol_ver: PROTOCOL_VERSION,
12630            control_ops: None,
12631            launch_nonce: None,
12632        })
12633        .unwrap();
12634        body["manifest"]["provides"] = json!([{
12635            "role": "management_surface",
12636            "operations": [
12637                {"name": "diagnostics.findings", "kind": "query"},
12638                {"name": "reset", "kind": "mutate"}
12639            ],
12640            "config_schema": {}, "observability": [], "identity_scope": []
12641        }]);
12642        body["manifest"]["capabilities"] = json!({ "provides": capabilities });
12643        if let Some(flow_operations) = flow_operations {
12644            body["manifest"]["flow_operations"] = flow_operations;
12645        }
12646        Frame::build(
12647            FrameType::Hello,
12648            control_flags(),
12649            0,
12650            0,
12651            401,
12652            serde_json::to_vec(&body).unwrap(),
12653        )
12654        .unwrap()
12655    }
12656
12657    #[test]
12658    fn catalog_list_mirrors_flow_operations_unchanged() {
12659        // Order is kept as declared (not sorted) and an empty list stays an
12660        // affirmative "none", distinct from an absent declaration.
12661        for (capability, flow_operations) in [
12662            ("flow-scopes/v1", json!(["reset", "diagnostics.findings"])),
12663            ("call-modes/v1", json!(["diagnostics.findings"])),
12664            ("flow-scopes/v1", json!([])),
12665        ] {
12666            let registry = Arc::new(Registry::default());
12667            let handler = ControlHandler::new(Arc::clone(&registry));
12668            let connection = ConnectionId::new(401);
12669            let response = handler
12670                .handle_control(
12671                    connection,
12672                    hello_with_flow_operations(json!([capability]), Some(flow_operations.clone())),
12673                )
12674                .unwrap();
12675            assert_eq!(response[0].header.ty, FrameType::HelloAck);
12676            let registered = registry
12677                .get_module_by_connection(connection)
12678                .unwrap()
12679                .unwrap()
12680                .manifest;
12681            assert_eq!(
12682                serde_json::to_value(&registered.flow_operations).unwrap(),
12683                flow_operations
12684            );
12685            let (raw, entry) = role_declarations_catalog(&handler);
12686            assert_eq!(raw["flow_operations"], flow_operations);
12687            assert_eq!(entry.flow_operations, registered.flow_operations);
12688        }
12689
12690        let registry = Arc::new(Registry::default());
12691        let handler = ControlHandler::new(Arc::clone(&registry));
12692        let response = handler
12693            .handle_control(
12694                ConnectionId::new(401),
12695                hello_with_flow_operations(json!(["flow-scopes/v1"]), None),
12696            )
12697            .unwrap();
12698        assert_eq!(response[0].header.ty, FrameType::HelloAck);
12699        let (raw, entry) = role_declarations_catalog(&handler);
12700        assert!(raw.get("flow_operations").is_none());
12701        assert_eq!(entry.flow_operations, None);
12702    }
12703
12704    #[test]
12705    fn operation_deferral_hello_accepts_valid_and_invalid_maps_and_catalog_relays_only_valid() {
12706        let oversized: serde_json::Map<String, Value> = (0..257)
12707            .map(|n| (format!("tool:t{n}"), json!(true)))
12708            .collect();
12709        for (capabilities, map, valid) in [
12710            (
12711                json!(["scope-federated/v1"]),
12712                json!({"tool:read":true,"management:reset":true}),
12713                true,
12714            ),
12715            (json!(["scope-federated/v1"]), json!({}), true),
12716            (json!([]), json!({"tool:read":true}), false),
12717            (
12718                json!(["scope-federated/v1"]),
12719                json!({"tool:valid":true,"tool:bad":false}),
12720                false,
12721            ),
12722            (
12723                json!(["scope-federated/v1"]),
12724                json!({"tool:valid":true,"management:missing":true}),
12725                false,
12726            ),
12727            (
12728                json!(["scope-federated/v1"]),
12729                Value::Object(oversized),
12730                false,
12731            ),
12732        ] {
12733            let capture = EventCapture::default();
12734            let _subscriber = tracing::subscriber::set_default(
12735                tracing_subscriber::registry().with(capture.clone()),
12736            );
12737            let registry = Arc::new(Registry::default());
12738            let handler = ControlHandler::new(Arc::clone(&registry));
12739            let hello = hello_with_flow_operations(capabilities, None);
12740            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
12741            body["manifest"]["operation_deferral"] = map.clone();
12742            let hello = Frame::build(
12743                FrameType::Hello,
12744                control_flags(),
12745                0,
12746                0,
12747                401,
12748                serde_json::to_vec(&body).unwrap(),
12749            )
12750            .unwrap();
12751            let response = handler
12752                .handle_control(ConnectionId::new(401), hello)
12753                .unwrap();
12754            assert_eq!(response[0].header.ty, FrameType::HelloAck);
12755            let (raw, entry) = role_declarations_catalog(&handler);
12756            if valid {
12757                assert_eq!(raw["operation_deferral"], map);
12758                assert_eq!(
12759                    serde_json::to_value(entry.operation_deferral.unwrap()).unwrap(),
12760                    map
12761                );
12762            } else {
12763                assert!(entry.operation_deferral.is_none());
12764                assert!(raw.get("operation_deferral").is_none());
12765                assert_eq!(
12766                    capture
12767                        .events()
12768                        .iter()
12769                        .filter(|event| event.fields.get("message").map(String::as_str)
12770                            == Some("dropping operation_deferral from HELLO"))
12771                        .count(),
12772                    1
12773                );
12774            }
12775        }
12776    }
12777
12778    fn assert_hello_drops_flow_operations_with_warning(
12779        capabilities: Value,
12780        flow_operations: Value,
12781        reason: &str,
12782    ) {
12783        let registry = Arc::new(Registry::default());
12784        let handler = ControlHandler::new(Arc::clone(&registry));
12785        let capture = EventCapture::default();
12786        let _subscriber =
12787            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12788        let response = handler
12789            .handle_control(
12790                ConnectionId::new(401),
12791                hello_with_flow_operations(capabilities, Some(flow_operations)),
12792            )
12793            .unwrap();
12794        assert_eq!(response[0].header.ty, FrameType::HelloAck);
12795        let registered = registry
12796            .get_module_by_connection(ConnectionId::new(401))
12797            .unwrap()
12798            .expect("the module still registers")
12799            .manifest;
12800        assert_eq!(registered.module_id, "flow-ops");
12801        assert_eq!(registered.flow_operations, None);
12802        let (raw, entry) = role_declarations_catalog(&handler);
12803        assert_eq!(raw["module_id"], "flow-ops");
12804        assert!(raw.get("flow_operations").is_none());
12805        assert_eq!(entry.flow_operations, None);
12806        let warnings: Vec<_> = capture
12807            .events()
12808            .into_iter()
12809            .filter(|event| {
12810                event.fields.get("message").map(String::as_str)
12811                    == Some("dropping flow_operations from HELLO")
12812            })
12813            .collect();
12814        assert_eq!(warnings.len(), 1);
12815        assert_eq!(warnings[0].level, tracing::Level::WARN);
12816        assert_eq!(warnings[0].fields["module_id"], "\"flow-ops\"");
12817        assert_eq!(warnings[0].fields["reason"], reason);
12818    }
12819
12820    #[test]
12821    fn hello_flow_operations_naming_unknown_operation_is_dropped_with_warning() {
12822        assert_hello_drops_flow_operations_with_warning(
12823            json!(["flow-scopes/v1"]),
12824            json!(["diagnostics.findings", "missing"]),
12825            "flow_operations names 'missing', which is not a declared management operation",
12826        );
12827    }
12828
12829    #[test]
12830    fn hello_flow_operations_without_flow_capability_is_dropped_with_warning() {
12831        assert_hello_drops_flow_operations_with_warning(
12832            json!([]),
12833            json!(["diagnostics.findings"]),
12834            "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
12835        );
12836        // A run-only capability does not admit flows either.
12837        assert_hello_drops_flow_operations_with_warning(
12838            json!(["agent-run-scopes/v1"]),
12839            json!([]),
12840            "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
12841        );
12842    }
12843
12844    #[test]
12845    fn hello_malformed_flow_operations_is_dropped_with_warning() {
12846        assert_hello_drops_flow_operations_with_warning(
12847            json!(["flow-scopes/v1"]),
12848            json!(["reset", "reset"]),
12849            "flow_operations names 'reset' more than once",
12850        );
12851        assert_hello_drops_flow_operations_with_warning(
12852            json!(["flow-scopes/v1"]),
12853            json!(vec!["reset"; 257]),
12854            "flow_operations lists 257 names, more than 256",
12855        );
12856        assert_hello_drops_flow_operations_with_warning(
12857            json!(["flow-scopes/v1"]),
12858            json!("reset"),
12859            "flow_operations must be an array of operation names",
12860        );
12861    }
12862
12863    #[test]
12864    fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
12865        let registry = Arc::new(Registry::default());
12866        let handler = ControlHandler::new(registry.clone());
12867        let descriptor = json!({
12868            "schema": "ckios.richui.manifest/1",
12869            "manifest_digest": format!("sha256:{}", "a".repeat(64)),
12870            "bytes": 128,
12871            "chunks": 2
12872        });
12873        let malformed = [
12874            ("schema", json!("")),
12875            ("schema", json!("a".repeat(65))),
12876            ("schema", json!("Bad schema")),
12877            (
12878                "manifest_digest",
12879                json!(format!("sha256:{}", "A".repeat(64))),
12880            ),
12881            ("manifest_digest", json!("sha256:abc")),
12882            ("bytes", json!(0)),
12883            ("bytes", json!(16_777_217)),
12884            ("bytes", json!("128")),
12885            ("chunks", json!(0)),
12886            ("chunks", json!(1025)),
12887            ("chunks", json!("2")),
12888        ];
12889        for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
12890            let mut invalid = descriptor.clone();
12891            invalid[field] = invalid_value;
12892            let response = handler
12893                .handle_control(
12894                    ConnectionId::new(210 + index as u64),
12895                    hello_frame_with_rich_ui(
12896                        &format!("bad-rich-ui-{index}"),
12897                        PROTOCOL_VERSION,
12898                        210 + index as u64,
12899                        None,
12900                        invalid,
12901                    ),
12902                )
12903                .unwrap();
12904            assert_eq!(response[0].header.ty, FrameType::Error);
12905            let error = parse_error(&response[0]);
12906            assert_eq!(error["code"], "invalid_manifest");
12907            assert!(error["message"]
12908                .as_str()
12909                .unwrap()
12910                .contains(&format!("rich_ui.{field}")));
12911            assert!(registry
12912                .get_module(&format!("bad-rich-ui-{index}"))
12913                .unwrap()
12914                .is_none());
12915        }
12916    }
12917
12918    #[test]
12919    fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
12920        let registry = Arc::new(Registry::default());
12921        let handler = ControlHandler::new(Arc::clone(&registry));
12922        let connection = ConnectionId::new(219);
12923        registry
12924            .register_with_control_ops(
12925                ModuleManifest::builder("invalid-rich-ui", "0.1.0")
12926                    .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
12927                        "bad schema",
12928                        format!("sha256:{}", "a".repeat(64)),
12929                        128,
12930                        2,
12931                    )))
12932                    .build(),
12933                PROTOCOL_VERSION,
12934                connection,
12935                Vec::new(),
12936            )
12937            .unwrap();
12938        let request =
12939            Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
12940        let response = handler
12941            .handle_catalog_update(connection, request, Vec::new(), None, None)
12942            .unwrap();
12943        assert_eq!(response[0].header.ty, FrameType::Error);
12944        let error = parse_error(&response[0]);
12945        assert_eq!(error["code"], "invalid_manifest");
12946        assert!(error["message"]
12947            .as_str()
12948            .unwrap()
12949            .contains("rich_ui.schema"));
12950    }
12951
12952    #[test]
12953    fn swap_candidate_hello_is_nonce_verified_after_promotion() {
12954        let registry = Arc::new(Registry::default());
12955        let supervisor = SupervisorHandle::new();
12956        supervisor.open_swap("provider", "candidate-nonce".to_string());
12957        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
12958        let connection = ConnectionId::new(218);
12959        let hello = handler
12960            .handle_control(
12961                connection,
12962                hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
12963            )
12964            .unwrap();
12965        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
12966        assert!(registry.list_modules().unwrap().1.is_empty());
12967
12968        registry.promote_candidate("provider").unwrap().unwrap();
12969        let request = Frame::build(
12970            FrameType::Request,
12971            control_flags(),
12972            0,
12973            0,
12974            219,
12975            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
12976        )
12977        .unwrap();
12978        let response = handler.handle_catalog_list(request, None).unwrap();
12979        let ClientControlResponse::CatalogList { modules, .. } =
12980            serde_json::from_slice(&response[0].body).unwrap()
12981        else {
12982            panic!("expected catalog.list");
12983        };
12984        assert_eq!(modules.len(), 1);
12985        assert!(modules[0].nonce_verified);
12986    }
12987
12988    #[test]
12989    fn catalog_list_with_provenance_mirrors_declared_build_bytes() {
12990        for provenance_json in [
12991            r#"{"build_git_sha":"0123456789abcdef0123456789abcdef01234567","build_lock_digest":"abababababababababababababababababababababababababababababababab","wire_crate_version":"0.34.0","store_schema_version":"7","launch_nonce_source":"fd"}"#,
12992            r#"{"build_git_sha_absence_reason":"no_git_dir","build_lock_digest":"abababababababababababababababababababababababababababababababab"}"#,
12993        ] {
12994            let registry = Arc::new(Registry::default());
12995            let handler = ControlHandler::new(registry);
12996            let hello = Frame::build(
12997                FrameType::Hello,
12998                control_flags(),
12999                0,
13000                0,
13001                220,
13002                serde_json::to_vec(&ModuleHelloBody {
13003                    manifest: ModuleManifest::builder("built", "0.1.0")
13004                        .provenance(Some(serde_json::from_str(provenance_json).unwrap()))
13005                        .build(),
13006                    protocol_ver: PROTOCOL_VERSION,
13007                    control_ops: None,
13008                    launch_nonce: None,
13009                })
13010                .unwrap(),
13011            )
13012            .unwrap();
13013            assert_eq!(
13014                handler
13015                    .handle_control(ConnectionId::new(220), hello)
13016                    .unwrap()[0]
13017                    .header
13018                    .ty,
13019                FrameType::HelloAck
13020            );
13021            let request = Frame::build(
13022                FrameType::Request,
13023                control_flags(),
13024                0,
13025                0,
13026                221,
13027                serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
13028            )
13029            .unwrap();
13030            let response = handler.handle_catalog_list(request, None).unwrap();
13031            let ClientControlResponse::CatalogList { modules, .. } =
13032                serde_json::from_slice(&response[0].body).unwrap()
13033            else {
13034                panic!("expected catalog.list");
13035            };
13036            assert_eq!(modules.len(), 1);
13037            assert_eq!(
13038                serde_json::to_vec(&catalog_legacy_projection(modules[0].clone())).unwrap(),
13039                format!(
13040                    r#"{{"module_id":"built","ready":true,"module_version":"0.1.0","provenance":{provenance_json},"roles":[],"control_ops":["route.bind","route.status"]}}"#
13041                )
13042                .as_bytes()
13043            );
13044        }
13045    }
13046
13047    #[test]
13048    fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
13049        let registry = Arc::new(Registry::default());
13050        let handler = ControlHandler::new(Arc::clone(&registry));
13051        let hello = Frame::build(
13052            FrameType::Hello,
13053            control_flags(),
13054            0,
13055            0,
13056            220,
13057            serde_json::to_vec(&ModuleHelloBody {
13058                manifest: ModuleManifest::builder("plain", "0.1.0").build(),
13059                protocol_ver: PROTOCOL_VERSION,
13060                control_ops: None,
13061                launch_nonce: Some("unverified".to_string()),
13062            })
13063            .unwrap(),
13064        )
13065        .unwrap();
13066        assert_eq!(
13067            handler
13068                .handle_control(ConnectionId::new(220), hello)
13069                .unwrap()[0]
13070                .header
13071                .ty,
13072            FrameType::HelloAck
13073        );
13074        let request = Frame::build(
13075            FrameType::Request,
13076            control_flags(),
13077            0,
13078            0,
13079            221,
13080            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
13081        )
13082        .unwrap();
13083        let response = handler.handle_catalog_list(request, None).unwrap();
13084        let wire: serde_json::Value = serde_json::from_slice(&response[0].body).unwrap();
13085        assert!(wire["modules"][0].get("provenance").is_none());
13086        let ClientControlResponse::CatalogList { modules, .. } =
13087            serde_json::from_slice(&response[0].body).unwrap()
13088        else {
13089            panic!("expected catalog.list");
13090        };
13091        assert_eq!(
13092            serde_json::to_vec(&catalog_legacy_projection(modules[0].clone())).unwrap(),
13093            br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
13094        );
13095        assert!(!modules[0].nonce_verified);
13096        assert!(modules[0].provenance.is_none());
13097    }
13098
13099    // Preserve the legacy-field byte pins while discovery adds optional facts.
13100    fn catalog_legacy_projection(mut entry: CatalogEntry) -> CatalogEntry {
13101        assert_eq!(entry.first_party, Some(false));
13102        entry.incarnation = None;
13103        entry.spawn = None;
13104        entry.first_party = None;
13105        entry.origin = None;
13106        entry.requested_declarations_digest = None;
13107        entry.approved_declarations_digest = None;
13108        entry
13109    }
13110
13111    #[test]
13112    fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
13113        let registry = Arc::new(Registry::default());
13114        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
13115            [("vault".to_string(), true), ("squatter".to_string(), true)],
13116            BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
13117        );
13118        let mut squatter = manifest("squatter", PROTOCOL_VERSION);
13119        squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
13120            provides: vec!["credentials-provider/v1".to_string()],
13121            requires: Vec::new(),
13122            must_never_reach: Vec::new(),
13123        });
13124        let frame = Frame::build(
13125            FrameType::Hello,
13126            control_flags(),
13127            0,
13128            0,
13129            77,
13130            serde_json::to_vec(&ModuleHelloBody {
13131                manifest: squatter,
13132                protocol_ver: PROTOCOL_VERSION,
13133                control_ops: None,
13134                launch_nonce: None,
13135            })
13136            .expect("HELLO serializes"),
13137        )
13138        .expect("HELLO frame builds");
13139        let response = handler
13140            .handle_control(ConnectionId::new(77), frame)
13141            .expect("reserved claim receives a typed refusal");
13142        assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
13143        assert_eq!(
13144            registry
13145                .active_registration_count()
13146                .expect("registry reads"),
13147            0,
13148            "a reserved capability refusal must not leave a catalog entry"
13149        );
13150    }
13151
13152    #[test]
13153    fn stale_relay_settlement_cannot_release_the_half_open_probe() {
13154        for settlement in ["timeout", "inconclusive", "drop"] {
13155            let breakers = RouteBindBreakers::default();
13156            let RouteBindAdmission::Admitted {
13157                guard: mut old,
13158                probe: false,
13159            } = breakers.admit("prov")
13160            else {
13161                panic!("ordinary relay admitted")
13162            };
13163            let RouteBindAdmission::Admitted {
13164                guard: mut opener, ..
13165            } = breakers.admit("prov")
13166            else {
13167                panic!("second relay admitted")
13168            };
13169            assert!(
13170                !opener
13171                    .record_timeout(1, Duration::ZERO)
13172                    .unwrap()
13173                    .reopened_after_probe
13174            );
13175            let RouteBindAdmission::Admitted {
13176                guard: mut probe,
13177                probe: true,
13178            } = breakers.admit("prov")
13179            else {
13180                panic!("one cooldown probe admitted")
13181            };
13182            match settlement {
13183                "timeout" => assert!(
13184                    !old.record_timeout(1, Duration::ZERO)
13185                        .unwrap()
13186                        .reopened_after_probe
13187                ),
13188                "inconclusive" => old.record_inconclusive(),
13189                "drop" => drop(old),
13190                _ => unreachable!(),
13191            }
13192            assert!(
13193                matches!(
13194                    breakers.admit("prov"),
13195                    RouteBindAdmission::Refused {
13196                        probe_in_flight: true,
13197                        ..
13198                    }
13199                ),
13200                "{settlement} of a pre-open relay cannot release the real probe"
13201            );
13202            assert!(
13203                probe
13204                    .record_timeout(1, Duration::ZERO)
13205                    .unwrap()
13206                    .reopened_after_probe
13207            );
13208            assert!(matches!(
13209                breakers.admit("prov"),
13210                RouteBindAdmission::Admitted { probe: true, .. }
13211            ));
13212        }
13213        let breakers = RouteBindBreakers::default();
13214        let admit = || match breakers.admit("prov") {
13215            RouteBindAdmission::Admitted { guard, .. } => guard,
13216            _ => panic!("relay admitted"),
13217        };
13218        admit().record_timeout(1, Duration::ZERO);
13219        let mut old_probe = admit();
13220        breakers.reset_for_new_module_connection("prov");
13221        admit().record_timeout(1, Duration::ZERO);
13222        let _new_probe = admit();
13223        old_probe.record_inconclusive();
13224        assert!(matches!(
13225            breakers.admit("prov"),
13226            RouteBindAdmission::Refused {
13227                probe_in_flight: true,
13228                ..
13229            }
13230        ));
13231    }
13232
13233    #[tokio::test]
13234    async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
13235        for candidate in [false, true] {
13236            let registry = Arc::new(Registry::default());
13237            let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
13238                [("vault".to_string(), true), ("squatter".to_string(), true)],
13239                BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
13240            );
13241            let conn = ConnectionId::new(77);
13242            let (ctx, mut rx) = route_ctx(conn);
13243            let initial = capability_manifest("squatter", &[], &[]);
13244            if candidate {
13245                registry
13246                    .register_candidate_with_control_ops(
13247                        initial.clone(),
13248                        PROTOCOL_VERSION,
13249                        conn,
13250                        module_baseline_control_ops(),
13251                    )
13252                    .unwrap();
13253                handler
13254                    .forwarding
13255                    .register_candidate_module_connection(
13256                        conn,
13257                        "squatter".to_string(),
13258                        PROTOCOL_VERSION,
13259                        manifest_concurrency(&initial),
13260                        ctx.egress.clone(),
13261                    )
13262                    .unwrap();
13263            } else {
13264                hello_via_sink(
13265                    &handler,
13266                    &ctx,
13267                    &mut rx,
13268                    hello_frame_with_manifest(initial.clone(), 1),
13269                )
13270                .await;
13271            }
13272            let response = handler
13273                .handle_control_frame(
13274                    &ctx,
13275                    catalog_update_with_capabilities_frame(
13276                        2,
13277                        capability_manifest("squatter", &["credentials-provider/v1"], &[])
13278                            .capabilities
13279                            .unwrap(),
13280                    ),
13281                )
13282                .await
13283                .unwrap();
13284            assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
13285            assert_eq!(
13286                registry
13287                    .get_module_by_connection(conn)
13288                    .unwrap()
13289                    .unwrap()
13290                    .manifest,
13291                initial
13292            );
13293        }
13294    }
13295
13296    #[test]
13297    #[cfg(not(feature = "dev-desktop-approval"))]
13298    fn server_describe_default_build_advertises_no_config_capability() {
13299        let assert_default = |handler: &ControlHandler| {
13300            let request = Frame::build(
13301                FrameType::Request,
13302                control_flags(),
13303                0,
13304                0,
13305                79,
13306                serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13307            )
13308            .unwrap();
13309            let responses = handler.handle_server_describe(request).unwrap();
13310            assert_eq!(responses.len(), 1);
13311            let response: Value = serde_json::from_slice(&responses[0].body).unwrap();
13312            assert_eq!(response["op"], "server.describe");
13313            let capabilities = response["capabilities"]
13314                .as_array()
13315                .expect("capability array");
13316            assert!(
13317                !capabilities.is_empty(),
13318                "exercise an actual describe response, not an empty fixture"
13319            );
13320            assert!(
13321                capabilities
13322                    .iter()
13323                    .all(|capability| !capability.as_str().unwrap().starts_with("config.")),
13324                "a default build must not announce configuration authority: {capabilities:?}"
13325            );
13326        };
13327        assert_default(&ControlHandler::new(Arc::new(Registry::default())));
13328        #[cfg(unix)]
13329        {
13330            // A local store does not prove that project-membership validation,
13331            // active-scope usage tracking and approved-program launch selection
13332            // are connected. Defaults must not advertise that authority.
13333            let (_rig, _owner, _supervisor, handler, _spec) =
13334                crate::config_gate::runtime::RuntimeOwner::unactivated_supervisor_fixture_for_test(
13335                );
13336            assert_default(&handler);
13337        }
13338    }
13339
13340    #[test]
13341    fn server_describe_surfaces_required_capability_verdict_fields() {
13342        let registry = Arc::new(Registry::default());
13343        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
13344            [
13345                ("consumer".to_string(), true),
13346                ("provider".to_string(), false),
13347            ],
13348            BTreeMap::new(),
13349        );
13350        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
13351        consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
13352            provides: Vec::new(),
13353            requires: vec![subc_protocol::manifest::CapabilityRequirement {
13354                capability: "credentials-provider/v1".to_string(),
13355                need: subc_protocol::manifest::CapabilityNeed::Required,
13356            }],
13357            must_never_reach: Vec::new(),
13358        });
13359        let hello = Frame::build(
13360            FrameType::Hello,
13361            control_flags(),
13362            0,
13363            0,
13364            78,
13365            serde_json::to_vec(&ModuleHelloBody {
13366                manifest: consumer,
13367                protocol_ver: PROTOCOL_VERSION,
13368                control_ops: None,
13369                launch_nonce: None,
13370            })
13371            .expect("HELLO serializes"),
13372        )
13373        .expect("HELLO frame builds");
13374        handler
13375            .handle_control(ConnectionId::new(78), hello)
13376            .expect("consumer registers");
13377        let describe = Frame::build(
13378            FrameType::Request,
13379            control_flags(),
13380            0,
13381            0,
13382            79,
13383            serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
13384                .expect("request serializes"),
13385        )
13386        .expect("describe frame builds");
13387        let response = handler
13388            .handle_server_describe(describe)
13389            .expect("server.describe succeeds");
13390        let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
13391        let requirement = &rendered["capability_requirements"][0];
13392        assert_eq!(requirement["consumer"], "consumer");
13393        assert_eq!(requirement["verdict"], "never_provided");
13394        assert_eq!(requirement["episode_seq"], 1);
13395        assert_eq!(requirement["config_satisfiable"], false);
13396        assert_eq!(requirement["runtime_available"], false);
13397        assert!(requirement["detail"]
13398            .as_str()
13399            .expect("detail string")
13400            .contains("credentials-provider/v1"));
13401    }
13402
13403    /// Register a consumer that requires `project-identity/v1` and the provider
13404    /// of it, optionally mark daemon shutdown as begun, then deregister the
13405    /// provider as its connection closes. Returns the `capability.requirement`
13406    /// events that deregistration logged and the verdict the evaluator then
13407    /// holds for the requirement (the one `route.open` refusals read).
13408    fn deregister_required_provider(
13409        daemon_shutdown_begun: bool,
13410    ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
13411        const CAPABILITY: &str = "project-identity/v1";
13412        let registry = Arc::new(Registry::default());
13413        let daemon_shutdown = DaemonShutdownFlag::default();
13414        let handler = ControlHandler::new(Arc::clone(&registry))
13415            .with_capability_config(
13416                [
13417                    ("consumer".to_string(), true),
13418                    ("provider".to_string(), true),
13419                ],
13420                BTreeMap::new(),
13421            )
13422            .with_daemon_shutdown_flag(daemon_shutdown.clone());
13423        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
13424        consumer.capabilities = Some(CapabilityDeclarations {
13425            provides: Vec::new(),
13426            requires: vec![subc_protocol::manifest::CapabilityRequirement {
13427                capability: CAPABILITY.to_string(),
13428                need: subc_protocol::manifest::CapabilityNeed::Required,
13429            }],
13430            must_never_reach: Vec::new(),
13431        });
13432        let provider = capability_manifest("provider", &[CAPABILITY], &[]);
13433        let provider_conn = ConnectionId::new(182);
13434        for (conn, manifest, corr) in [
13435            (ConnectionId::new(181), consumer, 181),
13436            (provider_conn, provider, 182),
13437        ] {
13438            let hello = handler
13439                .handle_control(conn, hello_frame_with_manifest(manifest, corr))
13440                .expect("module registers");
13441            assert_eq!(hello[0].header.ty, FrameType::HelloAck);
13442        }
13443        assert_eq!(
13444            handler.capability_evaluator.verdict("consumer", CAPABILITY),
13445            Some(CapabilityVerdict::Provided),
13446            "the registered provider must satisfy the requirement first"
13447        );
13448        if daemon_shutdown_begun {
13449            daemon_shutdown.set_for_test();
13450        }
13451
13452        let capture = EventCapture::default();
13453        let _subscriber =
13454            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13455        handler
13456            .cleanup_connection(provider_conn)
13457            .expect("provider deregisters");
13458        let events = capture
13459            .events()
13460            .into_iter()
13461            .filter(|event| {
13462                event.fields.get("message").map(String::as_str) == Some("capability.requirement")
13463            })
13464            .collect();
13465        (
13466            events,
13467            handler.capability_evaluator.verdict("consumer", CAPABILITY),
13468        )
13469    }
13470
13471    #[test]
13472    fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
13473        let (events, verdict) = deregister_required_provider(true);
13474        assert_eq!(
13475            verdict,
13476            Some(CapabilityVerdict::NeverProvided),
13477            "shutdown must not change the verdict route.open refusals read"
13478        );
13479        assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
13480        let event = &events[0];
13481        assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
13482        assert_eq!(
13483            event.fields.get("cause").map(String::as_str),
13484            Some("\"daemon_shutdown\""),
13485            "{event:?}"
13486        );
13487        assert_eq!(
13488            event.fields.get("verdict").map(String::as_str),
13489            Some("\"never_provided\""),
13490            "{event:?}"
13491        );
13492    }
13493
13494    #[test]
13495    fn required_provider_leaving_while_daemon_runs_still_logs_error() {
13496        let (events, verdict) = deregister_required_provider(false);
13497        assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
13498        assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
13499        let event = &events[0];
13500        assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
13501        assert_eq!(event.fields.get("cause"), None, "{event:?}");
13502        assert_eq!(
13503            event.fields.get("verdict").map(String::as_str),
13504            Some("\"never_provided\""),
13505            "{event:?}"
13506        );
13507    }
13508
13509    #[test]
13510    fn catalog_list_omits_capabilities_for_legacy_manifest() {
13511        let registry = Arc::new(Registry::default());
13512        let handler = ControlHandler::new(Arc::clone(&registry));
13513        let hello = handler
13514            .handle_control(
13515                ConnectionId::new(101),
13516                hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
13517            )
13518            .expect("legacy HELLO registers");
13519        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
13520
13521        let request = Frame::build(
13522            FrameType::Request,
13523            control_flags(),
13524            0,
13525            0,
13526            102,
13527            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
13528                .expect("catalog request serializes"),
13529        )
13530        .expect("catalog request frame builds");
13531        let response = handler
13532            .handle_catalog_list(request, None)
13533            .expect("catalog list succeeds");
13534        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
13535        assert!(
13536            body["modules"][0].get("capabilities").is_none(),
13537            "legacy manifest must retain an absent capabilities field on catalog.list"
13538        );
13539    }
13540
13541    #[test]
13542    fn hello_ack_omits_storage_when_no_storage_config() {
13543        let registry = Arc::new(Registry::default());
13544        let handler = ControlHandler::new(Arc::clone(&registry));
13545        let responses = handler
13546            .handle_control(
13547                ConnectionId::new(1),
13548                hello_frame("aft", PROTOCOL_VERSION, 7),
13549            )
13550            .unwrap();
13551        let ack = parse_ack(&responses[0]);
13552        assert_eq!(ack.storage, None, "no storage config -> no descriptor");
13553        assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
13554    }
13555
13556    #[tokio::test]
13557    async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
13558        let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
13559        let registry = Arc::new(Registry::default());
13560        let handler = ControlHandler::new(Arc::clone(&registry)).with_machine_id(Some(id.clone()));
13561        let responses = handler
13562            .handle_control(
13563                ConnectionId::new(1),
13564                hello_frame("aft", PROTOCOL_VERSION, 7),
13565            )
13566            .unwrap();
13567        let ack = parse_ack(&responses[0]);
13568        assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
13569
13570        let described = handler
13571            .handle_control_frame(
13572                &route_ctx(ConnectionId::new(2)).0,
13573                Frame::build(
13574                    FrameType::Request,
13575                    control_flags(),
13576                    0,
13577                    0,
13578                    9,
13579                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13580                )
13581                .unwrap(),
13582            )
13583            .await
13584            .unwrap();
13585        let ClientControlResponse::ServerDescribe { machine_id, .. } =
13586            serde_json::from_slice(&described[0].body).unwrap()
13587        else {
13588            panic!("server.describe answered with another shape");
13589        };
13590        assert_eq!(machine_id.as_deref(), Some(id.as_str()));
13591    }
13592
13593    #[test]
13594    fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
13595        // With a central sqlite storage policy, each registering module gets its
13596        // own resolved descriptor in HELLO_ACK, keyed by its module id.
13597        let registry = Arc::new(Registry::default());
13598        let handler = ControlHandler::new(Arc::clone(&registry)).with_storage_config(Some(
13599            crate::daemon_config::StorageConfig::Sqlite {
13600                data_home: std::path::PathBuf::from("/data"),
13601            },
13602        ));
13603
13604        let responses = handler
13605            .handle_control(
13606                ConnectionId::new(1),
13607                hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
13608            )
13609            .unwrap();
13610        let ack = parse_ack(&responses[0]);
13611        assert_eq!(
13612            ack.storage,
13613            Some(serde_json::json!({
13614                "module_id": "alfonso-routing",
13615                "storage_namespace": "default",
13616                "isolation": { "kind": "module" },
13617                "backend": {
13618                    "backend": "sqlite",
13619                    "path": "/data/cortexkit/alfonso-routing/store.db"
13620                }
13621            })),
13622            "the delivered descriptor is the module's own sqlite store path"
13623        );
13624    }
13625
13626    #[test]
13627    fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
13628        let registry = Arc::new(Registry::default());
13629        let handler = ControlHandler::new(Arc::clone(&registry));
13630        let conn = ConnectionId::new(1);
13631        let responses = handler
13632            .handle_control(
13633                conn,
13634                hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
13635            )
13636            .unwrap();
13637        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
13638        let registration = registry.get_module("aft").unwrap().unwrap();
13639        assert_eq!(registration.control_ops, module_baseline_control_ops());
13640
13641        let frame =
13642            Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
13643        assert!(handler
13644            .guard_module_control_op(&frame, "aft", "route.bind")
13645            .unwrap()
13646            .is_none());
13647        let error = handler
13648            .guard_module_control_op(&frame, "aft", "test.synthetic")
13649            .unwrap()
13650            .expect("synthetic ungranted op should be rejected");
13651        assert_eq!(error.header.ty, FrameType::Error);
13652        assert_eq!(parse_error(&error)["code"], "op_not_allowed");
13653    }
13654
13655    #[test]
13656    fn hello_control_ops_some_adds_optional_grants() {
13657        let registry = Arc::new(Registry::default());
13658        let handler = ControlHandler::new(Arc::clone(&registry));
13659        handler
13660            .handle_control(
13661                ConnectionId::new(1),
13662                hello_frame_with_control_ops(
13663                    "aft",
13664                    PROTOCOL_VERSION,
13665                    7,
13666                    Some(vec![
13667                        "future.synthetic".to_string(),
13668                        "route.bind".to_string(),
13669                    ]),
13670                ),
13671            )
13672            .unwrap();
13673        let registration = registry.get_module("aft").unwrap().unwrap();
13674        assert_eq!(
13675            registration.control_ops,
13676            vec![
13677                "route.bind".to_string(),
13678                "route.status".to_string(),
13679                "future.synthetic".to_string(),
13680            ]
13681        );
13682        let frame =
13683            Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
13684        assert!(handler
13685            .guard_module_control_op(&frame, "aft", "future.synthetic")
13686            .unwrap()
13687            .is_none());
13688    }
13689
13690    #[tokio::test]
13691    async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
13692        let registry = Arc::new(Registry::default());
13693        let forwarding = Arc::new(ForwardingTable::default());
13694        let handler =
13695            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13696        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
13697        hello_via_sink(
13698            &handler,
13699            &module_ctx,
13700            &mut module_rx,
13701            hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
13702        )
13703        .await;
13704
13705        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
13706        let responses = handler
13707            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
13708            .await
13709            .unwrap();
13710        assert_eq!(responses.len(), 1);
13711        assert_eq!(responses[0].header.ty, FrameType::Error);
13712        assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
13713        assert!(module_rx.try_recv().is_err());
13714    }
13715
13716    #[tokio::test]
13717    async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
13718        let registry = Arc::new(Registry::default());
13719        let forwarding = Arc::new(ForwardingTable::default());
13720        let handler =
13721            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13722        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
13723        hello_via_sink(
13724            &handler,
13725            &module_ctx,
13726            &mut module_rx,
13727            hello_frame_with_control_ops(
13728                "aft",
13729                PROTOCOL_VERSION,
13730                7,
13731                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13732            ),
13733        )
13734        .await;
13735
13736        let project_root = unique_project_root("demux");
13737        let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
13738        let route_handler = handler.clone();
13739        let route_task = tokio::spawn(async move {
13740            route_handler
13741                .handle_control_frame(
13742                    &route_client_ctx,
13743                    route_open_frame(100, "aft", project_root),
13744                )
13745                .await
13746                .unwrap()
13747        });
13748        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13749            .await
13750            .unwrap()
13751            .unwrap();
13752        assert!(matches!(
13753            serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
13754            ModuleControlRequest::RouteBind { .. }
13755        ));
13756
13757        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
13758        let health_handler = handler.clone();
13759        let health_task = tokio::spawn(async move {
13760            health_handler
13761                .handle_control_frame(
13762                    &health_client_ctx,
13763                    supervisor_health_probe_frame(101, "aft"),
13764                )
13765                .await
13766                .unwrap()
13767        });
13768        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13769            .await
13770            .unwrap()
13771            .unwrap();
13772        assert_eq!(
13773            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
13774            ModuleControlRequest::HealthCheck {}
13775        );
13776
13777        handler
13778            .handle_control_frame(
13779                &module_ctx,
13780                health_response(health_frame.header.corr, HealthStatus::Degraded),
13781            )
13782            .await
13783            .unwrap();
13784        let health_response = health_task.await.unwrap();
13785        assert_eq!(health_response.len(), 1);
13786        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
13787            ClientControlResponse::SupervisorHealthProbe {
13788                module_id,
13789                status,
13790                detail,
13791                metrics,
13792            } => {
13793                assert_eq!(module_id, "aft");
13794                assert_eq!(status, HealthStatus::Degraded);
13795                assert_eq!(detail.as_deref(), Some("warming"));
13796                assert_eq!(metrics, Some(json!({"queue_depth": 3})));
13797            }
13798            other => panic!("unexpected health response: {other:?}"),
13799        }
13800
13801        handler
13802            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13803            .await
13804            .unwrap();
13805        let route_response = route_task.await.unwrap();
13806        assert!(route_response.is_empty());
13807        let published = route_client_rx.recv().await.unwrap();
13808        assert!(matches!(
13809            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13810            ClientControlResponse::RouteOpen { .. }
13811        ));
13812    }
13813
13814    /// Start one `route.open` on `client_connection` and return its still-running
13815    /// handler task together with the `route.bind` the module received for it.
13816    /// The handler blocks until the module answers, so it has to run as a task
13817    /// while the test drives the module side.
13818    async fn relay_route_open(
13819        handler: &ControlHandler,
13820        client_connection: ConnectionId,
13821        client_egress: &FrameSink,
13822        module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
13823        corr: u64,
13824        module_id: &str,
13825        project_root_label: &str,
13826    ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
13827        let ctx = RouteCtx {
13828            connection_id: client_connection,
13829            egress: client_egress.clone(),
13830        };
13831        let handler = handler.clone();
13832        let project_root = unique_project_root(project_root_label);
13833        let module_id = module_id.to_string();
13834        let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
13835        let task = tokio::spawn(async move {
13836            let _guard = tracing::dispatcher::set_default(&dispatch);
13837            handler
13838                .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
13839                .await
13840                .unwrap()
13841        });
13842        let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
13843            .await
13844            .expect("module receives the relayed route.bind")
13845            .expect("module egress is open");
13846        (task, bind.frame)
13847    }
13848
13849    fn route_bind_channel(frame: &Frame) -> (u16, u32) {
13850        match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
13851            ModuleControlRequest::RouteBind {
13852                route_channel,
13853                epoch,
13854                ..
13855            } => (route_channel, epoch),
13856            other => panic!("expected a route.bind request, got {other:?}"),
13857        }
13858    }
13859
13860    fn published_route(frame: &Frame) -> (u16, u32) {
13861        match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
13862            ClientControlResponse::RouteOpen {
13863                route_channel,
13864                route_epoch,
13865                ..
13866            } => (route_channel, route_epoch),
13867            other => panic!("expected a route.open response, got {other:?}"),
13868        }
13869    }
13870
13871    /// Reproduction of a production outage. A client had `route.open`s in
13872    /// flight to a module and was already marked closing -- its egress had refused a
13873    /// module frame, so the daemon asked its connection to end -- while its sink
13874    /// was still open. When the module acked those binds, the daemon refused to
13875    /// commit a route for a closing client, and that refusal was returned from
13876    /// the MODULE connection's frame handler, where a router error that has no
13877    /// ERROR-frame translation ends the connection. The module saw EOF, exited 0,
13878    /// the supervisor correctly did not respawn a clean exit, and every seat lost
13879    /// its tools for hours -- one client's teardown took down a connection
13880    /// carrying ~170 other routes.
13881    ///
13882    /// The window is opened here by calling the production path that opens it
13883    /// (`escalate_client_delivery_failure`) rather than by closing a socket. The
13884    /// state that matters is "in `closing_connections`, sink still open, relay
13885    /// still pending", and it lasts only from the close request until the
13886    /// connection loop reacts to it; a socket-level test can flood a client into
13887    /// that escalation but cannot pin the module's ack inside the window. Closing
13888    /// the socket instead takes the other path entirely -- connection teardown
13889    /// removes the pending relay under the same lock, so the ack finds nothing.
13890    #[tokio::test]
13891    async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
13892        let registry = Arc::new(Registry::default());
13893        let forwarding = Arc::new(ForwardingTable::default());
13894        let handler =
13895            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13896
13897        let module_connection = ConnectionId::new(30);
13898        let (module_ctx, mut module_rx) = route_ctx(module_connection);
13899        hello_via_sink(
13900            &handler,
13901            &module_ctx,
13902            &mut module_rx,
13903            hello_frame("aft", PROTOCOL_VERSION, 7),
13904        )
13905        .await;
13906
13907        let dying_client = ConnectionId::new(31);
13908        let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
13909
13910        // A published route on the dying client. The escalation below only marks
13911        // a connection closing for a route it has already published.
13912        let (first_task, first_bind) = relay_route_open(
13913            &handler,
13914            dying_client,
13915            &dying_ctx.egress,
13916            &mut module_rx,
13917            100,
13918            "aft",
13919            "closing-first",
13920        )
13921        .await;
13922        handler
13923            .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
13924            .await
13925            .unwrap();
13926        assert!(first_task.await.unwrap().is_empty());
13927        let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
13928
13929        // A second route.open from the same client, relayed and awaiting its ack.
13930        let (second_task, second_bind) = relay_route_open(
13931            &handler,
13932            dying_client,
13933            &dying_ctx.egress,
13934            &mut module_rx,
13935            101,
13936            "aft",
13937            "closing-second",
13938        )
13939        .await;
13940        let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
13941
13942        // The window: the client is closing, its sink is still open, and its
13943        // second bind is still pending.
13944        assert!(forwarding
13945            .escalate_client_delivery_failure(
13946                dying_client,
13947                first_channel,
13948                first_epoch,
13949                CloseReason::new(
13950                    "module_to_client_delivery_failed",
13951                    "client egress refused a module frame",
13952                ),
13953                crate::forwarding::UndeliveredFrame {
13954                    module_id: None,
13955                    sink: &dying_ctx.egress,
13956                },
13957            )
13958            .unwrap());
13959        assert!(!dying_ctx.egress.is_closed());
13960
13961        // The frame that used to end the module connection.
13962        let ack = handler
13963            .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
13964            .await;
13965        let module_loop_error = ack.as_ref().err().map(ToString::to_string);
13966        if module_loop_error.is_some() {
13967            // What the server's connection loop does with a router error that has
13968            // no ERROR-frame translation: end the connection, which releases the
13969            // module's registration and every route on it.
13970            handler.cleanup_connection(module_connection).unwrap();
13971        }
13972        // Read the module's next frame before opening the co-tenant's route, so
13973        // the GOODBYE assertion below is about THIS ack and not about later
13974        // traffic. `None` means the module was told nothing.
13975        let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13976            .await
13977            .ok()
13978            .flatten();
13979
13980        // 1. The module connection is still registered.
13981        assert!(
13982            registry
13983                .get_module_by_connection(module_connection)
13984                .unwrap()
13985                .is_some(),
13986            "one client's closing connection ended the shared module connection: \
13987             {module_loop_error:?}"
13988        );
13989        // ...and still serving: another client can open and use a route on it.
13990        let cotenant = ConnectionId::new(32);
13991        let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
13992        let (cotenant_task, cotenant_bind) = relay_route_open(
13993            &handler,
13994            cotenant,
13995            &cotenant_ctx.egress,
13996            &mut module_rx,
13997            102,
13998            "aft",
13999            "closing-cotenant",
14000        )
14001        .await;
14002        handler
14003            .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
14004            .await
14005            .unwrap();
14006        assert!(cotenant_task.await.unwrap().is_empty());
14007        let (cotenant_channel, cotenant_epoch) =
14008            published_route(&cotenant_rx.recv().await.unwrap());
14009        assert!(matches!(
14010            forwarding
14011                .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
14012                .unwrap(),
14013            DataRoute::Client(DataRouteState::Bound(_))
14014        ));
14015
14016        // 2. The module was told to drop the binding it created for the route
14017        //    that will never be published.
14018        let goodbye = post_ack_module_frame
14019            .expect("module receives a GOODBYE for the abandoned route channel");
14020        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14021        assert_eq!(goodbye.header.channel, abandoned_channel);
14022        assert_eq!(goodbye.header.epoch, abandoned_epoch);
14023
14024        // 3. The dying client received nothing: no route was ever published to
14025        //    it. Its route.open is answered as unavailable, which the connection
14026        //    loop would write to a socket that is already going away.
14027        assert!(dying_rx.try_recv().is_err());
14028        let second_response = second_task.await.unwrap();
14029        assert_eq!(second_response.len(), 1);
14030        assert_eq!(
14031            parse_error(&second_response[0])["code"],
14032            "target_unavailable"
14033        );
14034    }
14035
14036    /// One `catalog.list` as a client sees it: the generation and the listed
14037    /// module ids with their readiness.
14038    async fn catalog_snapshot(
14039        handler: &ControlHandler,
14040        client_ctx: &RouteCtx,
14041        corr: u64,
14042    ) -> (u64, Vec<(String, bool)>) {
14043        let body =
14044            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
14045        let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
14046        let reply = handler
14047            .handle_control_frame(client_ctx, request)
14048            .await
14049            .unwrap()
14050            .pop()
14051            .unwrap();
14052        let ClientControlResponse::CatalogList {
14053            generation,
14054            modules,
14055            ..
14056        } = serde_json::from_slice(&reply.body).unwrap()
14057        else {
14058            panic!("not a catalog.list reply");
14059        };
14060        (
14061            generation,
14062            modules
14063                .into_iter()
14064                .map(|entry| (entry.module_id, entry.ready))
14065                .collect(),
14066        )
14067    }
14068
14069    /// A handler whose HELLO for module "aft" on connection 40 is parked
14070    /// between reserving its registry entry and installing its forwarding
14071    /// endpoint. Returns the handler, the two-party barrier (wait once to know
14072    /// the HELLO is parked, once more to release it), the HELLO's thread, and
14073    /// the module's context and receiver.
14074    fn hello_held_before_forwarding() -> (
14075        ControlHandler,
14076        Arc<std::sync::Barrier>,
14077        std::thread::JoinHandle<Vec<Frame>>,
14078        RouteCtx,
14079        mpsc::Receiver<crate::router::OutboundFrame>,
14080    ) {
14081        let hold = Arc::new(std::sync::Barrier::new(2));
14082        let handler = ControlHandler::with_forwarding(
14083            Arc::new(Registry::default()),
14084            Arc::new(ForwardingTable::default()),
14085        )
14086        .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
14087        let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
14088        let hello = {
14089            let handler = handler.clone();
14090            let module_ctx = module_ctx.clone();
14091            std::thread::spawn(move || {
14092                handler
14093                    .handle_hello(
14094                        module_ctx.connection_id,
14095                        Some(module_ctx.egress.clone()),
14096                        hello_frame("aft", PROTOCOL_VERSION, 7),
14097                    )
14098                    .unwrap()
14099            })
14100        };
14101        (handler, hold, hello, module_ctx, module_rx)
14102    }
14103
14104    /// If a lookup can acquire the registry while publication is parked, let
14105    /// the lookup finish before releasing HELLO. Otherwise release HELLO only
14106    /// after the lookup has tried the lock. Both schedules avoid timing guesses
14107    /// and return the actual lookup result, including an unpublished snapshot
14108    /// when the publication lock is missing.
14109    fn lookup_during_hello_publication<T: Send + 'static>(
14110        registry: &Registry,
14111        hold: &std::sync::Barrier,
14112        lookup: impl FnOnce() -> T + Send + 'static,
14113    ) -> T {
14114        let attempted = registry.observe_next_lock_attempt();
14115        let reader = std::thread::spawn(lookup);
14116        if attempted.recv().unwrap() {
14117            let result = reader.join().unwrap();
14118            hold.wait();
14119            result
14120        } else {
14121            hold.wait();
14122            reader.join().unwrap()
14123        }
14124    }
14125
14126    #[test]
14127    fn catalog_read_after_hello_ack_waits_for_publication() {
14128        let registry = Arc::new(Registry::default());
14129        let hold = Arc::new(std::sync::Barrier::new(2));
14130        let mut handler = ControlHandler::with_forwarding(
14131            Arc::clone(&registry),
14132            Arc::new(ForwardingTable::default()),
14133        );
14134        handler.hello_ack_to_publication_hold = Some(Arc::clone(&hold));
14135        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
14136        let hello = {
14137            let handler = handler.clone();
14138            std::thread::spawn(move || {
14139                handler
14140                    .handle_hello(
14141                        module_ctx.connection_id,
14142                        Some(module_ctx.egress),
14143                        hello_frame("aft", PROTOCOL_VERSION, 7),
14144                    )
14145                    .unwrap()
14146            })
14147        };
14148        hold.wait();
14149        assert_eq!(
14150            module_rx.try_recv().unwrap().frame.header.ty,
14151            FrameType::HelloAck
14152        );
14153        let replies = lookup_during_hello_publication(&registry, &hold, move || {
14154            // A different connection can issue catalog.list as soon as the
14155            // module has read its ACK, even while HELLO has not returned.
14156            let request = Frame::build(
14157                FrameType::Request,
14158                control_flags(),
14159                0,
14160                0,
14161                300,
14162                serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
14163            )
14164            .unwrap();
14165            handler.handle_catalog_list(request, None).unwrap()
14166        });
14167        assert!(hello.join().unwrap().is_empty());
14168        let ClientControlResponse::CatalogList {
14169            generation,
14170            modules,
14171            ..
14172        } = serde_json::from_slice(&replies[0].body).unwrap()
14173        else {
14174            panic!("not a catalog.list reply");
14175        };
14176        assert_eq!(
14177            modules
14178                .iter()
14179                .map(|entry| entry.module_id.as_str())
14180                .collect::<Vec<_>>(),
14181            vec!["aft"]
14182        );
14183        assert_eq!(generation, 1);
14184    }
14185
14186    /// While a HELLO has reserved its registry entry but its forwarding
14187    /// endpoint is not installed yet, `route.open` to the module can only be
14188    /// refused. `catalog.list` must not list the module in that window: a
14189    /// client that opens a route as soon as it sees a module listed and ready
14190    /// would be refused for a module the daemon had just advertised. The
14191    /// refusal in the window stays the retryable `target_unavailable`, never
14192    /// `unknown_module`, and the id stays claimed against a duplicate HELLO.
14193    /// Once the HELLO completes, the module is listed and a route to it opens.
14194    #[tokio::test]
14195    async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
14196        let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
14197        let client = ConnectionId::new(41);
14198        let (client_ctx, mut client_rx) = route_ctx(client);
14199
14200        hold.wait();
14201        // Parked: the registry holds "aft", forwarding has no endpoint for it.
14202        assert!(handler.registry.get_module("aft").unwrap().is_some());
14203        assert!(!handler
14204            .forwarding
14205            .has_live_module_connection("aft")
14206            .unwrap());
14207        let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
14208        assert!(
14209            listed.is_empty(),
14210            "catalog.list listed a module route.open cannot reach yet: {listed:?}"
14211        );
14212        let refused = handler
14213            .handle_control_frame(
14214                &client_ctx,
14215                route_open_frame(301, "aft", unique_project_root("listed-window")),
14216            )
14217            .await
14218            .unwrap();
14219        assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
14220        let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
14221        let duplicate = handler
14222            .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
14223            .await
14224            .unwrap();
14225        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
14226
14227        hold.wait();
14228        assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
14229        let ack = module_rx.try_recv().unwrap().frame;
14230        assert_eq!(ack.header.ty, FrameType::HelloAck);
14231        let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
14232        assert_eq!(listed, vec![("aft".to_string(), true)]);
14233
14234        let (open_task, bind) = relay_route_open(
14235            &handler,
14236            client,
14237            &client_ctx.egress,
14238            &mut module_rx,
14239            303,
14240            "aft",
14241            "listed-after",
14242        )
14243        .await;
14244        assert!(handler
14245            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
14246            .await
14247            .unwrap()
14248            .is_empty());
14249        assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
14250        published_route(&client_rx.try_recv().unwrap().frame);
14251    }
14252
14253    /// A catalog watcher keys on the generation: when it changes, it re-reads
14254    /// the listing. So the generation must advance exactly when the module
14255    /// appears. If it advanced while the module was still hidden, a watcher
14256    /// would read the new generation without the module, and the module would
14257    /// later appear under that same generation with nothing telling the
14258    /// watcher to look again.
14259    #[tokio::test]
14260    async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
14261        let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
14262        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
14263        let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
14264
14265        hold.wait();
14266        let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
14267        assert!(listed.is_empty());
14268        assert_eq!(
14269            during, before,
14270            "the generation moved while the module was not listed"
14271        );
14272
14273        hold.wait();
14274        assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
14275        let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
14276        assert_eq!(listed, vec![("aft".to_string(), true)]);
14277        assert_eq!(
14278            after,
14279            before + 1,
14280            "the module appeared without exactly one generation bump"
14281        );
14282    }
14283
14284    /// A HELLO whose forwarding endpoint cannot be installed (here the
14285    /// module's sink is already closed, so its HELLO_ACK cannot be queued) is
14286    /// refused and leaves nothing behind: no registry entry, no catalog entry,
14287    /// and the catalog generation where it was.
14288    #[tokio::test]
14289    async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
14290        let registry = Arc::new(Registry::default());
14291        let handler = ControlHandler::with_forwarding(
14292            Arc::clone(&registry),
14293            Arc::new(ForwardingTable::default()),
14294        );
14295        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
14296        let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
14297        let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
14298        drop(module_rx);
14299
14300        let replies = handler
14301            .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
14302            .await
14303            .unwrap();
14304        assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
14305        assert!(registry.get_module("aft").unwrap().is_none());
14306        assert!(registry
14307            .get_module_by_connection(module_ctx.connection_id)
14308            .unwrap()
14309            .is_none());
14310        assert_eq!(
14311            catalog_snapshot(&handler, &client_ctx, 301).await,
14312            (before, Vec::new())
14313        );
14314    }
14315
14316    /// A handler whose `route.open` parks after queueing the route.bind and
14317    /// before marking it enqueued, with module "aft" registered on connection
14318    /// 40. Returns the two-party barrier that parks and releases it.
14319    async fn route_open_held_after_bind_send() -> (
14320        ControlHandler,
14321        Arc<tokio::sync::Barrier>,
14322        RouteCtx,
14323        mpsc::Receiver<crate::router::OutboundFrame>,
14324    ) {
14325        let hold = Arc::new(tokio::sync::Barrier::new(2));
14326        let handler = ControlHandler::with_forwarding(
14327            Arc::new(Registry::default()),
14328            Arc::new(ForwardingTable::default()),
14329        )
14330        .with_route_bind_relay_sent_hold(Arc::clone(&hold));
14331        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
14332        hello_via_sink(
14333            &handler,
14334            &module_ctx,
14335            &mut module_rx,
14336            hello_frame("aft", PROTOCOL_VERSION, 7),
14337        )
14338        .await;
14339        (handler, hold, module_ctx, module_rx)
14340    }
14341
14342    /// A fast module acks the route.bind, and the reader commits the route,
14343    /// before `route.open` has marked the relay enqueued. The daemon used to
14344    /// read the missing relay as an abandoned bind and send the module a
14345    /// GOODBYE for the route it had just committed: the client was told its
14346    /// route was open, the module unbound the channel, and the client's first
14347    /// request was dropped on the module side as unbound.
14348    #[tokio::test]
14349    async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
14350        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
14351        let client = ConnectionId::new(41);
14352        let (client_ctx, mut client_rx) = route_ctx(client);
14353
14354        let (open_task, bind) = relay_route_open(
14355            &handler,
14356            client,
14357            &client_ctx.egress,
14358            &mut module_rx,
14359            200,
14360            "aft",
14361            "acked-before-mark",
14362        )
14363        .await;
14364        let (module_channel, module_epoch) = route_bind_channel(&bind);
14365        // route.open is now parked between queueing the bind and marking it.
14366        hold.wait().await;
14367
14368        // The module answers inside that window, and the route commits.
14369        assert!(handler
14370            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
14371            .await
14372            .unwrap()
14373            .is_empty());
14374        let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
14375
14376        hold.wait().await;
14377        assert!(
14378            open_task.await.unwrap().is_empty(),
14379            "route.open answers accepted"
14380        );
14381
14382        // Every frame route.open could have sent is already queued, because it
14383        // has returned; the module must have been sent nothing.
14384        match module_rx.try_recv() {
14385            Err(mpsc::error::TryRecvError::Empty) => {}
14386            Ok(outbound) => panic!(
14387                "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
14388                outbound.frame.header.ty,
14389                outbound.frame.header.channel,
14390                outbound.frame.header.epoch
14391            ),
14392            Err(err) => panic!("module egress closed: {err}"),
14393        }
14394
14395        // The client's first request on the route is the next thing the module
14396        // reads, on the binding it just created.
14397        let router = Router::with_control_handler(Arc::new(handler.clone()));
14398        let request = Frame::build(
14399            FrameType::Request,
14400            Flags::new(true, Priority::Interactive, false),
14401            client_channel,
14402            client_epoch,
14403            201,
14404            b"first request".to_vec(),
14405        )
14406        .unwrap();
14407        router
14408            .route_for_connection(&client_ctx, request)
14409            .await
14410            .unwrap();
14411        let delivered = module_rx.try_recv().unwrap().frame;
14412        assert_eq!(delivered.header.ty, FrameType::Request);
14413        assert_eq!(
14414            (delivered.header.channel, delivered.header.epoch),
14415            (module_channel, module_epoch)
14416        );
14417        assert_eq!(delivered.body, b"first request");
14418    }
14419
14420    /// The client goes away while `route.open` is still queueing the bind, so
14421    /// the relay is dropped with no module reply. The teardown cannot send the
14422    /// GOODBYE itself (it could reach the module ahead of the bind), so
14423    /// `route.open` must send it once the bind is queued; otherwise the module
14424    /// keeps a binding the daemon will never commit.
14425    #[tokio::test]
14426    async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
14427        let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
14428        let client = ConnectionId::new(42);
14429        let (client_ctx, _client_rx) = route_ctx(client);
14430
14431        let (open_task, bind) = relay_route_open(
14432            &handler,
14433            client,
14434            &client_ctx.egress,
14435            &mut module_rx,
14436            300,
14437            "aft",
14438            "released-before-mark",
14439        )
14440        .await;
14441        let (module_channel, module_epoch) = route_bind_channel(&bind);
14442        hold.wait().await;
14443
14444        handler.cleanup_connection(client).unwrap();
14445        assert!(
14446            module_rx.try_recv().is_err(),
14447            "client teardown must not GOODBYE a bind route.open is still queueing"
14448        );
14449
14450        hold.wait().await;
14451        let response = open_task.await.unwrap();
14452        assert_eq!(response.len(), 1);
14453        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
14454
14455        let goodbye = module_rx
14456            .try_recv()
14457            .expect("module is told to drop the binding the daemon abandoned")
14458            .frame;
14459        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14460        assert_eq!(
14461            (goodbye.header.channel, goodbye.header.epoch),
14462            (module_channel, module_epoch)
14463        );
14464        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
14465    }
14466
14467    /// The module acks inside the same window, but the client's egress closed
14468    /// first, so the reader refuses to commit. The ack proves the module holds
14469    /// the binding, so the reader sends the GOODBYE, and `route.open` must not
14470    /// send a second one.
14471    #[tokio::test]
14472    async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
14473        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
14474        let client = ConnectionId::new(43);
14475        let (client_ctx, client_rx) = route_ctx(client);
14476
14477        let (open_task, bind) = relay_route_open(
14478            &handler,
14479            client,
14480            &client_ctx.egress,
14481            &mut module_rx,
14482            400,
14483            "aft",
14484            "gone-client-before-mark",
14485        )
14486        .await;
14487        let (module_channel, module_epoch) = route_bind_channel(&bind);
14488        hold.wait().await;
14489
14490        drop(client_rx);
14491        assert!(client_ctx.egress.is_closed());
14492        handler
14493            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
14494            .await
14495            .unwrap();
14496
14497        hold.wait().await;
14498        let response = open_task.await.unwrap();
14499        assert_eq!(response.len(), 1);
14500        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
14501
14502        let goodbye = module_rx
14503            .try_recv()
14504            .expect("module is told to drop the binding it acked")
14505            .frame;
14506        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14507        assert_eq!(
14508            (goodbye.header.channel, goodbye.header.epoch),
14509            (module_channel, module_epoch)
14510        );
14511        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
14512        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
14513    }
14514
14515    /// The fence at the module-loop boundary, stated as its own contract: which
14516    /// forwarding failures are allowed to end the module connection that is being
14517    /// served. A `ConnectionClosing` naming some client is about that client, and
14518    /// a module connection is shared; the same error naming the module's own
14519    /// connection is about this connection and must stay fatal, as must failures
14520    /// that are about the forwarding table itself.
14521    #[test]
14522    fn only_the_modules_own_closing_connection_ends_the_module_loop() {
14523        let handler = ControlHandler::default();
14524        let module_connection = ConnectionId::new(30);
14525        let client_connection = ConnectionId::new(31);
14526
14527        handler
14528            .refuse_to_end_module_connection_for_a_client(
14529                module_connection,
14530                77,
14531                ForwardingError::ConnectionClosing {
14532                    connection_id: client_connection,
14533                },
14534            )
14535            .expect("a closing client must never end the module connection");
14536
14537        assert!(matches!(
14538            handler.refuse_to_end_module_connection_for_a_client(
14539                module_connection,
14540                78,
14541                ForwardingError::ConnectionClosing {
14542                    connection_id: module_connection,
14543                },
14544            ),
14545            Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
14546                connection_id
14547            })) if connection_id == module_connection
14548        ));
14549        assert!(matches!(
14550            handler.refuse_to_end_module_connection_for_a_client(
14551                module_connection,
14552                79,
14553                ForwardingError::Poisoned,
14554            ),
14555            Err(RouterError::Forwarding(ForwardingError::Poisoned))
14556        ));
14557        assert!(matches!(
14558            handler.refuse_to_end_module_connection_for_a_client(
14559                module_connection,
14560                80,
14561                ForwardingError::StaleModuleEndpoint,
14562            ),
14563            Err(RouterError::Forwarding(
14564                ForwardingError::StaleModuleEndpoint
14565            ))
14566        ));
14567    }
14568
14569    /// The spawn-attestation guard is what stops a connected module from claiming
14570    /// another module's identity and being stamped `Reserved` for it. Every other
14571    /// test that supplies a consumer_identity supplies a CORRECT one, because a
14572    /// correct one is what the rest of the flow needs -- so the guard's rejection
14573    /// branch was never the subject of an assertion, only its acceptance branch.
14574    ///
14575    /// Deleting the guard's EFFECT (granting Reserved unconditionally) leaves the
14576    /// whole subc-core library suite green; only the forwarding integration tests
14577    /// notice, and they notice for unrelated reasons. This test exists so the
14578    /// refusal itself is asserted where the guard lives: it fails if the identity
14579    /// check stops refusing, which is the direction that matters, since a guard
14580    /// that wrongly ACCEPTS is silent while one that wrongly REJECTS is loud.
14581    #[tokio::test]
14582    async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
14583        let registry = Arc::new(Registry::default());
14584        let forwarding = Arc::new(ForwardingTable::default());
14585        let supervisor = SupervisorHandle::new();
14586        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
14587        let handler =
14588            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14589                .with_supervisor(supervisor);
14590
14591        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
14592        hello_via_sink(
14593            &handler,
14594            &target_ctx,
14595            &mut target_rx,
14596            hello_frame("target", PROTOCOL_VERSION, 1),
14597        )
14598        .await;
14599
14600        // A real supervised module id presenting the wrong nonce. This is the
14601        // impersonation case: the attacker knows a privileged module_id, which is
14602        // public, and guesses at the nonce, which is not.
14603        let wrong_nonce = handler
14604            .handle_control_frame(
14605                &route_ctx(ConnectionId::new(91)).0,
14606                route_open_frame_with_admission_facts(
14607                    20,
14608                    "target",
14609                    unique_project_root("admission-facts"),
14610                    Some(subc_control::ConsumerIdentity {
14611                        module_id: "fed".to_string(),
14612                        launch_nonce: "not-the-real-nonce".to_string(),
14613                    }),
14614                    None,
14615                ),
14616            )
14617            .await
14618            .unwrap();
14619        assert_eq!(
14620            parse_error(&wrong_nonce[0])["code"],
14621            "bad_consumer_identity",
14622            "a mismatched launch nonce must be refused, not stamped Reserved"
14623        );
14624
14625        // A module id the supervisor never spawned at all, so no nonce exists to
14626        // compare against. An implementation that treats "no record" as "nothing
14627        // to check" fails open here while passing the case above.
14628        let never_spawned = handler
14629            .handle_control_frame(
14630                &route_ctx(ConnectionId::new(92)).0,
14631                route_open_frame_with_admission_facts(
14632                    21,
14633                    "target",
14634                    unique_project_root("admission-facts"),
14635                    Some(subc_control::ConsumerIdentity {
14636                        module_id: "never-spawned".to_string(),
14637                        launch_nonce: "any-nonce".to_string(),
14638                    }),
14639                    None,
14640                ),
14641            )
14642            .await
14643            .unwrap();
14644        assert_eq!(
14645            parse_error(&never_spawned[0])["code"],
14646            "bad_consumer_identity",
14647            "an unspawned module_id must be refused rather than accepted for lack of a record"
14648        );
14649    }
14650
14651    /// The refusal test above proves the guard says NO. Nothing proved it can say
14652    /// YES, and the difference is not academic: replacing the whole authorization
14653    /// with `false` -- admitting no consumer identity at all, revoking Reserved
14654    /// standing for every supervised module in the fleet -- leaves 110 of the 111
14655    /// library tests GREEN. The one that notices does so by HANGING, because it
14656    /// waits for a bind that can no longer happen.
14657    ///
14658    /// A hang is the weakest signal a suite can produce. In CI it reads as a slow
14659    /// or flaky test, invites a RETRY rather than an investigation, and the retry
14660    /// hangs too and gets blamed on the runner. So a total revocation of the
14661    /// daemon's trust grant would have shipped behind a symptom nobody attributes
14662    /// to code.
14663    ///
14664    /// The bias is structural rather than accidental. A REFUSAL looks like a
14665    /// failure someone writes a test for; a GRANT looks like the happy path. Every
14666    /// binary-outcome guard whose STRICTNESS is the point acquires a refusal-heavy
14667    /// suite for that reason, and this one is the purest case in the daemon.
14668    ///
14669    /// This test asserts the EFFECT rather than the absence of an error: the module
14670    /// receives a RouteBind and it carries `Reserved` naming the attested module.
14671    /// A guard that admitted nobody would produce no bind at all; one that admitted
14672    /// everybody would stamp the wrong principal, which the refusal test catches.
14673    #[tokio::test]
14674    async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
14675        let registry = Arc::new(Registry::default());
14676        let forwarding = Arc::new(ForwardingTable::default());
14677        let supervisor = SupervisorHandle::new();
14678        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
14679        let handler =
14680            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14681                .with_supervisor(supervisor);
14682
14683        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
14684        hello_via_sink(
14685            &handler,
14686            &target_ctx,
14687            &mut target_rx,
14688            hello_frame("target", PROTOCOL_VERSION, 1),
14689        )
14690        .await;
14691
14692        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
14693        let route_handler = handler.clone();
14694        let route_task = tokio::spawn(async move {
14695            route_handler
14696                .handle_control_frame(
14697                    &client_ctx,
14698                    route_open_frame_with_admission_facts(
14699                        30,
14700                        "target",
14701                        unique_project_root("admission-facts"),
14702                        Some(subc_control::ConsumerIdentity {
14703                            module_id: "fed".to_string(),
14704                            launch_nonce: "fed-nonce".to_string(),
14705                        }),
14706                        None,
14707                    ),
14708                )
14709                .await
14710                .unwrap()
14711        });
14712
14713        // BOUND THE WAIT. The first version of this test recv'd unbounded, and under
14714        // the very mutation it exists to catch -- a guard that admits nobody -- no
14715        // bind is ever sent, so it HUNG rather than failing. That reproduces the
14716        // exact defect being fixed: a total revocation detected only as a stalled
14717        // suite, which reads as flakiness and invites a retry. An acceptance test
14718        // that waits for an effect must bound the wait, or a red becomes a hang.
14719        let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14720            .await
14721            .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
14722            .expect("module control channel closed before route.bind");
14723        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
14724        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
14725            panic!("expected route.bind")
14726        };
14727        assert_eq!(
14728            principal,
14729            Some(Principal::Reserved {
14730                module_id: "fed".to_string()
14731            }),
14732            "a correctly attested consumer must be stamped Reserved for its own id"
14733        );
14734
14735        handler
14736            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
14737            .await
14738            .unwrap();
14739        assert!(route_task.await.unwrap().is_empty());
14740        assert!(
14741            matches!(
14742                serde_json::from_slice::<ClientControlResponse>(
14743                    &client_rx.recv().await.unwrap().body
14744                )
14745                .unwrap(),
14746                ClientControlResponse::RouteOpen { .. }
14747            ),
14748            "the route must actually open, not merely avoid an error"
14749        );
14750    }
14751
14752    #[tokio::test(start_paused = true)]
14753    async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
14754        let registry = Arc::new(Registry::default());
14755        let forwarding = Arc::new(ForwardingTable::default());
14756        let supervisor = SupervisorHandle::new();
14757        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
14758        let handler =
14759            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14760                .with_supervisor(supervisor);
14761
14762        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
14763        hello_via_sink(
14764            &handler,
14765            &target_ctx,
14766            &mut target_rx,
14767            hello_frame("target", PROTOCOL_VERSION, 1),
14768        )
14769        .await;
14770
14771        let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
14772        let direct_handler = handler.clone();
14773        let direct_open = tokio::spawn(async move {
14774            direct_handler
14775                .handle_control_frame(
14776                    &direct_ctx,
14777                    route_open_frame(2, "target", unique_project_root("route-census-direct")),
14778                )
14779                .await
14780                .unwrap()
14781        });
14782        let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14783            .await
14784            .expect("no direct route.bind within 5s")
14785            .expect("target control channel closed before direct route.bind");
14786        handler
14787            .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
14788            .await
14789            .unwrap();
14790        assert!(direct_open.await.unwrap().is_empty());
14791        let _ = direct_rx.recv().await.unwrap();
14792
14793        let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
14794        let reserved_handler = handler.clone();
14795        let reserved_open = tokio::spawn(async move {
14796            reserved_handler
14797                .handle_control_frame(
14798                    &reserved_ctx,
14799                    route_open_frame_with_admission_facts(
14800                        3,
14801                        "target",
14802                        unique_project_root("admission-facts"),
14803                        Some(ConsumerIdentity {
14804                            module_id: "fed".to_string(),
14805                            launch_nonce: "fed-nonce".to_string(),
14806                        }),
14807                        None,
14808                    ),
14809                )
14810                .await
14811                .unwrap()
14812        });
14813        let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14814            .await
14815            .expect("no reserved route.bind within 5s")
14816            .expect("target control channel closed before reserved route.bind");
14817        handler
14818            .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
14819            .await
14820            .unwrap();
14821        assert!(reserved_open.await.unwrap().is_empty());
14822        let _ = reserved_rx.recv().await.unwrap();
14823
14824        forwarding
14825            .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
14826            .unwrap();
14827        let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
14828        let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
14829            module_id: Some("target".to_string()),
14830        })
14831        .unwrap();
14832        let census_frame =
14833            Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
14834        let response = handler
14835            .handle_control_frame(&census_ctx, census_frame)
14836            .await
14837            .unwrap()
14838            .pop()
14839            .unwrap();
14840        let actual: Value = serde_json::from_slice(&response.body).unwrap();
14841        let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
14842        assert!(matches!(
14843            decoded,
14844            ClientControlResponse::SupervisorRoutes { .. }
14845        ));
14846        let routes = actual["modules"][0]["routes"].as_array().unwrap();
14847        assert_eq!(routes.len(), 2);
14848        assert!(routes.iter().all(|route| route["draining"] == true));
14849        // The census carries WHY: the reason the drain was begun with, in the
14850        // route.closing vocabulary, on every draining route this drain marked.
14851        assert!(
14852            routes.iter().all(|route| route["drain_reason"] == "reload"),
14853            "draining routes must name the drain's reason: {routes:?}"
14854        );
14855        assert!(routes.iter().any(|route| {
14856            route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
14857        }));
14858        assert!(routes.iter().any(|route| {
14859            route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
14860        }));
14861
14862        let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
14863            .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
14864        if std::env::var_os("UPDATE_GOLDEN").is_some() {
14865            std::fs::write(
14866                &golden_path,
14867                format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
14868            )
14869            .unwrap();
14870        }
14871        let expected: Value =
14872            serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
14873        assert_eq!(actual, expected);
14874    }
14875
14876    async fn query_live_roots(
14877        handler: &ControlHandler,
14878        module_ctx: &RouteCtx,
14879    ) -> ModuleControlResponseToModule {
14880        let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
14881        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
14882        let response = handler
14883            .handle_control_frame(module_ctx, frame)
14884            .await
14885            .unwrap()
14886            .pop()
14887            .unwrap();
14888        serde_json::from_slice(&response.body).unwrap()
14889    }
14890
14891    #[tokio::test(start_paused = true)]
14892    async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
14893        let registry = Arc::new(Registry::default());
14894        let forwarding = Arc::new(ForwardingTable::default());
14895        let handler = ControlHandler::with_forwarding(registry, forwarding);
14896        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
14897        hello_via_sink(
14898            &handler,
14899            &target_ctx,
14900            &mut target_rx,
14901            hello_frame("target", PROTOCOL_VERSION, 1),
14902        )
14903        .await;
14904        let root = unique_project_root("live-roots-known");
14905        let path = ProjectRootId::from_path_allowing_missing(root.path())
14906            .unwrap()
14907            .as_path()
14908            .to_path_buf();
14909        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
14910        let open_handler = handler.clone();
14911        let opened = tokio::spawn(async move {
14912            open_handler
14913                .handle_control_frame(&client_ctx, legacy_route_open_frame(2, "target", root))
14914                .await
14915                .unwrap()
14916        });
14917        let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14918            .await
14919            .unwrap()
14920            .unwrap();
14921        handler
14922            .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
14923            .await
14924            .unwrap();
14925        assert!(opened.await.unwrap().is_empty());
14926        let _ = client_rx.recv().await.unwrap();
14927
14928        let root = unique_project_root("live-roots-pending");
14929        let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
14930            .unwrap()
14931            .as_path()
14932            .to_path_buf();
14933        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
14934        let open_handler = handler.clone();
14935        let pending = tokio::spawn(async move {
14936            open_handler
14937                .handle_control_frame(&client_ctx, legacy_route_open_frame(3, "target", root))
14938                .await
14939                .unwrap()
14940        });
14941        let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14942            .await
14943            .unwrap()
14944            .unwrap();
14945        let actual = query_live_roots(&handler, &target_ctx).await;
14946        let ModuleControlResponseToModule::LiveRoots {
14947            roots,
14948            unknown_root_bindings,
14949            total_bindings,
14950        } = actual
14951        else {
14952            panic!("expected live roots")
14953        };
14954        assert_eq!(total_bindings, 2, "root-known arm must count live routes");
14955        assert_eq!(unknown_root_bindings, 0);
14956        assert_eq!(
14957            roots.len(),
14958            2,
14959            "root-known arm must retain each canonical root"
14960        );
14961        assert_eq!(
14962            total_bindings,
14963            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
14964        );
14965        let counts = roots
14966            .iter()
14967            .map(|root| (root.project_root.clone(), root.bound, root.pending))
14968            .collect::<Vec<_>>();
14969        let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
14970        expected.sort_by(|a, b| a.0.cmp(&b.0));
14971        assert_eq!(
14972            counts, expected,
14973            "roots must sort by path and count pending separately"
14974        );
14975        handler
14976            .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
14977            .await
14978            .unwrap();
14979        assert!(pending.await.unwrap().is_empty());
14980    }
14981
14982    #[tokio::test(start_paused = true)]
14983    async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
14984        let forwarding = Arc::new(ForwardingTable::default());
14985        let handler =
14986            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
14987        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
14988        hello_via_sink(
14989            &handler,
14990            &target_ctx,
14991            &mut target_rx,
14992            hello_frame("target", PROTOCOL_VERSION, 1),
14993        )
14994        .await;
14995        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
14996        let pending = forwarding
14997            .begin_route_bind_relay_for_test(
14998                client_ctx.connection_id,
14999                client_ctx.egress.clone(),
15000                2,
15001                "target",
15002            )
15003            .unwrap();
15004        forwarding
15005            .complete_pending_relay(
15006                target_ctx.connection_id,
15007                pending.corr,
15008                RouteBindRelayOutcome::Accepted,
15009            )
15010            .unwrap();
15011        let actual = query_live_roots(&handler, &target_ctx).await;
15012        let ModuleControlResponseToModule::LiveRoots {
15013            roots,
15014            unknown_root_bindings,
15015            total_bindings,
15016        } = actual
15017        else {
15018            panic!("expected live roots")
15019        };
15020        assert!(roots.is_empty(), "unknown-root arm must not invent a root");
15021        assert_eq!(
15022            unknown_root_bindings, 1,
15023            "unknown-root arm must not read as no bindings"
15024        );
15025        assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
15026        assert_eq!(
15027            total_bindings,
15028            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
15029        );
15030    }
15031
15032    /// A module reads HELLO_ACK as its first frame and exits on anything else,
15033    /// so the ack has to be on its outbound queue before the module is
15034    /// routable. The connection loop writes a handler's replies only after the
15035    /// handler returns; this test stops in exactly that gap, runs a real
15036    /// route.open from another connection, and only then writes whatever the
15037    /// HELLO handler returned, the way the loop would. If the ack were still a
15038    /// reply, the route.bind request would reach the module first.
15039    #[tokio::test(start_paused = true)]
15040    async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
15041        let forwarding = Arc::new(ForwardingTable::default());
15042        let handler =
15043            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
15044        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
15045        let replies = handler
15046            .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
15047            .await
15048            .unwrap();
15049        let queued_by_hello = module_rx.len();
15050
15051        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
15052        let open_handler = handler.clone();
15053        let open = tokio::spawn(async move {
15054            open_handler
15055                .handle_control_frame(
15056                    &client_ctx,
15057                    route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
15058                )
15059                .await
15060                .unwrap()
15061        });
15062        // Let the route.open run until its route.bind is on the module's queue.
15063        let mut spins = 0;
15064        while module_rx.len() == queued_by_hello {
15065            spins += 1;
15066            assert!(spins < 10_000, "route.open never queued a route.bind");
15067            tokio::task::yield_now().await;
15068        }
15069
15070        // Now the connection loop's half: write the HELLO handler's replies.
15071        for reply in replies {
15072            module_ctx.egress.send(reply).await.unwrap();
15073        }
15074
15075        let first = module_rx.recv().await.unwrap().frame;
15076        assert_eq!(
15077            first.header.ty,
15078            FrameType::HelloAck,
15079            "the first frame a registering module reads must be its HELLO_ACK"
15080        );
15081        assert_eq!(first.header.corr, 7);
15082        let second = module_rx.recv().await.unwrap().frame;
15083        assert_eq!(second.header.ty, FrameType::Request);
15084        assert!(
15085            matches!(
15086                serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
15087                ModuleControlRequest::RouteBind { .. }
15088            ),
15089            "the route.bind follows the ack"
15090        );
15091        assert!(module_rx.try_recv().is_err(), "nothing else was queued");
15092
15093        handler
15094            .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
15095            .await
15096            .unwrap();
15097        assert!(open.await.unwrap().is_empty());
15098        let _ = client_rx.recv().await.unwrap();
15099    }
15100
15101    #[tokio::test(start_paused = true)]
15102    async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
15103        let handler = ControlHandler::with_forwarding(
15104            Arc::new(Registry::default()),
15105            Arc::new(ForwardingTable::default()),
15106        );
15107        let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
15108        let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
15109        hello_via_sink(
15110            &handler,
15111            &first_ctx,
15112            &mut first_rx,
15113            hello_frame("first", PROTOCOL_VERSION, 1),
15114        )
15115        .await;
15116        hello_via_sink(
15117            &handler,
15118            &second_ctx,
15119            &mut second_rx,
15120            hello_frame("second", PROTOCOL_VERSION, 2),
15121        )
15122        .await;
15123        let root = unique_project_root("second-only");
15124        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
15125        let cloned = handler.clone();
15126        let open = tokio::spawn(async move {
15127            cloned
15128                .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
15129                .await
15130                .unwrap()
15131        });
15132        let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
15133            .await
15134            .unwrap()
15135            .unwrap();
15136        let first = query_live_roots(&handler, &first_ctx).await;
15137        let second = query_live_roots(&handler, &second_ctx).await;
15138        assert!(
15139            matches!(
15140                first,
15141                ModuleControlResponseToModule::LiveRoots {
15142                    total_bindings: 0,
15143                    ..
15144                }
15145            ),
15146            "cross-module scope must not expose another module's roots"
15147        );
15148        assert!(
15149            matches!(
15150                second,
15151                ModuleControlResponseToModule::LiveRoots {
15152                    total_bindings: 1,
15153                    ..
15154                }
15155            ),
15156            "second module must see its pending route"
15157        );
15158        handler
15159            .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
15160            .await
15161            .unwrap();
15162        assert!(open.await.unwrap().is_empty());
15163    }
15164
15165    #[tokio::test(start_paused = true)]
15166    async fn supervisor_live_roots_no_bindings_arm_is_empty() {
15167        let handler = ControlHandler::with_forwarding(
15168            Arc::new(Registry::default()),
15169            Arc::new(ForwardingTable::default()),
15170        );
15171        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
15172        hello_via_sink(
15173            &handler,
15174            &target_ctx,
15175            &mut target_rx,
15176            hello_frame("target", PROTOCOL_VERSION, 1),
15177        )
15178        .await;
15179        let actual = query_live_roots(&handler, &target_ctx).await;
15180        let ModuleControlResponseToModule::LiveRoots {
15181            roots,
15182            unknown_root_bindings,
15183            total_bindings,
15184        } = actual
15185        else {
15186            panic!("expected live roots")
15187        };
15188        assert!(roots.is_empty());
15189        assert_eq!(unknown_root_bindings, 0);
15190        assert_eq!(total_bindings, 0);
15191        assert_eq!(
15192            total_bindings,
15193            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
15194        );
15195    }
15196
15197    /// Read the vendored fed corpus rather than hand-building a package.
15198    ///
15199    /// A hand-built object encodes what the test author believed the carrier
15200    /// emits. These vectors are what it actually emits, and one of them exists
15201    /// specifically to pin OUR side of the seam: its note reads "SUBC relay
15202    /// ignores additive unknown fields at the traversal emit terminus."
15203    fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
15204        let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
15205            .join("tests/fixtures/fed/admission-facts-emit.jsonl");
15206        let text = std::fs::read_to_string(&path)
15207            .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
15208        let vectors: Vec<(String, Value)> = text
15209            .lines()
15210            .filter(|line| !line.trim().is_empty())
15211            .map(|line| {
15212                let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
15213                let id = entry["corpus_id"]
15214                    .as_str()
15215                    .expect("every vector carries a corpus_id")
15216                    .to_string();
15217                (id, entry["package"].clone())
15218            })
15219            .collect();
15220        // Pin the count: a corpus that silently shrinks would take its coverage
15221        // with it, and a suite reading N-1 vectors reports the same clean pass
15222        // as one reading N.
15223        assert_eq!(
15224            vectors.len(),
15225            3,
15226            "vendored fed corpus changed size; re-sync from subc-federation"
15227        );
15228
15229        // Pin what makes the corpus DISCRIMINATING, not just present.
15230        //
15231        // The relay test below takes its expected value from the corpus, so the
15232        // corpus supplies the test's power to detect a lossy relay rather than
15233        // its correctness. A relay that dropped unrecognised fields would still
15234        // be caught -- but only by a package carrying fields it does not know.
15235        // Shrink every package to the handful of keys any implementation would
15236        // recognise and the test keeps passing over an input that can no longer
15237        // fail, which is the same clean green as a corpus that shrank away.
15238        //
15239        // So assert the precondition rather than duplicating the packages here:
15240        // at least one vector must carry a field beyond the small common set.
15241        // That is one claim to maintain instead of nine, and it fails loudly if
15242        // a re-sync ever flattens the corpus.
15243        const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
15244        let richest = vectors
15245            .iter()
15246            .filter_map(|(_, package)| package.as_object())
15247            .map(|object| {
15248                object
15249                    .keys()
15250                    .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
15251                    .count()
15252            })
15253            .max()
15254            .unwrap_or(0);
15255        assert!(
15256            richest >= 2,
15257            "vendored corpus no longer carries a package with unmodelled fields, \
15258             so the relay test can no longer distinguish a verbatim relay from a lossy one"
15259        );
15260
15261        vectors
15262    }
15263
15264    /// The relay must carry the carrier's package through BYTE-FOR-BYTE.
15265    ///
15266    /// The gate test below proves the ACCESS RULE (who may send facts, to whom).
15267    /// This proves the PAYLOAD RULE, which the gate cannot: it hand-builds a
15268    /// three-key object, so a relay that quietly dropped fields it did not
15269    /// recognise would satisfy it. These vectors carry nine keys including ones
15270    /// this crate has no type for, so a typed relay fails here and only here.
15271    #[tokio::test]
15272    async fn admission_facts_relay_carries_vendored_packages_verbatim() {
15273        for (corpus_id, package) in fed_admission_facts_vectors() {
15274            let registry = Arc::new(Registry::default());
15275            let forwarding = Arc::new(ForwardingTable::default());
15276            let supervisor = SupervisorHandle::new();
15277            supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
15278            let handler =
15279                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15280                    .with_supervisor(supervisor)
15281                    .with_admission_facts_config(
15282                        Some("fed".to_string()),
15283                        Some(vec!["target".to_string()]),
15284                    );
15285
15286            let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
15287            hello_via_sink(
15288                &handler,
15289                &target_ctx,
15290                &mut target_rx,
15291                hello_frame("target", PROTOCOL_VERSION, 1),
15292            )
15293            .await;
15294
15295            let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
15296            let route_handler = handler.clone();
15297            let expected = package.clone();
15298            let route_task = tokio::spawn(async move {
15299                route_handler
15300                    .handle_control_frame(
15301                        &client_ctx,
15302                        route_open_frame_with_admission_facts(
15303                            20,
15304                            "target",
15305                            unique_project_root("admission-facts"),
15306                            Some(subc_control::ConsumerIdentity {
15307                                module_id: "fed".to_string(),
15308                                launch_nonce: "fed-nonce".to_string(),
15309                            }),
15310                            Some(package),
15311                        ),
15312                    )
15313                    .await
15314                    .unwrap()
15315            });
15316
15317            let bind_frame = target_rx.recv().await.unwrap();
15318            let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15319            let ModuleControlRequest::RouteBind {
15320                admission_facts, ..
15321            } = bind
15322            else {
15323                panic!("{corpus_id}: expected route.bind")
15324            };
15325            assert_eq!(
15326                admission_facts,
15327                Some(expected),
15328                "{corpus_id}: relay must not add, drop or reshape any field"
15329            );
15330
15331            handler
15332                .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
15333                .await
15334                .unwrap();
15335            route_task.await.unwrap();
15336        }
15337    }
15338
15339    #[tokio::test]
15340    async fn route_bind_never_relays_the_openers_build_class() {
15341        let registry = Arc::new(Registry::default());
15342        let forwarding = Arc::new(ForwardingTable::default());
15343        let handler =
15344            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
15345        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(80));
15346        hello_via_sink(
15347            &handler,
15348            &target_ctx,
15349            &mut target_rx,
15350            hello_frame("target", PROTOCOL_VERSION, 1),
15351        )
15352        .await;
15353
15354        let mut identity = BindIdentity::new("unit".to_string(), "s".to_string());
15355        identity.build_class = Some(subc_protocol::BUILD_CLASS_PRODUCTION.to_string());
15356        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
15357            busy_refusals: None,
15358            target: RouteTarget::ToolProvider {
15359                module_id: "target".to_string(),
15360            },
15361            identity,
15362            consumer_identity: None,
15363            consumer_capabilities: None,
15364            role_versions: None,
15365            admission_facts: None,
15366            scope: None,
15367        })
15368        .unwrap();
15369        let open = Frame::build(FrameType::Request, control_flags(), 0, 0, 30, body).unwrap();
15370
15371        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(81));
15372        let route_handler = handler.clone();
15373        let route_task = tokio::spawn(async move {
15374            route_handler
15375                .handle_control_frame(&client_ctx, open)
15376                .await
15377                .unwrap()
15378        });
15379        let bind_frame = target_rx.recv().await.unwrap();
15380        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15381        let ModuleControlRequest::RouteBind { identity, .. } = bind else {
15382            panic!("expected route.bind")
15383        };
15384        assert_eq!(
15385            identity.build_class, None,
15386            "route.bind carried the build class"
15387        );
15388        assert!(
15389            !String::from_utf8_lossy(&bind_frame.body).contains("build_class"),
15390            "the route.bind bytes still name build_class"
15391        );
15392        handler
15393            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
15394            .await
15395            .unwrap();
15396        route_task.await.unwrap();
15397    }
15398
15399    #[tokio::test]
15400    async fn admission_facts_gate_checks_carrier_target_and_precedence() {
15401        let registry = Arc::new(Registry::default());
15402        let forwarding = Arc::new(ForwardingTable::default());
15403        let supervisor = SupervisorHandle::new();
15404        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
15405        supervisor.set_spawn_nonce("other", "other-nonce".to_string());
15406        let handler =
15407            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15408                .with_supervisor(supervisor)
15409                .with_admission_facts_config(
15410                    Some("fed".to_string()),
15411                    Some(vec!["target".to_string()]),
15412                );
15413
15414        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
15415        hello_via_sink(
15416            &handler,
15417            &target_ctx,
15418            &mut target_rx,
15419            hello_frame("target", PROTOCOL_VERSION, 1),
15420        )
15421        .await;
15422        let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
15423        hello_via_sink(
15424            &handler,
15425            &other_ctx,
15426            &mut other_rx,
15427            hello_frame("other", PROTOCOL_VERSION, 2),
15428        )
15429        .await;
15430
15431        let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
15432        let expected_facts = facts.clone();
15433        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
15434        let route_handler = handler.clone();
15435        let route_task = tokio::spawn(async move {
15436            route_handler
15437                .handle_control_frame(
15438                    &client_ctx,
15439                    route_open_frame_with_admission_facts(
15440                        10,
15441                        "target",
15442                        unique_project_root("admission-facts"),
15443                        Some(subc_control::ConsumerIdentity {
15444                            module_id: "fed".to_string(),
15445                            launch_nonce: "fed-nonce".to_string(),
15446                        }),
15447                        Some(facts.clone()),
15448                    ),
15449                )
15450                .await
15451                .unwrap()
15452        });
15453        let bind_frame = target_rx.recv().await.unwrap();
15454        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15455        let ModuleControlRequest::RouteBind {
15456            admission_facts, ..
15457        } = bind
15458        else {
15459            panic!("expected route.bind")
15460        };
15461        assert_eq!(admission_facts, Some(expected_facts));
15462        handler
15463            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
15464            .await
15465            .unwrap();
15466        assert!(route_task.await.unwrap().is_empty());
15467        assert!(matches!(
15468            serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
15469                .unwrap(),
15470            ClientControlResponse::RouteOpen { .. }
15471        ));
15472
15473        let direct = handler
15474            .handle_control_frame(
15475                &route_ctx(ConnectionId::new(73)).0,
15476                route_open_frame_with_admission_facts(
15477                    11,
15478                    "target",
15479                    unique_project_root("admission-facts"),
15480                    None,
15481                    Some(json!({"x": 1})),
15482                ),
15483            )
15484            .await
15485            .unwrap();
15486        assert_eq!(
15487            parse_error(&direct[0])["code"],
15488            "admission_facts_not_permitted"
15489        );
15490
15491        let different_reserved = handler
15492            .handle_control_frame(
15493                &route_ctx(ConnectionId::new(77)).0,
15494                route_open_frame_with_admission_facts(
15495                    15,
15496                    "target",
15497                    unique_project_root("admission-facts"),
15498                    Some(subc_control::ConsumerIdentity {
15499                        module_id: "other".to_string(),
15500                        launch_nonce: "other-nonce".to_string(),
15501                    }),
15502                    Some(json!({"x": 1})),
15503                ),
15504            )
15505            .await
15506            .unwrap();
15507        assert_eq!(
15508            parse_error(&different_reserved[0])["code"],
15509            "admission_facts_not_permitted"
15510        );
15511
15512        let other_target = handler
15513            .handle_control_frame(
15514                &route_ctx(ConnectionId::new(74)).0,
15515                route_open_frame_with_admission_facts(
15516                    12,
15517                    "other",
15518                    unique_project_root("admission-facts"),
15519                    Some(subc_control::ConsumerIdentity {
15520                        module_id: "fed".to_string(),
15521                        launch_nonce: "fed-nonce".to_string(),
15522                    }),
15523                    Some(json!({"x": 1})),
15524                ),
15525            )
15526            .await
15527            .unwrap();
15528        assert_eq!(
15529            parse_error(&other_target[0])["code"],
15530            "admission_facts_target_not_allowed"
15531        );
15532
15533        let nonexistent = handler
15534            .handle_control_frame(
15535                &route_ctx(ConnectionId::new(75)).0,
15536                route_open_frame_with_admission_facts(
15537                    13,
15538                    "missing",
15539                    unique_project_root("admission-facts"),
15540                    None,
15541                    Some(json!({"x": 1})),
15542                ),
15543            )
15544            .await
15545            .unwrap();
15546        assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
15547
15548        let described = handler
15549            .handle_control_frame(
15550                &route_ctx(ConnectionId::new(76)).0,
15551                Frame::build(
15552                    FrameType::Request,
15553                    control_flags(),
15554                    0,
15555                    0,
15556                    14,
15557                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
15558                )
15559                .unwrap(),
15560            )
15561            .await
15562            .unwrap();
15563        let ClientControlResponse::ServerDescribe { capabilities, .. } =
15564            serde_json::from_slice(&described[0].body).unwrap()
15565        else {
15566            panic!("expected server.describe response")
15567        };
15568        assert!(capabilities
15569            .iter()
15570            .any(|cap| cap == "admission_facts_relay_v1"));
15571    }
15572
15573    #[tokio::test]
15574    async fn admission_facts_without_configured_carrier_are_rejected() {
15575        let registry = Arc::new(Registry::default());
15576        let forwarding = Arc::new(ForwardingTable::default());
15577        let handler = ControlHandler::with_forwarding(registry, forwarding);
15578        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
15579        hello_via_sink(
15580            &handler,
15581            &target_ctx,
15582            &mut target_rx,
15583            hello_frame("target", PROTOCOL_VERSION, 1),
15584        )
15585        .await;
15586
15587        let responses = handler
15588            .handle_control_frame(
15589                &route_ctx(ConnectionId::new(79)).0,
15590                route_open_frame_with_admission_facts(
15591                    16,
15592                    "target",
15593                    unique_project_root("admission-facts"),
15594                    None,
15595                    Some(json!({"x": 1})),
15596                ),
15597            )
15598            .await
15599            .unwrap();
15600        assert_eq!(
15601            parse_error(&responses[0])["code"],
15602            "admission_facts_not_permitted"
15603        );
15604    }
15605
15606    #[tokio::test]
15607    async fn route_open_relays_consumer_capabilities_verbatim() {
15608        let registry = Arc::new(Registry::default());
15609        let forwarding = Arc::new(ForwardingTable::default());
15610        let handler =
15611            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
15612        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
15613        hello_via_sink(
15614            &handler,
15615            &module_ctx,
15616            &mut module_rx,
15617            hello_frame("aft", PROTOCOL_VERSION, 7),
15618        )
15619        .await;
15620
15621        let expected = vec!["elicitation".to_string(), "roots".to_string()];
15622        let expected_for_request = expected.clone();
15623        let project_root = unique_project_root("consumer-capabilities-present");
15624        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
15625        let route_handler = handler.clone();
15626        let route_task = tokio::spawn(async move {
15627            route_handler
15628                .handle_control_frame(
15629                    &client_ctx,
15630                    route_open_frame_with_consumer_capabilities(
15631                        401,
15632                        "aft",
15633                        project_root,
15634                        Some(expected_for_request),
15635                    ),
15636                )
15637                .await
15638                .unwrap()
15639        });
15640        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
15641            .await
15642            .unwrap()
15643            .unwrap();
15644        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15645        let ModuleControlRequest::RouteBind {
15646            consumer_capabilities,
15647            ..
15648        } = bind
15649        else {
15650            panic!("expected route.bind request, got {bind:?}");
15651        };
15652        assert_eq!(consumer_capabilities, Some(expected.clone()));
15653
15654        handler
15655            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
15656            .await
15657            .unwrap();
15658        let route_response = route_task.await.unwrap();
15659        assert!(route_response.is_empty());
15660        let published = client_rx.recv().await.unwrap();
15661        assert!(matches!(
15662            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
15663            ClientControlResponse::RouteOpen { .. }
15664        ));
15665    }
15666
15667    #[tokio::test]
15668    async fn route_open_without_consumer_capabilities_relays_none() {
15669        let registry = Arc::new(Registry::default());
15670        let forwarding = Arc::new(ForwardingTable::default());
15671        let handler =
15672            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
15673        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
15674        hello_via_sink(
15675            &handler,
15676            &module_ctx,
15677            &mut module_rx,
15678            hello_frame("aft", PROTOCOL_VERSION, 7),
15679        )
15680        .await;
15681
15682        let project_root = unique_project_root("consumer-capabilities-absent");
15683        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
15684        let route_handler = handler.clone();
15685        let route_task = tokio::spawn(async move {
15686            route_handler
15687                .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
15688                .await
15689                .unwrap()
15690        });
15691        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
15692            .await
15693            .unwrap()
15694            .unwrap();
15695        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15696        let ModuleControlRequest::RouteBind {
15697            consumer_capabilities,
15698            ..
15699        } = bind
15700        else {
15701            panic!("expected route.bind request, got {bind:?}");
15702        };
15703        assert_eq!(consumer_capabilities, None);
15704
15705        handler
15706            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
15707            .await
15708            .unwrap();
15709        let route_response = route_task.await.unwrap();
15710        assert!(route_response.is_empty());
15711        let published = client_rx.recv().await.unwrap();
15712        assert!(matches!(
15713            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
15714            ClientControlResponse::RouteOpen { .. }
15715        ));
15716    }
15717
15718    /// Opens a route to a freshly registered `aft` with `sent` as the
15719    /// route.open's role_versions, acks the bind, and returns the role_versions
15720    /// the module's bind carried.
15721    async fn bind_role_versions_for(
15722        sent: Option<BTreeMap<String, String>>,
15723        connection: u64,
15724    ) -> Option<BTreeMap<String, String>> {
15725        let registry = Arc::new(Registry::default());
15726        let forwarding = Arc::new(ForwardingTable::default());
15727        let handler =
15728            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
15729        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
15730        hello_via_sink(
15731            &handler,
15732            &module_ctx,
15733            &mut module_rx,
15734            hello_frame("aft", PROTOCOL_VERSION, 7),
15735        )
15736        .await;
15737        let project_root = unique_project_root("role-versions");
15738        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
15739        let route_handler = handler.clone();
15740        let route_task = tokio::spawn(async move {
15741            route_handler
15742                .handle_control_frame(
15743                    &client_ctx,
15744                    route_open_frame_with_role_versions(403, "aft", project_root, sent),
15745                )
15746                .await
15747                .unwrap()
15748        });
15749        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
15750            .await
15751            .expect("a well-formed route.open reaches the module as a bind")
15752            .unwrap();
15753        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15754        let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
15755            panic!("expected route.bind request, got {bind:?}");
15756        };
15757        handler
15758            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
15759            .await
15760            .unwrap();
15761        assert!(route_task.await.unwrap().is_empty());
15762        let published = client_rx.recv().await.unwrap();
15763        assert!(matches!(
15764            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
15765            ClientControlResponse::RouteOpen { .. }
15766        ));
15767        role_versions
15768    }
15769
15770    #[tokio::test]
15771    async fn route_open_relays_role_versions_verbatim() {
15772        let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
15773        assert_eq!(
15774            bind_role_versions_for(Some(sent.clone()), 141).await,
15775            Some(sent)
15776        );
15777    }
15778
15779    /// An empty map declares nothing, so the provider sees no field rather
15780    /// than an empty object it would have to treat as a second "none".
15781    #[tokio::test]
15782    async fn route_open_with_empty_or_absent_role_versions_relays_none() {
15783        assert_eq!(bind_role_versions_for(None, 143).await, None);
15784        assert_eq!(
15785            bind_role_versions_for(Some(BTreeMap::new()), 145).await,
15786            None
15787        );
15788    }
15789
15790    #[tokio::test]
15791    async fn route_open_refuses_malformed_role_versions_before_any_bind() {
15792        let registry = Arc::new(Registry::default());
15793        let forwarding = Arc::new(ForwardingTable::default());
15794        let handler =
15795            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
15796        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
15797        hello_via_sink(
15798            &handler,
15799            &module_ctx,
15800            &mut module_rx,
15801            hello_frame("aft", PROTOCOL_VERSION, 7),
15802        )
15803        .await;
15804
15805        let nine: BTreeMap<String, String> = (0..9)
15806            .map(|index| (format!("role-{index}"), "v1".to_string()))
15807            .collect();
15808        for (label, malformed) in [
15809            (
15810                "invalid role name",
15811                role_versions(&[("Tool_Provider", "v1")]),
15812            ),
15813            ("invalid version", role_versions(&[("tool-provider", "v0")])),
15814            ("nine entries", nine),
15815        ] {
15816            // A refused open answers at once; one that reached the module
15817            // would wait for its bind ack and trip this timeout.
15818            let responses = tokio::time::timeout(
15819                Duration::from_secs(1),
15820                handler.handle_control_frame(
15821                    &route_ctx(ConnectionId::new(148)).0,
15822                    route_open_frame_with_role_versions(
15823                        404,
15824                        "aft",
15825                        unique_project_root("role-versions-malformed"),
15826                        Some(malformed),
15827                    ),
15828                ),
15829            )
15830            .await
15831            .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
15832            .unwrap();
15833            assert_eq!(responses.len(), 1, "{label}");
15834            assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
15835            let error = parse_error(&responses[0]);
15836            assert_eq!(error["code"], "invalid_request", "{label}: {error}");
15837            assert_eq!(
15838                error["detail"]["field"], "role_versions",
15839                "{label}: {error}"
15840            );
15841            assert!(
15842                !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
15843                "{label}: a malformed declaration is terminal"
15844            );
15845            assert!(
15846                module_rx.try_recv().is_err(),
15847                "{label}: the module must never see a bind"
15848            );
15849        }
15850    }
15851
15852    /// `route-role-versions/v1` is in HELLO_ACK and `server.describe`, so a
15853    /// consumer can tell this daemon forwards the field from one that would
15854    /// drop it.
15855    #[tokio::test]
15856    async fn route_role_versions_capability_is_advertised() {
15857        let handler = ControlHandler::new(Arc::new(Registry::default()));
15858        let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
15859        let ack = hello_via_sink(
15860            &handler,
15861            &ctx,
15862            &mut rx,
15863            hello_frame("m", PROTOCOL_VERSION, 1),
15864        )
15865        .await;
15866        let ack = parse_ack(&ack);
15867        assert!(
15868            ack.subc_capabilities
15869                .iter()
15870                .any(|c| c == "route-role-versions/v1"),
15871            "{:?}",
15872            ack.subc_capabilities
15873        );
15874
15875        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15876        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15877        let reply = handler
15878            .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
15879            .await
15880            .unwrap()
15881            .pop()
15882            .unwrap();
15883        let ClientControlResponse::ServerDescribe { capabilities, .. } =
15884            serde_json::from_slice(&reply.body).unwrap()
15885        else {
15886            panic!("not a server.describe reply");
15887        };
15888        assert!(
15889            capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
15890            "{capabilities:?}"
15891        );
15892    }
15893
15894    /// `catalog-events/v1` is in `server.describe`, so a consumer can tell a
15895    /// catalog entry without `events` (the module declares none) from a daemon
15896    /// that never reports declared events.
15897    #[tokio::test]
15898    async fn catalog_events_capability_is_advertised() {
15899        let handler = ControlHandler::new(Arc::new(Registry::default()));
15900        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15901        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15902        let reply = handler
15903            .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
15904            .await
15905            .unwrap()
15906            .pop()
15907            .unwrap();
15908        let ClientControlResponse::ServerDescribe { capabilities, .. } =
15909            serde_json::from_slice(&reply.body).unwrap()
15910        else {
15911            panic!("not a server.describe reply");
15912        };
15913        assert!(
15914            capabilities
15915                .iter()
15916                .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
15917            "{capabilities:?}"
15918        );
15919    }
15920
15921    #[tokio::test]
15922    async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
15923        let handler = ControlHandler::new(Arc::new(Registry::default()));
15924        let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
15925        let ack = hello_via_sink(
15926            &handler,
15927            &module,
15928            &mut module_rx,
15929            hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
15930        )
15931        .await;
15932        let ack = parse_ack(&ack);
15933        assert!(
15934            ack.subc_capabilities
15935                .iter()
15936                .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
15937            "{:?}",
15938            ack.subc_capabilities
15939        );
15940        assert!(ack
15941            .subc_capabilities
15942            .iter()
15943            .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
15944
15945        let (client, _client_rx) = route_ctx(ConnectionId::new(153));
15946        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15947        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15948        let reply = handler
15949            .handle_control_frame(&client, frame)
15950            .await
15951            .unwrap()
15952            .pop()
15953            .unwrap();
15954        let ClientControlResponse::ServerDescribe { capabilities, .. } =
15955            serde_json::from_slice(&reply.body).unwrap()
15956        else {
15957            panic!("not a server.describe reply");
15958        };
15959        assert!(
15960            capabilities
15961                .iter()
15962                .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
15963            "{capabilities:?}"
15964        );
15965        assert!(capabilities
15966            .iter()
15967            .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
15968    }
15969
15970    #[tokio::test]
15971    async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
15972        let handler = ControlHandler::new(Arc::new(Registry::default()));
15973        let (module, mut rx) = route_ctx(ConnectionId::new(156));
15974        let ack = hello_via_sink(
15975            &handler,
15976            &module,
15977            &mut rx,
15978            hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
15979        )
15980        .await;
15981        assert!(parse_ack(&ack)
15982            .subc_capabilities
15983            .iter()
15984            .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
15985        assert!(parse_ack(&ack)
15986            .subc_capabilities
15987            .iter()
15988            .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
15989        let (client, _rx) = route_ctx(ConnectionId::new(157));
15990        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15991        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15992        let reply = handler
15993            .handle_control_frame(&client, frame)
15994            .await
15995            .unwrap()
15996            .pop()
15997            .unwrap();
15998        let ClientControlResponse::ServerDescribe { capabilities, .. } =
15999            serde_json::from_slice(&reply.body).unwrap()
16000        else {
16001            panic!("server.describe reply")
16002        };
16003        assert!(capabilities
16004            .iter()
16005            .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
16006        assert!(capabilities
16007            .iter()
16008            .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
16009    }
16010
16011    #[tokio::test]
16012    async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
16013        let handler = ControlHandler::new(Arc::new(Registry::default()));
16014        let (module, mut rx) = route_ctx(ConnectionId::new(158));
16015        let ack = hello_via_sink(
16016            &handler,
16017            &module,
16018            &mut rx,
16019            hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
16020        )
16021        .await;
16022        assert!(parse_ack(&ack)
16023            .subc_capabilities
16024            .iter()
16025            .any(|cap| cap == "scope-workdir/v1"));
16026        let (client, _rx) = route_ctx(ConnectionId::new(159));
16027        let frame = Frame::build(
16028            FrameType::Request,
16029            control_flags(),
16030            0,
16031            0,
16032            5,
16033            serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
16034        )
16035        .unwrap();
16036        let reply = handler
16037            .handle_control_frame(&client, frame)
16038            .await
16039            .unwrap()
16040            .remove(0);
16041        let ClientControlResponse::ServerDescribe { capabilities, .. } =
16042            serde_json::from_slice(&reply.body).unwrap()
16043        else {
16044            panic!("server.describe reply")
16045        };
16046        assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
16047    }
16048
16049    #[tokio::test]
16050    async fn worker_runs_capability_is_advertised_in_hello_ack_and_server_describe() {
16051        let handler = ControlHandler::new(Arc::new(Registry::default()));
16052        let (module, mut rx) = route_ctx(ConnectionId::new(160));
16053        let ack = hello_via_sink(
16054            &handler,
16055            &module,
16056            &mut rx,
16057            hello_frame("worker-run-owner", PROTOCOL_VERSION, 1),
16058        )
16059        .await;
16060        assert!(parse_ack(&ack)
16061            .subc_capabilities
16062            .iter()
16063            .any(|cap| cap == "worker-runs/v1"));
16064        let (client, _rx) = route_ctx(ConnectionId::new(161));
16065        let frame = Frame::build(
16066            FrameType::Request,
16067            control_flags(),
16068            0,
16069            0,
16070            5,
16071            serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
16072        )
16073        .unwrap();
16074        let reply = handler
16075            .handle_control_frame(&client, frame)
16076            .await
16077            .unwrap()
16078            .remove(0);
16079        let ClientControlResponse::ServerDescribe { capabilities, .. } =
16080            serde_json::from_slice(&reply.body).unwrap()
16081        else {
16082            panic!("server.describe reply")
16083        };
16084        assert!(capabilities.iter().any(|cap| cap == "worker-runs/v1"));
16085    }
16086
16087    #[tokio::test]
16088    async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
16089        let handler = ControlHandler::new(Arc::new(Registry::default()));
16090        let (module, mut rx) = route_ctx(ConnectionId::new(154));
16091        let ack = hello_via_sink(
16092            &handler,
16093            &module,
16094            &mut rx,
16095            hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
16096        )
16097        .await;
16098        assert!(parse_ack(&ack)
16099            .subc_capabilities
16100            .iter()
16101            .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
16102        let (client, _rx) = route_ctx(ConnectionId::new(155));
16103        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
16104        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
16105        let reply = handler
16106            .handle_control_frame(&client, frame)
16107            .await
16108            .unwrap()
16109            .pop()
16110            .unwrap();
16111        let ClientControlResponse::ServerDescribe { capabilities, .. } =
16112            serde_json::from_slice(&reply.body).unwrap()
16113        else {
16114            panic!("not a server.describe reply")
16115        };
16116        assert!(capabilities
16117            .iter()
16118            .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
16119    }
16120
16121    #[tokio::test]
16122    async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
16123        let registry = Arc::new(Registry::default());
16124        let forwarding = Arc::new(ForwardingTable::default());
16125        let handler =
16126            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
16127                .with_health_probe_timeout(Duration::from_secs(5));
16128        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
16129        hello_via_sink(
16130            &handler,
16131            &module_ctx,
16132            &mut module_rx,
16133            non_routable_hello_frame_with_control_ops(
16134                "mcp",
16135                300,
16136                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
16137            ),
16138        )
16139        .await;
16140        assert!(registry
16141            .get_module("mcp")
16142            .unwrap()
16143            .unwrap()
16144            .manifest
16145            .provides
16146            .is_empty());
16147
16148        let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
16149        let route_response = handler
16150            .handle_control_frame(
16151                &route_client_ctx,
16152                route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
16153            )
16154            .await
16155            .unwrap();
16156        assert_eq!(route_response[0].header.ty, FrameType::Error);
16157        assert_eq!(
16158            parse_error(&route_response[0])["code"],
16159            "target_unavailable"
16160        );
16161        assert!(parse_error(&route_response[0])["message"]
16162            .as_str()
16163            .unwrap()
16164            .contains("does not provide the requested target"));
16165        assert!(module_rx.try_recv().is_err());
16166
16167        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
16168        let health_handler = handler.clone();
16169        let health_task = tokio::spawn(async move {
16170            health_handler
16171                .handle_control_frame(
16172                    &health_client_ctx,
16173                    supervisor_health_probe_frame(302, "mcp"),
16174                )
16175                .await
16176                .unwrap()
16177        });
16178        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
16179            .await
16180            .unwrap()
16181            .unwrap();
16182        assert_eq!(
16183            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
16184            ModuleControlRequest::HealthCheck {}
16185        );
16186        handler
16187            .handle_control_frame(
16188                &module_ctx,
16189                health_response(health_frame.header.corr, HealthStatus::Ok),
16190            )
16191            .await
16192            .unwrap();
16193        let health_response = health_task.await.unwrap();
16194        assert_eq!(health_response[0].header.ty, FrameType::Response);
16195        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
16196            ClientControlResponse::SupervisorHealthProbe {
16197                module_id, status, ..
16198            } => {
16199                assert_eq!(module_id, "mcp");
16200                assert_eq!(status, HealthStatus::Ok);
16201            }
16202            other => panic!("unexpected health response: {other:?}"),
16203        }
16204
16205        // Exercise the forwarding cleanup path directly while leaving the registry
16206        // advertisement in place. If cleanup leaves a stale control sink behind,
16207        // the next probe will enqueue onto it and wait for the long probe timeout
16208        // instead of returning an immediate no-connection error.
16209        forwarding
16210            .cleanup_connection(module_ctx.connection_id)
16211            .unwrap();
16212        let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
16213        let cleanup_response = tokio::time::timeout(
16214            Duration::from_millis(200),
16215            handler.handle_control_frame(
16216                &cleanup_probe_ctx,
16217                supervisor_health_probe_frame(303, "mcp"),
16218            ),
16219        )
16220        .await
16221        .expect("probe should fail immediately when the control lane is gone")
16222        .unwrap();
16223        assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
16224        assert_eq!(
16225            parse_error(&cleanup_response[0])["code"],
16226            "target_unavailable"
16227        );
16228        assert!(parse_error(&cleanup_response[0])["message"]
16229            .as_str()
16230            .unwrap()
16231            .contains("no module connection"));
16232
16233        handler
16234            .cleanup_connection(module_ctx.connection_id)
16235            .unwrap();
16236    }
16237
16238    #[tokio::test]
16239    async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
16240        let registry = Arc::new(Registry::default());
16241        let supervisor_handle = SupervisorHandle::new();
16242        let supervisor =
16243            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
16244                .with_handle(supervisor_handle.clone())
16245                .with_connection_file_path(
16246                    std::env::temp_dir()
16247                        .join(format!("subc-route-open-warming-{}", std::process::id())),
16248                );
16249        let module = supervisor
16250            .supervise_configured(
16251                ModuleSpec {
16252                    script: None,
16253                    carry_containment: false,
16254                    module_id: "warming".to_string(),
16255                    program: fake_aft_stub_path(),
16256                    args: Vec::new(),
16257                    env: Vec::new(),
16258                    reserved: false,
16259                    reserved_prefixes: Vec::new(),
16260                    protocol: ModuleProtocol::Subc,
16261                    overlap: Default::default(),
16262                },
16263                true,
16264            )
16265            .unwrap();
16266        assert_eq!(module.state().unwrap(), ModuleState::Running);
16267
16268        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
16269        let (ctx, _rx) = route_ctx(ConnectionId::new(39));
16270        let response = handler
16271            .handle_control_frame(
16272                &ctx,
16273                route_open_frame(304, "warming", unique_project_root("warming")),
16274            )
16275            .await
16276            .unwrap();
16277        module.stop().await.unwrap();
16278
16279        assert_eq!(response[0].header.ty, FrameType::Error);
16280        let error = parse_error(&response[0]);
16281        assert_eq!(error["code"], "module_warming");
16282        assert!(error["message"]
16283            .as_str()
16284            .unwrap()
16285            .contains("state=running, enabled=true, live=false"));
16286    }
16287
16288    #[test]
16289    fn route_open_connection_cap_logs_admission_reason_and_capacity() {
16290        let handler = ControlHandler::new(Arc::new(Registry::default()));
16291        let capture = EventCapture::default();
16292        let _subscriber =
16293            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
16294        let (ctx, _rx) = route_ctx(ConnectionId::new(96));
16295        let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
16296        let pending = (0..limit).collect::<Vec<_>>();
16297        let response = handler
16298            .route_open_capacity_refusal(
16299                &ctx,
16300                &route_open_frame(396, "busy", unique_project_root("connection-cap")),
16301                "busy",
16302                pending.len(),
16303                limit,
16304            )
16305            .unwrap();
16306        assert_eq!(parse_error(&response)["code"], "target_unavailable");
16307        let event = capture
16308            .events()
16309            .into_iter()
16310            .find(|event| {
16311                event.target == "control"
16312                    && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
16313            })
16314            .expect("connection admission refusal event");
16315        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
16316        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
16317        // Refused before `handle_route_open` parses the request, so the caller
16318        // fields are read from the frame itself.
16319        assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
16320        assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
16321    }
16322
16323    #[test]
16324    fn route_open_target_cap_logs_admission_reason_and_capacity() {
16325        let handler = ControlHandler::new(Arc::new(Registry::default()));
16326        let capture = EventCapture::default();
16327        let _subscriber =
16328            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
16329        let (ctx, _rx) = route_ctx(ConnectionId::new(97));
16330        let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
16331        let guards = (0..limit)
16332            .map(|_| {
16333                handler
16334                    .route_bind_concurrency
16335                    .try_admit("busy", limit)
16336                    .unwrap()
16337            })
16338            .collect::<Vec<_>>();
16339        let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
16340            Err(in_flight) => in_flight,
16341            Ok(_) => panic!("target cap must refuse after {limit} admissions"),
16342        };
16343        let response = handler
16344            .route_open_target_capacity_refusal(
16345                &ctx,
16346                &RouteOpenCaller::unknown(),
16347                &route_open_frame(397, "busy", unique_project_root("target-cap")),
16348                "busy",
16349                in_flight,
16350            )
16351            .unwrap();
16352        assert_eq!(parse_error(&response)["code"], "target_unavailable");
16353        let event = capture
16354            .events()
16355            .into_iter()
16356            .find(|event| {
16357                event.target == "control"
16358                    && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
16359            })
16360            .expect("target admission refusal event");
16361        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
16362        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
16363        drop(guards);
16364    }
16365
16366    #[cfg(unix)]
16367    #[tokio::test(start_paused = true)]
16368    async fn supervisor_list_shows_a_failed_modules_next_retry() {
16369        let registry = Arc::new(Registry::default());
16370        let handle = SupervisorHandle::new();
16371        let supervisor =
16372            Supervisor::new_for_test(registry.clone(), RestartPolicy::new(0, Duration::ZERO))
16373                .with_handle(handle.clone());
16374        let module = supervisor
16375            .spawn(crate::ModuleSpec {
16376                script: None,
16377                carry_containment: false,
16378                module_id: "retry-list".into(),
16379                program: PathBuf::from("/bin/sh"),
16380                args: vec!["-c".into(), "exit 17".into()],
16381                env: vec![],
16382                reserved: false,
16383                reserved_prefixes: vec![],
16384                protocol: ModuleProtocol::None,
16385                overlap: Default::default(),
16386            })
16387            .unwrap();
16388        let deadline = std::time::Instant::now() + Duration::from_secs(10);
16389        while module.state().unwrap() != crate::ModuleState::Failed {
16390            assert!(
16391                std::time::Instant::now() < deadline,
16392                "waiting for budget failure"
16393            );
16394            tokio::task::yield_now().await;
16395        }
16396        let handler = ControlHandler::new(registry).with_supervisor(handle);
16397        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
16398        let replies = handler.handle_supervisor_list(frame).await.unwrap();
16399        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16400        assert_eq!(body["modules"][0]["state"], "failed");
16401        let retry = body["modules"][0]["next_retry_at_ms"]
16402            .as_u64()
16403            .expect("scheduled retry on the wire");
16404        let now = std::time::SystemTime::now()
16405            .duration_since(std::time::UNIX_EPOCH)
16406            .unwrap()
16407            .as_millis() as u64;
16408        assert!((599_000..=600_000).contains(&retry.saturating_sub(now)));
16409        module.set_enabled(false).await.unwrap();
16410        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
16411        let replies = handler.handle_supervisor_list(frame).await.unwrap();
16412        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16413        assert!(body["modules"][0].get("next_retry_at_ms").is_none());
16414    }
16415
16416    #[tokio::test]
16417    async fn slow_step_before_pid_read_does_not_spend_its_budget() {
16418        let dir = cortexkit_test_support::ScratchDir::new("pid-read-budget");
16419        let supervisor =
16420            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
16421        let module = supervisor
16422            .spawn(crate::ModuleSpec {
16423                script: None,
16424                carry_containment: false,
16425                module_id: "pid-read-budget".into(),
16426                program: std::env::current_exe().unwrap(),
16427                args: vec![
16428                    "--exact".into(),
16429                    "control::tests::pid_resource_read_fixture".into(),
16430                ],
16431                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
16432                    .into_iter()
16433                    .map(|key| (key.into(), dir.display().to_string()))
16434                    .chain(std::iter::once((
16435                        "SUBC_SUBTREE_PID_FIXTURE".into(),
16436                        "1".into(),
16437                    )))
16438                    .collect(),
16439                reserved: false,
16440                reserved_prefixes: Vec::new(),
16441                protocol: ModuleProtocol::None,
16442                overlap: Default::default(),
16443            })
16444            .unwrap();
16445        let ready_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
16446        while !matches!(
16447            module.child_resource_usage(),
16448            subc_control::ChildResourceUsage::Measured(_)
16449        ) {
16450            assert!(
16451                tokio::time::Instant::now() < ready_deadline,
16452                "fixture process was not measurable"
16453            );
16454            tokio::time::sleep(Duration::from_millis(10)).await;
16455        }
16456        tokio::time::sleep(crate::subtree_resources::READ_TIMEOUT * 2).await;
16457        let reading = {
16458            let _guard = crate::subtree_resources::fault_read_guard().await;
16459            crate::subtree_resources::read_process(module.clone()).await
16460        };
16461        stop_resource_fixture(&module, &dir).await;
16462        assert!(
16463            matches!(reading, subc_control::ChildResourceUsage::Measured(_)),
16464            "{reading:?}"
16465        );
16466    }
16467
16468    #[test]
16469    fn pid_resource_read_fixture() {
16470        if std::env::var_os("SUBC_SUBTREE_PID_FIXTURE").is_some() {
16471            let dir = std::path::PathBuf::from(std::env::var_os("XDG_DATA_HOME").unwrap());
16472            let deadline = std::time::Instant::now() + Duration::from_secs(60);
16473            while !dir.join("stop-module").exists() && std::time::Instant::now() < deadline {
16474                std::thread::sleep(Duration::from_millis(5));
16475            }
16476            std::fs::write(dir.join("module-stopped"), "").unwrap();
16477        }
16478    }
16479
16480    async fn stop_resource_fixture(
16481        module: &crate::supervise::SupervisedModule,
16482        dir: &std::path::Path,
16483    ) {
16484        let stopping = module.stop();
16485        tokio::pin!(stopping);
16486        tokio::select! {
16487            result = &mut stopping => result.unwrap(),
16488            () = async {
16489                while module.state().unwrap() != crate::supervise::ModuleState::Draining {
16490                    tokio::time::sleep(Duration::from_millis(5)).await;
16491                }
16492                std::fs::write(dir.join("stop-module"), "").unwrap();
16493            } => stopping.await.unwrap(),
16494        }
16495        #[cfg(windows)]
16496        assert!(
16497            dir.join("module-stopped").exists(),
16498            "the resource fixture must exit cooperatively, not wait for the drain kill"
16499        );
16500        assert_eq!(
16501            module.state().unwrap(),
16502            crate::supervise::ModuleState::Stopped
16503        );
16504    }
16505
16506    #[tokio::test]
16507    async fn disabled_module_tree_is_not_running_instead_of_permanently_pending() {
16508        let registry = Arc::new(Registry::default());
16509        let handle = SupervisorHandle::new();
16510        let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16511            .with_handle(handle.clone());
16512        supervisor
16513            .supervise_configured(
16514                crate::ModuleSpec {
16515                    script: None,
16516                    module_id: "disabled-resource-reading".into(),
16517                    program: std::env::current_exe().unwrap(),
16518                    args: Vec::new(),
16519                    env: Vec::new(),
16520                    reserved: false,
16521                    reserved_prefixes: Vec::new(),
16522                    protocol: ModuleProtocol::None,
16523                    overlap: Default::default(),
16524                    carry_containment: false,
16525                },
16526                false,
16527            )
16528            .unwrap();
16529        let handler = ControlHandler::new(registry).with_supervisor(handle);
16530        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16531        let replies = handler.handle_supervisor_list(frame).await.unwrap();
16532        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16533        assert!(body["modules"][0].get("tree").is_none());
16534        assert_eq!(body["modules"][0]["tree_unavailable"], "not_running");
16535    }
16536
16537    #[tokio::test]
16538    async fn supervisor_list_reads_process_resources_before_a_stalled_tree() {
16539        let dir = cortexkit_test_support::ScratchDir::new("list-process-priority");
16540        let registry = Arc::new(Registry::default());
16541        let handle = SupervisorHandle::new();
16542        let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16543            .with_handle(handle.clone());
16544        let module = supervisor
16545            .spawn(crate::ModuleSpec {
16546                script: None,
16547                carry_containment: false,
16548                module_id: "list-process-priority".into(),
16549                program: std::env::current_exe().unwrap(),
16550                args: vec![
16551                    "--exact".into(),
16552                    "control::tests::pid_resource_read_fixture".into(),
16553                ],
16554                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
16555                    .into_iter()
16556                    .map(|key| (key.into(), dir.display().to_string()))
16557                    .chain(std::iter::once((
16558                        "SUBC_SUBTREE_PID_FIXTURE".into(),
16559                        "1".into(),
16560                    )))
16561                    .collect(),
16562                reserved: false,
16563                reserved_prefixes: Vec::new(),
16564                protocol: ModuleProtocol::None,
16565                overlap: Default::default(),
16566            })
16567            .unwrap();
16568        let ready_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
16569        loop {
16570            let image = module.running_image_agreement().await;
16571            if matches!(image, subc_control::RunningImageAgreement::Match { .. }) {
16572                break;
16573            }
16574            assert!(
16575                tokio::time::Instant::now() < ready_deadline,
16576                "the fixture process never became observable: {image:?}"
16577            );
16578            tokio::time::sleep(Duration::from_millis(10)).await;
16579        }
16580        let guard = crate::subtree_resources::fault_read_guard().await;
16581        let (target, release, _) = crate::subtree_resources::held();
16582        module.set_subtree_target_for_test(target);
16583        let handler = ControlHandler::new(registry).with_supervisor(handle);
16584        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16585        let replies = handler.handle_supervisor_list(frame).await.unwrap();
16586        drop(release);
16587        drop(guard);
16588        stop_resource_fixture(&module, &dir).await;
16589        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16590        assert_eq!(
16591            body["modules"][0]["resources"]["status"], "measured",
16592            "the optional tree timeout must not discard the process reading: {body}"
16593        );
16594        assert!(body["modules"][0].get("tree").is_none());
16595        assert_eq!(body["modules"][0]["tree_unavailable"], "not_yet_measured");
16596        assert_eq!(
16597            body["modules"][0]["pending_reload"]["image"]["status"], "match",
16598            "the tree timeout must not hide a completed image observation"
16599        );
16600    }
16601
16602    #[cfg(any(target_os = "linux", target_os = "macos"))]
16603    #[tokio::test]
16604    async fn cold_explicit_provenance_completes_while_list_stays_bounded() {
16605        let mut handler = ControlHandler::new(Arc::new(Registry::default()))
16606            .with_daemon_provenance(
16607                std::process::id(),
16608                0,
16609                Some(std::env::current_exe().unwrap()),
16610                None,
16611                None,
16612            );
16613        let (probe, release, began) = ExecutableIdentityProbe::held();
16614        handler.daemon_provenance.probe = probe;
16615        let observe = |handler: ControlHandler, corr| {
16616            tokio::spawn(async move {
16617                let (ctx, _) = route_ctx(ConnectionId::new(corr));
16618                let frame =
16619                    Frame::build(FrameType::Request, control_flags(), 0, 0, corr, Vec::new())
16620                        .unwrap();
16621                handler
16622                    .handle_supervisor_provenance(&ctx, frame, None)
16623                    .await
16624                    .unwrap()
16625            })
16626        };
16627        let explicit = observe(handler.clone(), 114);
16628        began.await.unwrap();
16629        let concurrent = observe(handler.clone(), 115);
16630        // Keep explicit provenance waiting on its first image observation beyond
16631        // the list deadline; release it only after the concurrent list replies.
16632        tokio::time::sleep(Duration::from_millis(300)).await;
16633        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
16634        let list = tokio::time::timeout(
16635            Duration::from_secs(1),
16636            handler.handle_supervisor_list(frame),
16637        )
16638        .await;
16639        let explicit_was_pending = !explicit.is_finished() && !concurrent.is_finished();
16640        drop(release);
16641        let replies = explicit.await.unwrap();
16642        let concurrent_replies = concurrent.await.unwrap();
16643        assert!(
16644            explicit_was_pending,
16645            "explicit provenance must await the full observation"
16646        );
16647        let list = list
16648            .expect("a concurrent explicit probe must not stall list")
16649            .unwrap();
16650        let list_body: Value = serde_json::from_slice(&list[0].body).unwrap();
16651        assert_eq!(list_body["op"], "supervisor.list");
16652        for replies in [replies, concurrent_replies] {
16653            let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16654            assert_eq!(
16655                body["daemon"]["daemon_observed"]["running_image"]["status"],
16656                "match"
16657            );
16658        }
16659    }
16660
16661    #[tokio::test]
16662    async fn supervisor_list_slow_subtree_does_not_hold_connection_reader() {
16663        let registry = Arc::new(Registry::default());
16664        let handle = SupervisorHandle::new();
16665        let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16666            .with_handle(handle.clone());
16667        let module = supervisor
16668            .supervise_configured(
16669                crate::ModuleSpec {
16670                    script: None,
16671                    carry_containment: false,
16672                    module_id: "memory-read".into(),
16673                    program: PathBuf::from("disabled-module"),
16674                    args: Vec::new(),
16675                    env: Vec::new(),
16676                    reserved: false,
16677                    reserved_prefixes: Vec::new(),
16678                    protocol: ModuleProtocol::Subc,
16679                    overlap: Default::default(),
16680                },
16681                true,
16682            )
16683            .unwrap();
16684        let _guard = crate::subtree_resources::fault_read_guard().await;
16685        let (target, release, began) = crate::subtree_resources::held();
16686        // Optional list work may be skipped when its deadline is already spent.
16687        // Establish the stalled fixture before asking the list to observe it.
16688        let stalled_target = target.clone();
16689        let stalled = tokio::task::spawn_blocking(move || {
16690            stalled_target.tree(&subc_os::ProcessTable::default())
16691        });
16692        began.await.unwrap();
16693        module.set_subtree_target_for_test(target);
16694        let handler = ControlHandler::new(registry).with_supervisor(handle);
16695        let (ctx, mut rx) = route_ctx(ConnectionId::new(112));
16696        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16697        let replies = handler
16698            .handle_client_control_request(
16699                &ctx,
16700                frame,
16701                ClientControlRequest::SupervisorList {},
16702                &mut None,
16703            )
16704            .await
16705            .unwrap();
16706        assert!(
16707            replies.is_empty(),
16708            "list reply must be detached from the connection reader"
16709        );
16710        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
16711        let replies = handler
16712            .handle_client_control_request(
16713                &ctx,
16714                frame,
16715                ClientControlRequest::SupervisorSpawnSnapshot {},
16716                &mut None,
16717            )
16718            .await
16719            .unwrap();
16720        assert_eq!(
16721            replies.len(),
16722            1,
16723            "another request must complete while the subtree read stalls"
16724        );
16725        let reply = tokio::time::timeout(Duration::from_secs(5), rx.recv())
16726            .await
16727            .unwrap()
16728            .unwrap();
16729        release.send(()).unwrap();
16730        drop(release);
16731        stalled.await.unwrap();
16732        let body: Value = serde_json::from_slice(&reply.frame.body).unwrap();
16733        assert!(
16734            body["modules"][0].get("tree").is_none(),
16735            "an unknown tree reading must be omitted after the bounded read expires"
16736        );
16737        // Lists export completed work rather than measuring synchronously.
16738        // Complete a refresh after the fixture is released before asserting
16739        // that a later list contains its measured tree.
16740        handler
16741            .resource_tracker
16742            .read(handler.supervisor.list(), false, "released_tree_fixture")
16743            .await;
16744        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
16745        let replies = handler.handle_supervisor_list(frame).await.unwrap();
16746        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16747        assert_eq!(
16748            body["modules"][0]["tree"]["source"], "process_tree",
16749            "the next request must export the completed refresh"
16750        );
16751        assert_eq!(body["modules"][0]["tree"]["process_count"], 1);
16752    }
16753
16754    #[tokio::test]
16755    async fn supervisor_list_expired_budget_keeps_committed_accounting_rows() {
16756        use subc_control::{
16757            AgentCommandResources, ChildMemoryKind, ChildResourceReading,
16758            ChildResourceUnavailableReason, ChildResourceUsage, CpuLastHour, DaemonResources,
16759            TreeReading, TreeSource,
16760        };
16761        let registry = Arc::new(Registry::default());
16762        let handle = SupervisorHandle::new();
16763        let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16764            .with_handle(handle.clone());
16765        supervisor
16766            .supervise_configured(
16767                crate::ModuleSpec {
16768                    script: None,
16769                    carry_containment: false,
16770                    module_id: "budget-worker".into(),
16771                    program: PathBuf::from("disabled-module"),
16772                    args: Vec::new(),
16773                    env: Vec::new(),
16774                    reserved: false,
16775                    reserved_prefixes: Vec::new(),
16776                    protocol: ModuleProtocol::Subc,
16777                    overlap: Default::default(),
16778                },
16779                true,
16780            )
16781            .unwrap();
16782        let handler = ControlHandler::new(registry).with_supervisor(handle);
16783        let now = u64::try_from(
16784            std::time::SystemTime::now()
16785                .duration_since(std::time::UNIX_EPOCH)
16786                .unwrap()
16787                .as_millis(),
16788        )
16789        .unwrap();
16790        let measured = now - 1_000;
16791        let hour = CpuLastHour::new(100, 60_000);
16792        let daemon = DaemonResources::new(7, 1, "test")
16793            .with_measured_at_ms(Some(measured))
16794            .with_resources(Some(ChildResourceUsage::Measured(
16795                ChildResourceReading::new(20, ChildMemoryKind::ResidentSet, 10, 0),
16796            )))
16797            .with_cpu_last_hour(Some(hour.clone()));
16798        let agents = AgentCommandResources::new("budget-worker", 50, 30, 1)
16799            .with_measured_at_ms(Some(measured))
16800            .with_cpu_last_hour(Some(hour.clone()));
16801        let tree = TreeReading::new(
16802            150,
16803            50,
16804            ChildMemoryKind::ResidentSet,
16805            1,
16806            TreeSource::ProcessTree,
16807        )
16808        .with_measured_at_ms(Some(measured));
16809        handler.resource_tracker.seed_committed_for_test(
16810            &handler.supervisor.list(),
16811            crate::resource_totals::Readings {
16812                modules: BTreeMap::from([("budget-worker".into(), (Some(tree), Some(hour)))]),
16813                daemon: Some(daemon),
16814                agent_commands: Some(vec![agents]),
16815                ..Default::default()
16816            },
16817            now,
16818        );
16819        list_diagnostics::scope(async {
16820            tokio::time::sleep(Duration::from_millis(110)).await;
16821            assert!(list_diagnostics::deadline().unwrap() <= tokio::time::Instant::now());
16822            let request =
16823                Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16824            let replies = handler.handle_supervisor_list_inner(request).await.unwrap();
16825            let ClientControlResponse::SupervisorList {
16826                modules,
16827                daemon,
16828                agent_commands,
16829                ..
16830            } = serde_json::from_slice(&replies[0].body).unwrap()
16831            else {
16832                panic!("expected a list response")
16833            };
16834            assert!(matches!(
16835                modules[0].resources,
16836                Some(ChildResourceUsage::Unavailable {
16837                    reason: ChildResourceUnavailableReason::Unreadable
16838                })
16839            ));
16840            assert_eq!(
16841                (
16842                    modules[0].tree.as_ref().unwrap().cpu_ms,
16843                    modules[0].tree.as_ref().unwrap().measured_at_ms
16844                ),
16845                (150, Some(measured))
16846            );
16847            let daemon = daemon.unwrap();
16848            let agents = agent_commands.unwrap();
16849            assert_eq!(daemon.measured_at_ms, Some(measured));
16850            assert_eq!(
16851                (agents.len(), agents[0].cpu_ms, agents[0].measured_at_ms),
16852                (1, 50, Some(measured))
16853            );
16854            for hour in [
16855                modules[0].cpu_last_hour.as_ref(),
16856                daemon.cpu_last_hour.as_ref(),
16857                agents[0].cpu_last_hour.as_ref(),
16858            ] {
16859                let hour = hour.unwrap();
16860                assert_eq!((hour.cpu_ms, hour.covered_ms), (100, 60_000));
16861            }
16862        })
16863        .await;
16864    }
16865
16866    #[tokio::test]
16867    async fn supervisor_list_reports_loaded_config_and_omits_unconfigured() {
16868        let registry = Arc::new(Registry::default());
16869        let handle = SupervisorHandle::new();
16870        let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16871            .with_handle(handle.clone());
16872        supervisor
16873            .supervise_configured(
16874                crate::ModuleSpec {
16875                    script: None,
16876                    carry_containment: false,
16877                    module_id: "drift".into(),
16878                    program: PathBuf::from("disabled-module"),
16879                    args: Vec::new(),
16880                    env: Vec::new(),
16881                    reserved: false,
16882                    reserved_prefixes: Vec::new(),
16883                    protocol: ModuleProtocol::Subc,
16884                    overlap: Default::default(),
16885                },
16886                false,
16887            )
16888            .unwrap();
16889        let handler = ControlHandler::new(registry).with_supervisor(handle);
16890        for configured in [None, Some(false), Some(true)] {
16891            handler.capability_evaluator.configure(
16892                configured.map(|enabled| ("drift".to_string(), enabled)),
16893                BTreeMap::new(),
16894            );
16895            let frame =
16896                Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16897            let replies = handler
16898                .handle_supervisor_set_enabled(frame, "drift".into(), false, Principal::Direct)
16899                .await
16900                .unwrap();
16901            let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16902            assert!(body.get("configured_enabled").is_none());
16903            assert_eq!(body["op"], "supervisor.ack");
16904            let frame =
16905                Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
16906            let replies = handler.handle_supervisor_list(frame).await.unwrap();
16907            let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16908            assert_eq!(
16909                body["modules"][0].get("configured_enabled"),
16910                configured.map(Value::Bool).as_ref()
16911            );
16912            assert!(body["modules"][0].get("tree").is_none());
16913            assert!(body["modules"][0].get("subtree").is_none());
16914        }
16915    }
16916
16917    #[tokio::test(flavor = "current_thread")]
16918    async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
16919        let handler = ControlHandler::new(Arc::new(Registry::default()));
16920        let capture = EventCapture::default();
16921        let _subscriber =
16922            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
16923        let (ctx, _rx) = route_ctx(ConnectionId::new(101));
16924        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16925
16926        let _ = handler
16927            .handle_client_control_request(
16928                &ctx,
16929                frame,
16930                ClientControlRequest::SupervisorSetEnabled {
16931                    module_id: "broca".to_string(),
16932                    enabled: false,
16933                },
16934                &mut None,
16935            )
16936            .await
16937            .unwrap();
16938
16939        let events = capture
16940            .events()
16941            .into_iter()
16942            .filter(|event| {
16943                event.target == "control"
16944                    && event.fields.get("message").map(String::as_str)
16945                        == Some("supervisor request received")
16946            })
16947            .collect::<Vec<_>>();
16948        assert_eq!(events.len(), 1, "one supervisor request log line");
16949        let event = &events[0];
16950        assert_eq!(event.level, tracing::Level::INFO);
16951        assert_eq!(
16952            event.fields.get("op"),
16953            Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
16954        );
16955        assert_eq!(
16956            event.fields.get("module_id"),
16957            Some(&"\"broca\"".to_string())
16958        );
16959        assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
16960        assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
16961        assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
16962    }
16963
16964    #[tokio::test(flavor = "current_thread")]
16965    async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
16966        let registry = Arc::new(Registry::default());
16967        let module_connection = ConnectionId::new(303);
16968        registry
16969            .register_with_control_ops(
16970                manifest("aft", PROTOCOL_VERSION),
16971                PROTOCOL_VERSION,
16972                module_connection,
16973                module_baseline_control_ops(),
16974            )
16975            .unwrap();
16976        let handler = ControlHandler::new(registry);
16977        let capture = EventCapture::default();
16978        let _subscriber =
16979            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
16980        let (ctx, _rx) = route_ctx(module_connection);
16981        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16982
16983        let _ = handler
16984            .handle_client_control_request(
16985                &ctx,
16986                frame,
16987                ClientControlRequest::SupervisorSetEnabled {
16988                    module_id: "broca".to_string(),
16989                    enabled: false,
16990                },
16991                &mut None,
16992            )
16993            .await
16994            .unwrap();
16995
16996        let callers = capture
16997            .events()
16998            .into_iter()
16999            .filter(|event| {
17000                event.fields.get("message").map(String::as_str)
17001                    == Some("supervisor request received")
17002            })
17003            .map(|event| event.fields.get("caller").cloned())
17004            .collect::<Vec<_>>();
17005        assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
17006    }
17007
17008    #[tokio::test(flavor = "current_thread")]
17009    async fn supervisor_list_does_not_log_request_received() {
17010        let handler = ControlHandler::new(Arc::new(Registry::default()));
17011        let capture = EventCapture::default();
17012        let _subscriber =
17013            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17014        let (ctx, _rx) = route_ctx(ConnectionId::new(102));
17015        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
17016
17017        handler
17018            .handle_client_control_request(
17019                &ctx,
17020                frame,
17021                ClientControlRequest::SupervisorList {},
17022                &mut None,
17023            )
17024            .await
17025            .unwrap();
17026
17027        assert!(capture.events().into_iter().all(|event| {
17028            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
17029        }));
17030    }
17031
17032    #[tokio::test(flavor = "current_thread")]
17033    async fn supervisor_rescan_preview_does_not_log_request_received() {
17034        let handler = ControlHandler::new(Arc::new(Registry::default()));
17035        let capture = EventCapture::default();
17036        let _subscriber =
17037            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17038        let (ctx, _rx) = route_ctx(ConnectionId::new(103));
17039        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
17040
17041        handler
17042            .handle_client_control_request(
17043                &ctx,
17044                frame,
17045                ClientControlRequest::SupervisorRescan { preview: true },
17046                &mut None,
17047            )
17048            .await
17049            .unwrap();
17050
17051        assert!(capture.events().into_iter().all(|event| {
17052            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
17053        }));
17054    }
17055
17056    struct ApproveOperator;
17057    impl crate::operator_confirm::OperatorProvider for ApproveOperator {
17058        fn prompt(
17059            &self,
17060            _: &str,
17061            _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
17062        ) -> crate::operator_confirm::ProviderResult {
17063            crate::operator_confirm::ProviderResult::Approved
17064        }
17065    }
17066
17067    #[tokio::test]
17068    async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
17069        assert_eq!(
17070            ControlHandler::default().privileged_operator_prompt,
17071            PrivilegedOperatorPrompt::Log
17072        );
17073        for (initial, changed, running) in [
17074            ("log", "enforce", PrivilegedOperatorPrompt::Log),
17075            ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
17076        ] {
17077            let dir = ScratchDir::new("rescan-privileged-operator-prompt");
17078            let path = dir.join("subc.jsonc");
17079            std::fs::write(
17080                &path,
17081                json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
17082            )
17083            .unwrap();
17084            let config = crate::daemon_config::load(&path).unwrap().unwrap();
17085            let registry = Arc::new(Registry::default());
17086            let handle = SupervisorHandle::new();
17087            let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
17088                .with_handle(handle.clone());
17089            let handler = ControlHandler::new(registry)
17090                .with_supervisor(handle)
17091                .with_privileged_operator_prompt(config.privileged_operator_prompt)
17092                .with_supervisor_rescan(supervisor, &path, config.port);
17093            handler.forwarding.operator_confirms().configure(
17094                Arc::new(ApproveOperator),
17095                crate::operator_confirm::OperatorLimits::default(),
17096            );
17097            let (ctx, _rx) = route_ctx(ConnectionId::new(104));
17098
17099            for (disk_mode, needs_restart) in [
17100                (Some(initial), false),
17101                (Some(changed), true),
17102                (Some(changed), true),
17103                (None, running == PrivilegedOperatorPrompt::Enforce),
17104                (Some(initial), false),
17105            ] {
17106                let mut doc = json!({"version":1});
17107                if let Some(mode) = disk_mode {
17108                    doc["privileged_operator_prompt"] = json!(mode);
17109                }
17110                std::fs::write(&path, doc.to_string()).unwrap();
17111                for preview in [true, false] {
17112                    let request = ClientControlRequest::SupervisorRescan { preview };
17113                    let frame = Frame::build(
17114                        FrameType::Request,
17115                        control_flags(),
17116                        0,
17117                        0,
17118                        1,
17119                        serde_json::to_vec(&request).unwrap(),
17120                    )
17121                    .unwrap();
17122                    let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
17123                    assert_eq!(frames.len(), 1);
17124                    let ClientControlResponse::SupervisorRescan { result } =
17125                        serde_json::from_slice(&frames[0].body).unwrap()
17126                    else {
17127                        panic!("expected a rescan response");
17128                    };
17129                    let expected = if needs_restart {
17130                        vec!["privileged_operator_prompt"]
17131                    } else {
17132                        Vec::new()
17133                    };
17134                    assert_eq!(result.restart_required, expected);
17135                    assert_eq!(handler.privileged_operator_prompt, running);
17136                    assert_eq!(
17137                        handler.rescan.as_ref().unwrap().privileged_operator_prompt,
17138                        running
17139                    );
17140                }
17141            }
17142        }
17143    }
17144
17145    #[tokio::test]
17146    async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
17147        let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
17148        let path = dir.join("subc.jsonc");
17149        let registry = Arc::new(Registry::default());
17150        let handle = SupervisorHandle::new();
17151        let supervisor =
17152            Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
17153        let handler = ControlHandler::new(registry)
17154            .with_supervisor(handle)
17155            .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
17156            .with_supervisor_rescan(supervisor, &path, None);
17157        handler.forwarding.operator_confirms().configure(
17158            Arc::new(ApproveOperator),
17159            crate::operator_confirm::OperatorLimits::default(),
17160        );
17161        let (ctx, _rx) = route_ctx(ConnectionId::new(105));
17162        for preview in [true, false] {
17163            std::fs::write(
17164                &path,
17165                r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
17166            )
17167            .unwrap();
17168            let request = ClientControlRequest::SupervisorRescan { preview };
17169            let frame = Frame::build(
17170                FrameType::Request,
17171                control_flags(),
17172                0,
17173                0,
17174                1,
17175                serde_json::to_vec(&request).unwrap(),
17176            )
17177            .unwrap();
17178            let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
17179            assert_eq!(frames.len(), 1);
17180            assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
17181            assert_eq!(
17182                handler.privileged_operator_prompt,
17183                PrivilegedOperatorPrompt::Enforce
17184            );
17185        }
17186    }
17187
17188    /// One wire code has several senders, so the refusal line names the check
17189    /// that refused. This drives the shared refusal path for ordinary refusals
17190    /// with an unregistered
17191    /// target and requires the branch label on the event.
17192    #[tokio::test]
17193    async fn route_open_refusal_names_the_check_that_refused() {
17194        let handler = ControlHandler::new(Arc::new(Registry::default()));
17195        let capture = EventCapture::default();
17196        let _subscriber =
17197            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17198        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
17199        let response = handler
17200            .handle_control_frame(
17201                &ctx,
17202                route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
17203            )
17204            .await
17205            .unwrap();
17206
17207        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
17208        let event = capture
17209            .events()
17210            .into_iter()
17211            .find(|event| {
17212                event.target == "control"
17213                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
17214            })
17215            .expect("route.open refusal event");
17216        assert_eq!(
17217            event.fields.get("reason"),
17218            Some(&"\"not_registered\"".to_string())
17219        );
17220    }
17221
17222    #[tokio::test]
17223    async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
17224        let registry = Arc::new(Registry::default());
17225        let supervisor_handle = SupervisorHandle::new();
17226        let supervisor =
17227            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
17228                .with_handle(supervisor_handle.clone())
17229                .with_connection_file_path(std::env::temp_dir().join(format!(
17230                    "subc-route-open-refusal-info-{}",
17231                    std::process::id()
17232                )));
17233        let module = supervisor
17234            .supervise_configured(
17235                ModuleSpec {
17236                    script: None,
17237                    carry_containment: false,
17238                    module_id: "warming".to_string(),
17239                    program: fake_aft_stub_path(),
17240                    args: Vec::new(),
17241                    env: Vec::new(),
17242                    reserved: false,
17243                    reserved_prefixes: Vec::new(),
17244                    protocol: ModuleProtocol::Subc,
17245                    overlap: Default::default(),
17246                },
17247                true,
17248            )
17249            .unwrap();
17250        assert_eq!(module.state().unwrap(), ModuleState::Running);
17251
17252        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
17253        assert!(handler
17254            .counters()
17255            .snapshot()
17256            .get("route_open_refused_by_code")
17257            .is_none());
17258        let capture = EventCapture::default();
17259        let _subscriber =
17260            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17261        let (ctx, _rx) = route_ctx(ConnectionId::new(94));
17262        let response = handler
17263            .handle_control_frame(
17264                &ctx,
17265                route_open_frame(394, "warming", unique_project_root("refusal-info")),
17266            )
17267            .await
17268            .unwrap();
17269        module.stop().await.unwrap();
17270
17271        assert_eq!(parse_error(&response[0])["code"], "module_warming");
17272        let event = capture
17273            .events()
17274            .into_iter()
17275            .find(|event| {
17276                event.target == "control"
17277                    && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
17278            })
17279            .expect("route.open refusal event");
17280        assert_eq!(
17281            event.fields.get("module_id"),
17282            Some(&"\"warming\"".to_string())
17283        );
17284        assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
17285        assert_eq!(
17286            event.fields.get("reason"),
17287            Some(&"\"supervised_not_registered\"".to_string())
17288        );
17289        assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
17290        assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
17291        assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
17292        assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
17293        assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
17294        assert_eq!(event.fields.get("harness"), Some(&"\"unit\"".to_string()));
17295        assert_eq!(
17296            handler.counters().snapshot()["route_open_refused_by_code"],
17297            json!({ "module_warming": 1 })
17298        );
17299    }
17300
17301    const OUTAGE_START: &str = "route.open refusing module: not serving";
17302    const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
17303
17304    fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
17305        capture
17306            .events()
17307            .into_iter()
17308            .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
17309            .collect()
17310    }
17311
17312    fn supervise_stub(
17313        registry: &Arc<Registry>,
17314        module_id: &str,
17315        enabled: bool,
17316    ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
17317        let supervisor_handle = SupervisorHandle::new();
17318        let supervisor =
17319            Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
17320                .with_handle(supervisor_handle.clone())
17321                .with_connection_file_path(std::env::temp_dir().join(format!(
17322                    "subc-route-outage-{module_id}-{}",
17323                    std::process::id()
17324                )));
17325        let module = supervisor
17326            .supervise_configured(
17327                ModuleSpec {
17328                    script: None,
17329                    carry_containment: false,
17330                    module_id: module_id.to_string(),
17331                    program: fake_aft_stub_path(),
17332                    args: Vec::new(),
17333                    env: Vec::new(),
17334                    reserved: false,
17335                    reserved_prefixes: Vec::new(),
17336                    protocol: ModuleProtocol::Subc,
17337                    overlap: Default::default(),
17338                },
17339                enabled,
17340            )
17341            .unwrap();
17342        (supervisor_handle, module)
17343    }
17344
17345    fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
17346        let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
17347            module_id: module_id.to_string(),
17348            drain_timeout_ms: Some(50),
17349        })
17350        .unwrap();
17351        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
17352    }
17353
17354    /// Two handlers built over one forwarding table must share one outage
17355    /// tracker; separate trackers would each log their own opening line for
17356    /// the same outage.
17357    #[test]
17358    fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
17359        let registry = Arc::new(Registry::default());
17360        let forwarding = Arc::new(ForwardingTable::default());
17361        let first = ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
17362        let second = ControlHandler::with_forwarding(registry, forwarding);
17363        assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
17364    }
17365
17366    /// A client can name any module id it likes. Refusing an unknown one,
17367    /// however often, must not create outage state or outage lines, or the
17368    /// tracker would be a memory sink any client could fill.
17369    #[tokio::test(flavor = "current_thread")]
17370    async fn route_open_unknown_module_refusals_add_no_outage_state() {
17371        let handler = ControlHandler::new(Arc::new(Registry::default()));
17372        let capture = EventCapture::default();
17373        let _subscriber =
17374            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17375        let (ctx, _rx) = route_ctx(ConnectionId::new(90));
17376        for corr in 0..8 {
17377            let response = handler
17378                .handle_control_frame(
17379                    &ctx,
17380                    route_open_frame(
17381                        380 + corr,
17382                        &format!("nobody-{corr}"),
17383                        unique_project_root("outage-unknown"),
17384                    ),
17385                )
17386                .await
17387                .unwrap();
17388            assert_eq!(parse_error(&response[0])["code"], "unknown_module");
17389        }
17390
17391        assert_eq!(handler.route_outages.tracked_module_count(), 0);
17392        assert!(outage_lines(&capture, OUTAGE_START).is_empty());
17393        assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
17394    }
17395
17396    /// Drives the refusal path end to end: a supervised module that served
17397    /// before and stopped being registered with no instruction to stop is a
17398    /// WARN, and the same module refused after an operator `supervisor.restart`
17399    /// is an INFO.
17400    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17401    async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
17402        let registry = Arc::new(Registry::default());
17403        let (supervisor_handle, module) = supervise_stub(&registry, "outage-restart", true);
17404        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
17405        let capture = EventCapture::default();
17406        let _subscriber =
17407            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17408        let (ctx, _rx) = route_ctx(ConnectionId::new(91));
17409        // The stub never registers, so pretend it served once: otherwise every
17410        // refusal would fall in its startup window.
17411        handler.route_outages.record_accepted("outage-restart");
17412
17413        let response = handler
17414            .handle_control_frame(
17415                &ctx,
17416                route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
17417            )
17418            .await
17419            .unwrap();
17420        assert_eq!(response[0].header.ty, FrameType::Error);
17421        let starts = outage_lines(&capture, OUTAGE_START);
17422        assert_eq!(starts.len(), 1, "{starts:?}");
17423        assert_eq!(starts[0].level, tracing::Level::WARN);
17424        assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
17425        assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
17426        assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
17427        handler.route_outages.record_accepted("outage-restart");
17428        assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
17429
17430        let restart = handler
17431            .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
17432            .await
17433            .unwrap();
17434        assert_eq!(
17435            restart[0].header.ty,
17436            FrameType::Response,
17437            "{:?}",
17438            parse_error(&restart[0])
17439        );
17440        handler
17441            .handle_control_frame(
17442                &ctx,
17443                route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
17444            )
17445            .await
17446            .unwrap();
17447        module.stop().await.unwrap();
17448
17449        let starts = outage_lines(&capture, OUTAGE_START);
17450        assert_eq!(starts.len(), 2, "{starts:?}");
17451        assert_eq!(starts[1].level, tracing::Level::INFO);
17452        assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
17453    }
17454
17455    /// A restart refused before it touched the module (here: the module is
17456    /// disabled) must clear its operator mark, so the next real outage is
17457    /// still reported as a warning.
17458    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17459    async fn failed_operator_restart_leaves_no_operator_mark() {
17460        let registry = Arc::new(Registry::default());
17461        let (supervisor_handle, _module) = supervise_stub(&registry, "outage-disabled", false);
17462        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
17463        let capture = EventCapture::default();
17464        let _subscriber =
17465            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17466        let (ctx, _rx) = route_ctx(ConnectionId::new(92));
17467        handler.route_outages.record_accepted("outage-disabled");
17468
17469        let restart = handler
17470            .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
17471            .await
17472            .unwrap();
17473        assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
17474        assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
17475
17476        handler
17477            .handle_control_frame(
17478                &ctx,
17479                route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
17480            )
17481            .await
17482            .unwrap();
17483        let starts = outage_lines(&capture, OUTAGE_START);
17484        assert_eq!(starts.len(), 1, "{starts:?}");
17485        assert_eq!(starts[0].level, tracing::Level::WARN);
17486    }
17487
17488    #[tokio::test(flavor = "current_thread")]
17489    async fn route_open_unknown_module_escapes_target_module_id() {
17490        let handler = ControlHandler::new(Arc::new(Registry::default()));
17491        let capture = EventCapture::default();
17492        let _subscriber =
17493            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17494        let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
17495        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
17496        let response = handler
17497            .handle_control_frame(
17498                &ctx,
17499                route_open_frame(
17500                    395,
17501                    hostile_module_id,
17502                    unique_project_root("hostile-target-module-id"),
17503                ),
17504            )
17505            .await
17506            .unwrap();
17507
17508        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
17509        let event = capture
17510            .events()
17511            .into_iter()
17512            .find(|event| {
17513                event.target == "control"
17514                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
17515            })
17516            .expect("route.open unknown-module refusal event");
17517        let logged = event.fields.get("module_id").expect("module_id field");
17518        assert!(!logged.bytes().any(|byte| byte < 0x20));
17519        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
17520    }
17521
17522    /// A route.open refusal is the daemon's own answer, so its ERROR carries
17523    /// the daemon-origin bit on the frame the client receives.
17524    #[tokio::test(flavor = "current_thread")]
17525    async fn route_open_refusal_carries_daemon_origin() {
17526        let handler = ControlHandler::new(Arc::new(Registry::default()));
17527        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
17528        let response = handler
17529            .handle_control_frame(
17530                &ctx,
17531                route_open_frame(395, "nobody", unique_project_root("refusal-origin")),
17532            )
17533            .await
17534            .unwrap();
17535
17536        assert_eq!(response.len(), 1);
17537        assert_eq!(response[0].header.ty, FrameType::Error);
17538        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
17539        assert!(
17540            response[0].header.flags.is_daemon_origin(),
17541            "route.open refusal must carry daemon origin: {:?}",
17542            response[0].header.flags
17543        );
17544    }
17545
17546    /// The drain `module_reloading` refusal is daemon-synthesized for an
17547    /// endpoint that is reloading, so it carries the daemon-origin bit on the
17548    /// frame the client receives.
17549    #[tokio::test(flavor = "current_thread")]
17550    async fn route_open_to_draining_module_refuses_module_reloading_with_daemon_origin() {
17551        let registry = Arc::new(Registry::default());
17552        let forwarding = Arc::new(ForwardingTable::default());
17553        let handler =
17554            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
17555        let module_connection = ConnectionId::new(95);
17556        let (module_ctx, mut module_rx) = route_ctx(module_connection);
17557        hello_via_sink(
17558            &handler,
17559            &module_ctx,
17560            &mut module_rx,
17561            hello_frame("aft", PROTOCOL_VERSION, 395),
17562        )
17563        .await;
17564        forwarding
17565            .begin_module_drain("aft", RouteCloseReason::Restart)
17566            .unwrap()
17567            .expect("the module endpoint drains");
17568
17569        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(96));
17570        let response = handler
17571            .handle_control_frame(
17572                &client_ctx,
17573                route_open_frame(396, "aft", unique_project_root("drain-origin")),
17574            )
17575            .await
17576            .unwrap();
17577
17578        assert_eq!(response.len(), 1);
17579        assert_eq!(response[0].header.ty, FrameType::Error);
17580        assert_eq!(parse_error(&response[0])["code"], "module_reloading");
17581        assert!(
17582            response[0].header.flags.is_daemon_origin(),
17583            "drain module_reloading must carry daemon origin: {:?}",
17584            response[0].header.flags
17585        );
17586    }
17587
17588    /// A module that refuses a bind with a daemon code still cannot mint a
17589    /// daemon refusal: the relayed `module_reloading` body reaches the client
17590    /// with the daemon-origin bit clear.
17591    #[tokio::test(flavor = "current_thread")]
17592    async fn route_open_module_rejection_with_daemon_code_carries_no_daemon_origin() {
17593        let registry = Arc::new(Registry::default());
17594        let forwarding = Arc::new(ForwardingTable::default());
17595        let handler =
17596            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
17597        let module_connection = ConnectionId::new(95);
17598        let (module_ctx, mut module_rx) = route_ctx(module_connection);
17599        hello_via_sink(
17600            &handler,
17601            &module_ctx,
17602            &mut module_rx,
17603            hello_frame("aft", PROTOCOL_VERSION, 395),
17604        )
17605        .await;
17606
17607        let client_connection = ConnectionId::new(96);
17608        let (client_ctx, _client_rx) = route_ctx(client_connection);
17609        let (route_task, bind) = relay_route_open(
17610            &handler,
17611            client_connection,
17612            &client_ctx.egress,
17613            &mut module_rx,
17614            396,
17615            "aft",
17616            "spoofed-reloading",
17617        )
17618        .await;
17619        // The module sets the origin bit on its own rejection too: neither
17620        // the bit nor the daemon code may survive the relay as daemon-owned.
17621        let rejection = Frame::build(
17622            FrameType::Error,
17623            crate::router::daemon_error_flags(),
17624            0,
17625            0,
17626            bind.header.corr,
17627            serde_json::to_vec(&ErrorBody::new(
17628                "module_reloading",
17629                "module refused route.bind",
17630            ))
17631            .unwrap(),
17632        )
17633        .unwrap();
17634        handler
17635            .handle_control_frame(&module_ctx, rejection)
17636            .await
17637            .unwrap();
17638
17639        let response = route_task.await.unwrap();
17640        assert_eq!(response.len(), 1);
17641        assert_eq!(response[0].header.ty, FrameType::Error);
17642        assert_eq!(parse_error(&response[0])["code"], "module_reloading");
17643        assert!(
17644            !response[0].header.flags.is_daemon_origin(),
17645            "a module's own rejection must not carry daemon origin: {:?}",
17646            response[0].header.flags
17647        );
17648    }
17649
17650    #[tokio::test(flavor = "current_thread")]
17651    async fn route_open_module_rejection_uses_daemon_counter_key() {
17652        let registry = Arc::new(Registry::default());
17653        let forwarding = Arc::new(ForwardingTable::default());
17654        let handler =
17655            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
17656        let module_connection = ConnectionId::new(95);
17657        let (module_ctx, mut module_rx) = route_ctx(module_connection);
17658        hello_via_sink(
17659            &handler,
17660            &module_ctx,
17661            &mut module_rx,
17662            hello_frame("aft", PROTOCOL_VERSION, 395),
17663        )
17664        .await;
17665
17666        let client_connection = ConnectionId::new(96);
17667        let (client_ctx, _client_rx) = route_ctx(client_connection);
17668        let capture = EventCapture::default();
17669        // tracing-core decides once per callsite whether its events are enabled,
17670        // and caches the answer. If a parallel test with no subscriber reaches
17671        // this callsite first, the cached answer is "disabled" and this test's
17672        // capture sees nothing. Keeping a second dispatch alive makes tracing-core
17673        // ask every live subscriber instead, while only this test's default
17674        // receives the events.
17675        static INTEREST_ANCHOR: std::sync::OnceLock<tracing::Dispatch> = std::sync::OnceLock::new();
17676        INTEREST_ANCHOR.get_or_init(|| tracing::Dispatch::new(tracing_subscriber::registry()));
17677        let _subscriber =
17678            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17679        let (route_task, bind) = relay_route_open(
17680            &handler,
17681            client_connection,
17682            &client_ctx.egress,
17683            &mut module_rx,
17684            396,
17685            "aft",
17686            "hostile-module-code",
17687        )
17688        .await;
17689        let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
17690        let rejection = Frame::build(
17691            FrameType::Error,
17692            control_flags(),
17693            0,
17694            0,
17695            bind.header.corr,
17696            serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
17697        )
17698        .unwrap();
17699        handler
17700            .handle_control_frame(&module_ctx, rejection)
17701            .await
17702            .unwrap();
17703
17704        let response = route_task.await.unwrap();
17705        assert_eq!(parse_error(&response[0])["code"], hostile_code);
17706        let counters = handler.counters().snapshot();
17707        assert_eq!(
17708            counters["route_open_refused_by_code"],
17709            json!({ "module_rejected": 1 })
17710        );
17711        assert!(counters["route_open_refused_by_code"]
17712            .get(hostile_code)
17713            .is_none());
17714
17715        let event = capture
17716            .events()
17717            .into_iter()
17718            .find(|event| {
17719                event.target == "control"
17720                    && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
17721            })
17722            .expect("route.open module-rejection refusal event");
17723        let logged = event.fields.get("module_code").expect("module_code field");
17724        assert!(!logged.bytes().any(|byte| byte < 0x20));
17725        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
17726    }
17727
17728    #[tokio::test]
17729    async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
17730        let registry = Arc::new(Registry::default());
17731        let supervisor_handle = SupervisorHandle::new();
17732        let missing_program = std::env::temp_dir().join(format!(
17733            "subc-route-open-missing-program-{}",
17734            std::process::id()
17735        ));
17736        let supervisor =
17737            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
17738                .with_handle(supervisor_handle.clone());
17739        let module = supervisor
17740            .supervise_configured(
17741                ModuleSpec {
17742                    script: None,
17743                    carry_containment: false,
17744                    module_id: "failed".to_string(),
17745                    program: missing_program,
17746                    args: Vec::new(),
17747                    env: Vec::new(),
17748                    reserved: false,
17749                    reserved_prefixes: Vec::new(),
17750                    protocol: ModuleProtocol::Subc,
17751                    overlap: Default::default(),
17752                },
17753                true,
17754            )
17755            .unwrap();
17756        assert_eq!(module.state().unwrap(), ModuleState::Failed);
17757
17758        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
17759        let (ctx, _rx) = route_ctx(ConnectionId::new(40));
17760        let response = handler
17761            .handle_control_frame(
17762                &ctx,
17763                route_open_frame(305, "failed", unique_project_root("failed")),
17764            )
17765            .await
17766            .unwrap();
17767
17768        assert_eq!(response[0].header.ty, FrameType::Error);
17769        let error = parse_error(&response[0]);
17770        assert_eq!(error["code"], "target_unavailable");
17771        assert!(error["message"]
17772            .as_str()
17773            .unwrap()
17774            .contains("state=failed, enabled=true, live=false"));
17775    }
17776
17777    #[tokio::test]
17778    async fn route_open_role_mismatch_remains_target_unavailable() {
17779        let registry = Arc::new(Registry::default());
17780        let handler = ControlHandler::new(Arc::clone(&registry));
17781        handler
17782            .handle_control(
17783                ConnectionId::new(41),
17784                non_routable_hello_frame_with_control_ops("health-only", 306, None),
17785            )
17786            .unwrap();
17787
17788        let (ctx, _rx) = route_ctx(ConnectionId::new(42));
17789        let response = handler
17790            .handle_control_frame(
17791                &ctx,
17792                route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
17793            )
17794            .await
17795            .unwrap();
17796
17797        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
17798        assert!(parse_error(&response[0])["message"]
17799            .as_str()
17800            .unwrap()
17801            .contains("does not provide the requested target"));
17802    }
17803
17804    #[tokio::test]
17805    async fn route_open_inactive_registration_remains_target_unavailable() {
17806        let registry = Arc::new(Registry::default());
17807        let handler = ControlHandler::new(Arc::clone(&registry));
17808        handler
17809            .handle_control(
17810                ConnectionId::new(43),
17811                hello_frame("inactive", PROTOCOL_VERSION, 308),
17812            )
17813            .unwrap();
17814        assert!(registry
17815            .set_module_state_for_test("inactive", ChannelState::Closed)
17816            .unwrap());
17817
17818        let (ctx, _rx) = route_ctx(ConnectionId::new(44));
17819        let response = handler
17820            .handle_control_frame(
17821                &ctx,
17822                route_open_frame(309, "inactive", unique_project_root("inactive")),
17823            )
17824            .await
17825            .unwrap();
17826
17827        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
17828        assert!(parse_error(&response[0])["message"]
17829            .as_str()
17830            .unwrap()
17831            .contains("is not active"));
17832    }
17833
17834    #[tokio::test]
17835    async fn late_health_reply_is_recorded_through_the_module_response_path() {
17836        let registry = Arc::new(Registry::default());
17837        let forwarding = Arc::new(ForwardingTable::default());
17838        let supervisor_handle = SupervisorHandle::new();
17839        let supervisor =
17840            Supervisor::new_for_test(Arc::clone(&registry), crate::RestartPolicy::default())
17841                .with_forwarding(Arc::clone(&forwarding))
17842                .with_handle(supervisor_handle.clone());
17843        let module = supervisor
17844            .supervise_configured(
17845                crate::ModuleSpec {
17846                    script: None,
17847                    carry_containment: false,
17848                    module_id: "late-health-response".to_string(),
17849                    program: PathBuf::from("disabled-module"),
17850                    args: Vec::new(),
17851                    env: Vec::new(),
17852                    reserved: false,
17853                    reserved_prefixes: Vec::new(),
17854                    protocol: ModuleProtocol::Subc,
17855                    overlap: Default::default(),
17856                },
17857                false,
17858            )
17859            .unwrap();
17860        let handler =
17861            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
17862                .with_supervisor(supervisor_handle);
17863        let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
17864        handler
17865            .handle_control_frame(
17866                &module_ctx,
17867                hello_frame_with_control_ops(
17868                    "late-health-response",
17869                    PROTOCOL_VERSION,
17870                    7,
17871                    Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
17872                ),
17873            )
17874            .await
17875            .unwrap();
17876        let probe_started_at = Instant::now() - Duration::from_millis(80);
17877        let pending = forwarding
17878            .begin_health_probe_rpc_for(
17879                "late-health-response",
17880                MODULE_CONTROL_OP_HEALTH_CHECK,
17881                probe_started_at,
17882                Instant::now() - Duration::from_millis(1),
17883            )
17884            .unwrap();
17885        assert!(forwarding
17886            .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
17887            .unwrap());
17888
17889        let responses = handler
17890            .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
17891            .await
17892            .unwrap();
17893
17894        assert!(responses.is_empty());
17895        let health = module.status().unwrap().health;
17896        assert_eq!(health.late_answer_count, 1);
17897        assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
17898    }
17899
17900    #[tokio::test]
17901    async fn health_probe_timeout_and_module_death_are_typed() {
17902        let registry = Arc::new(Registry::default());
17903        let forwarding = Arc::new(ForwardingTable::default());
17904        let handler =
17905            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
17906                .with_health_probe_timeout(Duration::from_millis(50));
17907        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
17908        hello_via_sink(
17909            &handler,
17910            &module_ctx,
17911            &mut module_rx,
17912            hello_frame_with_control_ops(
17913                "aft",
17914                PROTOCOL_VERSION,
17915                7,
17916                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
17917            ),
17918        )
17919        .await;
17920
17921        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
17922        let responses = handler
17923            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
17924            .await
17925            .unwrap();
17926        assert_eq!(responses[0].header.ty, FrameType::Error);
17927        assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
17928        let _ = module_rx.try_recv();
17929
17930        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
17931        let health_handler = handler.clone();
17932        let death_task = tokio::spawn(async move {
17933            health_handler
17934                .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
17935                .await
17936                .unwrap()
17937        });
17938        tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
17939            .await
17940            .unwrap()
17941            .unwrap();
17942        handler
17943            .cleanup_connection(module_ctx.connection_id)
17944            .unwrap();
17945        let responses = death_task.await.unwrap();
17946        assert_eq!(responses[0].header.ty, FrameType::Error);
17947        assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
17948    }
17949
17950    #[test]
17951    fn hello_requires_exact_protocol_version() {
17952        for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
17953            let registry = Arc::new(Registry::default());
17954            let handler = ControlHandler::new(Arc::clone(&registry));
17955            let responses = handler
17956                .handle_control(
17957                    ConnectionId::new(connection),
17958                    hello_frame("aft", offered, 9),
17959                )
17960                .unwrap();
17961
17962            assert_eq!(responses.len(), 1);
17963            assert_eq!(responses[0].header.ty, FrameType::Error);
17964            let error = parse_error(&responses[0]);
17965            assert_eq!(error["code"], "version_unsupported");
17966            assert!(registry.get_module("aft").unwrap().is_none());
17967            assert_eq!(registry.active_registration_count().unwrap(), 0);
17968        }
17969    }
17970
17971    #[test]
17972    fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
17973        let registry = Arc::new(Registry::default());
17974        let forwarding = Arc::new(ForwardingTable::default());
17975        let handler =
17976            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
17977        let module_connection = ConnectionId::new(301);
17978        let registration = registry
17979            .register_with_control_ops(
17980                manifest("aft-push", PROTOCOL_VERSION),
17981                PROTOCOL_VERSION,
17982                module_connection,
17983                module_baseline_control_ops(),
17984            )
17985            .unwrap();
17986        let (module_tx, _module_rx) = mpsc::channel(8);
17987        let endpoint = forwarding
17988            .register_module_connection(
17989                module_connection,
17990                "aft-push".to_string(),
17991                PROTOCOL_VERSION,
17992                manifest_concurrency(&registration.manifest),
17993                FrameSink::new(module_tx),
17994            )
17995            .unwrap();
17996
17997        // A push op this version does not know is ignored (forward-compat), not errored.
17998        let unknown = Frame::build(
17999            FrameType::Push,
18000            control_flags(),
18001            0,
18002            0,
18003            5,
18004            serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
18005        )
18006        .unwrap();
18007        let out = handler.handle_status_update(endpoint, unknown).unwrap();
18008        assert!(
18009            out.is_empty(),
18010            "unknown push op must be ignored, got {out:?}"
18011        );
18012
18013        // A malformed body for a KNOWN op is a real error worth surfacing.
18014        let malformed = Frame::build(
18015            FrameType::Push,
18016            control_flags(),
18017            0,
18018            0,
18019            6,
18020            serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
18021        )
18022        .unwrap();
18023        let out = handler.handle_status_update(endpoint, malformed).unwrap();
18024        assert_eq!(out.len(), 1);
18025        assert_eq!(out[0].header.ty, FrameType::Error);
18026        assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
18027    }
18028
18029    #[test]
18030    fn hello_rejected_when_connection_already_owns_client_routes() {
18031        let registry = Arc::new(Registry::default());
18032        let forwarding = Arc::new(ForwardingTable::default());
18033        let handler =
18034            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
18035        // Commits a client route on connection 202 (bound to a module on conn 101).
18036        let _ = bind_liveness_route(&registry, &forwarding, "aft-module");
18037        let client_connection = ConnectionId::new(202);
18038
18039        // That same connection now tries to register as a module: rejected, so one
18040        // connection never holds both client-route and module-endpoint state.
18041        let responses = handler
18042            .handle_control(
18043                client_connection,
18044                hello_frame("aft-second", PROTOCOL_VERSION, 9),
18045            )
18046            .unwrap();
18047        assert_eq!(responses[0].header.ty, FrameType::Error);
18048        assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
18049        assert!(registry.get_module("aft-second").unwrap().is_none());
18050    }
18051
18052    #[tokio::test]
18053    async fn second_hello_preserves_registration_routes_and_launch_nonce() {
18054        let registry = Arc::new(Registry::default());
18055        let forwarding = Arc::new(ForwardingTable::default());
18056        let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
18057        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
18058        hello_via_sink(
18059            &handler,
18060            &module_ctx,
18061            &mut module_rx,
18062            hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
18063        )
18064        .await;
18065        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
18066        let pending = forwarding
18067            .begin_route_bind_relay_for_test(
18068                client_ctx.connection_id,
18069                client_ctx.egress.clone(),
18070                2,
18071                "alpha",
18072            )
18073            .unwrap();
18074        forwarding
18075            .complete_pending_relay(
18076                module_ctx.connection_id,
18077                pending.corr,
18078                RouteBindRelayOutcome::Accepted,
18079            )
18080            .unwrap();
18081        client_rx.try_recv().unwrap();
18082        for module_id in ["beta", "alpha"] {
18083            let replies = handler
18084                .handle_control_frame(
18085                    &module_ctx,
18086                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
18087                )
18088                .await
18089                .unwrap();
18090            assert_eq!(replies.len(), 1, "second HELLO must be refused");
18091            assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
18092        }
18093        assert_eq!(registry.list_modules().unwrap().1.len(), 1);
18094        assert!(registry.get_module("beta").unwrap().is_none());
18095        assert!(matches!(
18096            forwarding
18097                .lookup_data_route(
18098                    client_ctx.connection_id,
18099                    pending.client_channel,
18100                    pending.client_epoch,
18101                )
18102                .unwrap(),
18103            DataRoute::Client(DataRouteState::Bound(_))
18104        ));
18105        assert!(handler
18106            .hello_launch_nonces
18107            .lock()
18108            .unwrap()
18109            .presented(module_ctx.connection_id, Some("alpha-nonce")));
18110        assert!(module_rx.try_recv().is_err());
18111    }
18112
18113    #[test]
18114    fn reserved_module_hello_requires_matching_launch_nonce() {
18115        let registry = Arc::new(Registry::default());
18116        let supervisor = SupervisorHandle::new();
18117        // The supervisor recorded the nonce it injected when it spawned the reserved
18118        // module; the HELLO verifier checks against the same shared handle.
18119        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
18120        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
18121
18122        // A HELLO with NO nonce is rejected.
18123        let no_nonce = handler
18124            .handle_control(
18125                ConnectionId::new(1),
18126                hello_frame("vault", PROTOCOL_VERSION, 1),
18127            )
18128            .unwrap();
18129        assert_eq!(no_nonce[0].header.ty, FrameType::Error);
18130        assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
18131        assert!(registry.get_module("vault").unwrap().is_none());
18132
18133        // A HELLO with the WRONG nonce is rejected.
18134        let wrong = handler
18135            .handle_control(
18136                ConnectionId::new(2),
18137                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
18138            )
18139            .unwrap();
18140        assert_eq!(wrong[0].header.ty, FrameType::Error);
18141        assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
18142        assert!(registry.get_module("vault").unwrap().is_none());
18143
18144        // A HELLO with the CORRECT nonce registers.
18145        let ok = handler
18146            .handle_control(
18147                ConnectionId::new(3),
18148                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
18149            )
18150            .unwrap();
18151        assert_eq!(ok[0].header.ty, FrameType::HelloAck);
18152        assert!(registry.get_module("vault").unwrap().is_some());
18153    }
18154
18155    #[test]
18156    fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
18157        let registry = Arc::new(Registry::default());
18158        let supervisor = SupervisorHandle::new();
18159        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
18160        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
18161        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
18162
18163        let squat = handler
18164            .handle_control(
18165                ConnectionId::new(1),
18166                hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
18167            )
18168            .unwrap();
18169        assert_eq!(squat[0].header.ty, FrameType::Error);
18170        assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
18171        assert!(parse_error(&squat[0])["message"]
18172            .as_str()
18173            .unwrap()
18174            .contains("fed:"));
18175
18176        let accepted_peer = handler
18177            .handle_control(
18178                ConnectionId::new(2),
18179                hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
18180            )
18181            .unwrap();
18182        assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
18183
18184        let accepted_short = handler
18185            .handle_control(
18186                ConnectionId::new(3),
18187                hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
18188            )
18189            .unwrap();
18190        assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
18191
18192        for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
18193            let response = handler
18194                .handle_control(
18195                    ConnectionId::new(conn),
18196                    hello_frame(module_id, PROTOCOL_VERSION, conn),
18197                )
18198                .unwrap();
18199            assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
18200        }
18201    }
18202
18203    #[test]
18204    fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
18205        let registry = Arc::new(Registry::default());
18206        let supervisor = SupervisorHandle::new();
18207        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
18208        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
18209        supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
18210        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
18211
18212        let owner_nonce = handler
18213            .handle_control(
18214                ConnectionId::new(1),
18215                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
18216            )
18217            .unwrap();
18218        assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
18219        assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
18220        assert!(registry.get_module("fed:special").unwrap().is_none());
18221
18222        let exact_nonce = handler
18223            .handle_control(
18224                ConnectionId::new(2),
18225                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
18226            )
18227            .unwrap();
18228        assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
18229        assert!(registry.get_module("fed:special").unwrap().is_some());
18230    }
18231
18232    #[test]
18233    fn non_reserved_module_ignores_launch_nonce() {
18234        let registry = Arc::new(Registry::default());
18235        // No reserved nonce recorded for these ids: they are not reserved, so HELLO
18236        // registration succeeds whether a spawned process echoes a nonce or not.
18237        let handler = ControlHandler::new(Arc::clone(&registry));
18238        let no_nonce = handler
18239            .handle_control(
18240                ConnectionId::new(1),
18241                hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
18242            )
18243            .unwrap();
18244        assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
18245        assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
18246
18247        let echoed_nonce = handler
18248            .handle_control(
18249                ConnectionId::new(2),
18250                hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
18251            )
18252            .unwrap();
18253        assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
18254        assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
18255    }
18256
18257    #[test]
18258    fn malformed_hello_returns_error_and_handler_still_answers_ping() {
18259        let handler = ControlHandler::default();
18260        let conn = ConnectionId::new(1);
18261        let malformed = Frame::build(
18262            FrameType::Hello,
18263            control_flags(),
18264            0,
18265            0,
18266            3,
18267            b"{not json".to_vec(),
18268        )
18269        .unwrap();
18270
18271        let error = handler.handle_control(conn, malformed).unwrap();
18272        assert_eq!(error[0].header.ty, FrameType::Error);
18273        assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
18274
18275        let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
18276        let pong = handler.handle_control(conn, ping).unwrap();
18277        assert_eq!(pong[0].header.ty, FrameType::Pong);
18278        assert_eq!(pong[0].header.corr, 4);
18279    }
18280
18281    #[test]
18282    fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
18283        let registry = Arc::new(Registry::default());
18284        let handler = ControlHandler::new(Arc::clone(&registry));
18285
18286        handler
18287            .handle_control(
18288                ConnectionId::new(1),
18289                hello_frame("aft", PROTOCOL_VERSION, 1),
18290            )
18291            .unwrap();
18292        let duplicate = handler
18293            .handle_control(
18294                ConnectionId::new(2),
18295                hello_frame("aft", PROTOCOL_VERSION, 2),
18296            )
18297            .unwrap();
18298
18299        assert_eq!(duplicate[0].header.ty, FrameType::Error);
18300        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
18301        let registration = registry.get_module("aft").unwrap().unwrap();
18302        assert_eq!(registration.connection_id, ConnectionId::new(1));
18303    }
18304
18305    #[test]
18306    fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
18307        let registry = Arc::new(Registry::default());
18308        let forwarding = Arc::new(ForwardingTable::default());
18309        let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
18310        let handler =
18311            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
18312                .with_process_liveness(process_liveness);
18313        let (ctx, route_channel, route_epoch) =
18314            bind_liveness_route(&registry, &forwarding, "aft-dead");
18315        let responses = handler
18316            .handle_route_poll(
18317                &ctx,
18318                route_poll_frame(41, PollKind::Liveness, route_channel),
18319                route_channel,
18320                route_epoch,
18321                PollKind::Liveness,
18322            )
18323            .unwrap();
18324
18325        assert_eq!(responses.len(), 1);
18326        assert_eq!(responses[0].header.ty, FrameType::Response);
18327        assert_route_poll_liveness(&responses[0], false);
18328    }
18329
18330    #[test]
18331    fn liveness_poll_without_process_source_uses_bound_route() {
18332        let registry = Arc::new(Registry::default());
18333        let forwarding = Arc::new(ForwardingTable::default());
18334        let handler =
18335            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
18336        let (ctx, route_channel, route_epoch) =
18337            bind_liveness_route(&registry, &forwarding, "aft-bound-only");
18338        let responses = handler
18339            .handle_route_poll(
18340                &ctx,
18341                route_poll_frame(42, PollKind::Liveness, route_channel),
18342                route_channel,
18343                route_epoch,
18344                PollKind::Liveness,
18345            )
18346            .unwrap();
18347
18348        assert_route_poll_liveness(&responses[0], true);
18349    }
18350
18351    #[test]
18352    fn liveness_poll_untracked_process_source_uses_bound_route() {
18353        let registry = Arc::new(Registry::default());
18354        let forwarding = Arc::new(ForwardingTable::default());
18355        let process_liveness = Arc::new(FakeProcessLiveness { live: None });
18356        let handler =
18357            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
18358                .with_process_liveness(process_liveness);
18359        let (ctx, route_channel, route_epoch) =
18360            bind_liveness_route(&registry, &forwarding, "aft-untracked");
18361        let responses = handler
18362            .handle_route_poll(
18363                &ctx,
18364                route_poll_frame(43, PollKind::Liveness, route_channel),
18365                route_channel,
18366                route_epoch,
18367                PollKind::Liveness,
18368            )
18369            .unwrap();
18370
18371        assert_route_poll_liveness(&responses[0], true);
18372    }
18373
18374    #[tokio::test]
18375    async fn unknown_op_returns_unknown_control_op() {
18376        let handler = ControlHandler::default();
18377        let (ctx, _rx) = route_ctx(ConnectionId::new(77));
18378        let request = Frame::build(
18379            FrameType::Request,
18380            control_flags(),
18381            0,
18382            0,
18383            55,
18384            br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
18385        )
18386        .unwrap();
18387
18388        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
18389
18390        assert_eq!(response.len(), 1);
18391        assert_eq!(response[0].header.ty, FrameType::Error);
18392        assert_eq!(response[0].header.corr, 55);
18393        assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
18394    }
18395
18396    #[tokio::test]
18397    async fn supervisor_provenance_rejects_unknown_exact_module() {
18398        let handler = ControlHandler::default();
18399        let (ctx, _rx) = route_ctx(ConnectionId::new(79));
18400        let request = Frame::build(
18401            FrameType::Request,
18402            control_flags(),
18403            0,
18404            0,
18405            57,
18406            br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
18407        )
18408        .unwrap();
18409
18410        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
18411
18412        assert_eq!(response.len(), 1);
18413        assert_eq!(response[0].header.ty, FrameType::Error);
18414        assert_eq!(response[0].header.corr, 57);
18415        let error = parse_error(&response[0]);
18416        assert_eq!(error["code"], "unknown_module");
18417        assert_eq!(error["message"], "module_id 'missing' is not supervised");
18418    }
18419
18420    #[test]
18421    fn provenance_probe_override_keeps_handler_tests_deterministic() {
18422        let expected = subc_control::RunningImageAgreement::Unavailable {
18423            reason: subc_control::RunningImageUnavailableReason::HashFailed,
18424        };
18425        let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
18426        assert_eq!(handler.provenance_probe_override, Some(expected));
18427    }
18428
18429    #[test]
18430    fn reload_verdict_detects_configured_program_different_from_spawned_path() {
18431        let verdict = reload_verdict(
18432            std::path::Path::new("/bin/new"),
18433            Some(std::path::Path::new("/bin/old")),
18434            subc_control::RunningImageAgreement::Unavailable {
18435                reason: subc_control::RunningImageUnavailableReason::HashFailed,
18436            },
18437        );
18438        assert!(matches!(
18439            verdict.path,
18440            subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
18441                if configured == std::path::Path::new("/bin/new")
18442                    && spawned_from == std::path::Path::new("/bin/old")
18443        ));
18444    }
18445
18446    #[test]
18447    fn launch_spec_without_running_digest_is_unavailable_not_match() {
18448        use subc_control::{
18449            LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
18450        };
18451        let configured = crate::supervise::ModuleSpec {
18452            script: None,
18453            carry_containment: false,
18454            module_id: "legacy".into(),
18455            program: "/module".into(),
18456            args: Vec::new(),
18457            env: Vec::new(),
18458            reserved: false,
18459            reserved_prefixes: Vec::new(),
18460            protocol: subc_control::ModuleProtocol::Subc,
18461            overlap: Default::default(),
18462        };
18463        let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
18464        assert_eq!(
18465            launch_spec_agreement(&configured, false, Some(&digest)),
18466            Agreement::Unavailable {
18467                reason: Reason::NotRunning
18468            }
18469        );
18470        assert_eq!(
18471            launch_spec_agreement(&configured, true, None),
18472            Agreement::Unavailable {
18473                reason: Reason::SpawnedSpecUnavailable
18474            }
18475        );
18476        assert_eq!(
18477            launch_spec_agreement(&configured, true, Some(&digest)),
18478            Agreement::Match
18479        );
18480    }
18481
18482    #[cfg(windows)]
18483    #[test]
18484    fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
18485        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
18486        let configured = dir.join("module.exe");
18487        std::fs::write(&configured, b"executable path fixture").unwrap();
18488        let spawned = std::fs::canonicalize(&configured).unwrap();
18489        assert_ne!(
18490            configured, spawned,
18491            "fixture must exercise different path spellings"
18492        );
18493        let forward_slashes = configured.to_string_lossy().replace('\\', "/");
18494        let lower_case = configured.to_string_lossy().to_lowercase();
18495        for spelling in [
18496            &configured,
18497            Path::new(&forward_slashes),
18498            Path::new(&lower_case),
18499        ] {
18500            let verdict = reload_verdict(
18501                spelling,
18502                Some(&spawned),
18503                subc_control::RunningImageAgreement::Unavailable {
18504                    reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
18505                },
18506            );
18507            assert_eq!(
18508                verdict.path,
18509                ReloadPathAgreement::Match,
18510                "configured {spelling:?}, spawned {spawned:?}"
18511            );
18512        }
18513    }
18514
18515    #[cfg(windows)]
18516    #[test]
18517    fn windows_reload_path_keeps_different_existing_program_pending() {
18518        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
18519        let old = dir.join("old.exe");
18520        let new = dir.join("new.exe");
18521        std::fs::write(&old, b"same executable content").unwrap();
18522        std::fs::copy(&old, &new).unwrap();
18523        let spawned = std::fs::canonicalize(&old).unwrap();
18524        let image = subc_control::RunningImageAgreement::Unavailable {
18525            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
18526        };
18527        let verdict = reload_verdict(&new, Some(&spawned), image.clone());
18528        assert!(
18529            matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
18530            if configured == new && spawned_from == spawned)
18531        );
18532        let missing = dir.join("missing.exe");
18533        assert!(matches!(
18534            reload_verdict(&missing, Some(&spawned), image).path,
18535            ReloadPathAgreement::Mismatch { .. }
18536        ));
18537    }
18538
18539    #[test]
18540    fn reload_verdict_detects_replaced_image_at_same_path() {
18541        let image = subc_control::RunningImageAgreement::Mismatch {
18542            running: subc_control::RunningImageEvidence::LinuxProcSha256 {
18543                digest: "old".into(),
18544            },
18545            disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
18546                digest: "new".into(),
18547            },
18548        };
18549        let verdict = reload_verdict(
18550            std::path::Path::new("/bin/same"),
18551            Some(std::path::Path::new("/bin/same")),
18552            image.clone(),
18553        );
18554        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
18555        assert_eq!(verdict.image, image);
18556    }
18557
18558    #[test]
18559    fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
18560        let image = subc_control::RunningImageAgreement::Unavailable {
18561            reason: subc_control::RunningImageUnavailableReason::NotRunning,
18562        };
18563        let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
18564        assert_eq!(
18565            verdict.path,
18566            subc_control::ReloadPathAgreement::Unavailable {
18567                reason: subc_control::ReloadPathUnavailableReason::NotRunning,
18568            }
18569        );
18570        assert_eq!(verdict.image, image);
18571
18572        let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
18573            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
18574        };
18575        let verdict = reload_verdict(
18576            std::path::Path::new("/bin/same"),
18577            Some(std::path::Path::new("/bin/same")),
18578            unconfirmed.clone(),
18579        );
18580        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
18581        assert_eq!(verdict.image, unconfirmed);
18582    }
18583
18584    #[test]
18585    fn reload_verdict_preserves_each_image_unavailability_reason() {
18586        use subc_control::RunningImageUnavailableReason as Reason;
18587
18588        for reason in [
18589            Reason::NotRunning,
18590            Reason::UnsupportedPlatform,
18591            Reason::RunningExecutableUnreadable,
18592            Reason::SpawnedPathUnreadable,
18593            Reason::HashFailed,
18594            Reason::ProcessIdentityUnconfirmed,
18595            Reason::Unknown("future_probe_reason".to_string()),
18596        ] {
18597            let image = subc_control::RunningImageAgreement::Unavailable {
18598                reason: reason.clone(),
18599            };
18600            let verdict = reload_verdict(
18601                std::path::Path::new("/bin/same"),
18602                Some(std::path::Path::new("/bin/same")),
18603                image.clone(),
18604            );
18605            assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
18606            assert_eq!(verdict.image, image, "{reason:?}");
18607        }
18608    }
18609
18610    #[tokio::test]
18611    async fn malformed_control_bodies_return_invalid_control_body() {
18612        let handler = ControlHandler::default();
18613        let (ctx, _rx) = route_ctx(ConnectionId::new(78));
18614
18615        for (corr, body) in [
18616            (56, br#"{"route_channel":1}"#.as_slice()),
18617            (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
18618            (
18619                58,
18620                br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
18621            ),
18622        ] {
18623            let request = Frame::build(
18624                FrameType::Request,
18625                control_flags(),
18626                0,
18627                0,
18628                corr,
18629                body.to_vec(),
18630            )
18631            .unwrap();
18632            let response = handler.handle_control_frame(&ctx, request).await.unwrap();
18633
18634            assert_eq!(response.len(), 1);
18635            assert_eq!(response[0].header.ty, FrameType::Error);
18636            assert_eq!(response[0].header.corr, corr);
18637            assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
18638        }
18639    }
18640
18641    #[tokio::test]
18642    async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
18643        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
18644        let registry = Arc::new(Registry::default());
18645        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
18646        let router = Router::with_control_handler(Arc::clone(&control));
18647        let connection = router.begin_connection();
18648        let (ctx, mut rx) = route_ctx(connection.id());
18649
18650        router
18651            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
18652            .await
18653            .unwrap();
18654        let response = rx.recv().await.unwrap();
18655        let ack = parse_ack(&response);
18656        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
18657        let channel = 1;
18658
18659        let goodbye =
18660            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
18661        router.route_for_connection(&ctx, goodbye).await.unwrap();
18662        assert!(rx.try_recv().is_err());
18663        assert!(registry.get_module("aft").unwrap().is_none());
18664
18665        router
18666            .route_for_connection(&ctx, channel_request(channel, 13))
18667            .await
18668            .unwrap();
18669        let error_frame = rx.recv().await.unwrap();
18670        assert_eq!(error_frame.header.ty, FrameType::Error);
18671        assert_eq!(error_frame.header.channel, channel);
18672        let captured = crate::router::test_log::captured_logs(&logs);
18673        assert_eq!(
18674            captured
18675                .lines()
18676                .filter(|line| {
18677                    line.contains("module_id=aft")
18678                        && line.contains("reason=explicit_goodbye")
18679                        && line.contains("module registration ended")
18680                })
18681                .count(),
18682            1,
18683            "unexpected GOODBYE registry log: {captured}"
18684        );
18685    }
18686
18687    #[tokio::test]
18688    async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
18689        let registry = Arc::new(Registry::default());
18690        let handler = ControlHandler::new(registry).with_capability_config(
18691            [("prov".to_string(), true), ("cons".to_string(), true)],
18692            BTreeMap::new(),
18693        );
18694        let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
18695        register_capability_manifest(
18696            &handler,
18697            &provider_ctx,
18698            &mut provider_rx,
18699            capability_manifest("prov", &["thing/v1"], &[]),
18700            1,
18701        )
18702        .await;
18703        let mut consumer = capability_manifest("cons", &[], &[]);
18704        consumer.capabilities.as_mut().unwrap().requires.push(
18705            subc_protocol::manifest::CapabilityRequirement {
18706                capability: "thing/v1".to_string(),
18707                need: subc_protocol::manifest::CapabilityNeed::Required,
18708            },
18709        );
18710        let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
18711        register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
18712        assert_eq!(
18713            handler.capability_evaluator.verdict("cons", "thing/v1"),
18714            Some(CapabilityVerdict::Provided)
18715        );
18716        let (mut client_rx, _) = open_route_for_capability_test(
18717            &handler,
18718            &provider_ctx,
18719            &mut provider_rx,
18720            703,
18721            3,
18722            "prov",
18723            None,
18724        )
18725        .await;
18726        let goodbye =
18727            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
18728        handler
18729            .handle_control_frame(&provider_ctx, goodbye)
18730            .await
18731            .unwrap();
18732        assert_eq!(
18733            handler.capability_evaluator.verdict("cons", "thing/v1"),
18734            Some(CapabilityVerdict::NeverProvided)
18735        );
18736        let closed = client_rx
18737            .try_recv()
18738            .expect("GOODBYE pushes route.closed before route GOODBYE");
18739        assert!(
18740            matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
18741            ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
18742        );
18743        assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
18744        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
18745    }
18746
18747    struct RouteClosedFixture {
18748        handler: ControlHandler,
18749        #[cfg(unix)]
18750        supervisor: Supervisor,
18751        module_ctx: RouteCtx,
18752        #[cfg(unix)]
18753        module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18754        client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18755        route: Arc<crate::forwarding::RouteBinding>,
18756    }
18757
18758    async fn route_closed_fixture() -> RouteClosedFixture {
18759        let registry = Arc::new(Registry::default());
18760        let handler = ControlHandler::new(Arc::clone(&registry));
18761        #[cfg(unix)]
18762        let supervisor = Supervisor::new(registry, RestartPolicy::default())
18763            .with_forwarding(Arc::clone(&handler.forwarding));
18764        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
18765        register_capability_manifest(
18766            &handler,
18767            &module_ctx,
18768            &mut module_rx,
18769            capability_manifest("prov", &[], &[]),
18770            1,
18771        )
18772        .await;
18773        let (client_rx, bind) = open_route_for_capability_test(
18774            &handler,
18775            &module_ctx,
18776            &mut module_rx,
18777            702,
18778            2,
18779            "prov",
18780            None,
18781        )
18782        .await;
18783        let ModuleControlRequest::RouteBind {
18784            route_channel: channel,
18785            epoch: route_epoch,
18786            ..
18787        } = bind
18788        else {
18789            panic!("expected route.bind");
18790        };
18791        let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
18792            handler
18793                .forwarding
18794                .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
18795                .unwrap()
18796        else {
18797            panic!("the acknowledged route must be bound");
18798        };
18799        RouteClosedFixture {
18800            handler,
18801            #[cfg(unix)]
18802            supervisor,
18803            module_ctx,
18804            #[cfg(unix)]
18805            module_rx,
18806            client_rx,
18807            route,
18808        }
18809    }
18810
18811    impl RouteClosedFixture {
18812        #[cfg(unix)]
18813        async fn cut(&mut self) {
18814            let notices = async {
18815                let mut draining = self.module_rx.recv().await.unwrap();
18816                assert!(matches!(
18817                    serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
18818                        &draining.body
18819                    )
18820                    .unwrap(),
18821                    subc_protocol::session::ModuleControlCommand::Draining {
18822                        reason: RouteCloseReason::Disable,
18823                        ..
18824                    }
18825                ));
18826                draining.flushed.take().unwrap().send(()).unwrap();
18827                let mut closing = self.client_rx.recv().await.unwrap();
18828                assert!(matches!(
18829                    serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
18830                    ClientControlPush::RouteClosing {
18831                        reason: RouteCloseReason::Restart,
18832                        ..
18833                    }
18834                ));
18835                closing.flushed.take().unwrap().send(()).unwrap();
18836            };
18837            let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
18838            result.unwrap();
18839        }
18840
18841        fn disconnect(&mut self) -> ClientControlPush {
18842            self.handler
18843                .cleanup_connection(self.module_ctx.connection_id)
18844                .unwrap();
18845            let closed = self.client_rx.try_recv().unwrap();
18846            assert_eq!(closed.header.ty, FrameType::Push);
18847            let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
18848            assert!(matches!(
18849                &push,
18850                ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
18851                    if channels == &[self.route.client_channel]
18852            ));
18853            assert_eq!(
18854                self.client_rx.try_recv().unwrap().header.ty,
18855                FrameType::Goodbye
18856            );
18857            assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
18858            push
18859        }
18860    }
18861
18862    #[cfg(unix)]
18863    #[tokio::test(start_paused = true)]
18864    async fn daemon_cut_quiet_routes_report_restart_and_drained() {
18865        let mut fixture = route_closed_fixture().await;
18866        fixture.cut().await;
18867        assert!(matches!(
18868            fixture.disconnect(),
18869            ClientControlPush::RouteClosed {
18870                reason: RouteCloseReason::Restart,
18871                drained: true,
18872                ..
18873            }
18874        ));
18875    }
18876
18877    #[cfg(unix)]
18878    #[tokio::test(start_paused = true)]
18879    async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
18880        let mut fixture = route_closed_fixture().await;
18881        fixture.route.flow.acquire_tagged(99, false).await.unwrap();
18882        let started = tokio::time::Instant::now();
18883        fixture.cut().await;
18884        assert!(started.elapsed() >= Duration::from_secs(2));
18885        assert_eq!(fixture.route.flow.in_flight(), 1);
18886        assert!(matches!(
18887            fixture.disconnect(),
18888            ClientControlPush::RouteClosed {
18889                reason: RouteCloseReason::Restart,
18890                drained: false,
18891                ..
18892            }
18893        ));
18894    }
18895
18896    #[tokio::test]
18897    async fn module_connection_crash_reports_crash_and_not_drained() {
18898        let mut fixture = route_closed_fixture().await;
18899        // The server calls cleanup_connection on module EOF. Without a planned
18900        // daemon shutdown, that disconnect must remain a crash with no prior
18901        // route.closing notice, even when the route has no in-flight calls.
18902        assert!(matches!(
18903            fixture.disconnect(),
18904            ClientControlPush::RouteClosed {
18905                reason: RouteCloseReason::Crash,
18906                drained: false,
18907                ..
18908            }
18909        ));
18910    }
18911
18912    #[tokio::test]
18913    async fn dropping_router_connection_releases_registration() {
18914        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
18915        let registry = Arc::new(Registry::default());
18916        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
18917        let router = Router::with_control_handler(Arc::clone(&control));
18918        let connection = router.begin_connection();
18919        let connection_id = connection.id();
18920        let (ctx, mut rx) = route_ctx(connection_id);
18921
18922        router
18923            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
18924            .await
18925            .unwrap();
18926        let response = rx.recv().await.unwrap();
18927        let ack = parse_ack(&response);
18928        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
18929        assert!(registry.get_module("aft").unwrap().is_some());
18930
18931        drop(connection);
18932
18933        assert!(registry.get_module("aft").unwrap().is_none());
18934        assert_eq!(registry.active_registration_count().unwrap(), 0);
18935
18936        control
18937            .cleanup_connection(ConnectionId::new(u64::MAX))
18938            .unwrap();
18939        let captured = crate::router::test_log::captured_logs(&logs);
18940        println!("captured registry lifecycle logs:\n{captured}");
18941        let events: Vec<_> = captured
18942            .lines()
18943            .filter(|line| {
18944                line.contains("module registered module_id=aft ")
18945                    || line.contains("module registration ended")
18946            })
18947            .collect();
18948        assert_eq!(
18949            events.len(),
18950            2,
18951            "unexpected registry lifecycle logs: {captured}"
18952        );
18953        assert!(events[0].contains("module registered module_id=aft "));
18954        assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
18955        assert!(events[1].contains(&format!(
18956            "module_id=aft connection_id={}",
18957            connection_id.get()
18958        )));
18959        assert!(events[1].contains("reason=connection_closed"));
18960        assert!(events[1].contains("module registration ended"));
18961        assert_eq!(
18962            captured
18963                .lines()
18964                .filter(|line| line.contains("module registered module_id=aft "))
18965                .count(),
18966            1,
18967            "legacy registration admission line must appear once: {captured}"
18968        );
18969    }
18970
18971    #[tokio::test]
18972    async fn hello_registration_keeps_legacy_module_registered_line_once() {
18973        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
18974        let registry = Arc::new(Registry::default());
18975        let control = ControlHandler::new(Arc::clone(&registry));
18976        let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
18977        hello_via_sink(
18978            &control,
18979            &ctx,
18980            &mut rx,
18981            hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
18982        )
18983        .await;
18984
18985        let captured = crate::router::test_log::captured_logs(&logs);
18986        let admissions: Vec<_> = captured
18987            .lines()
18988            .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
18989            .collect();
18990        assert_eq!(
18991            admissions.len(),
18992            1,
18993            "legacy admission line must remain exactly once: {captured}"
18994        );
18995        assert!(admissions[0].contains("routable_provider=true"));
18996        assert!(admissions[0].contains("connection_id=777"));
18997    }
18998
18999    fn capability_manifest(
19000        module_id: &str,
19001        provides: &[&str],
19002        must_never_reach: &[&str],
19003    ) -> ModuleManifest {
19004        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
19005        manifest.capabilities = Some(CapabilityDeclarations {
19006            provides: provides
19007                .iter()
19008                .map(|capability| (*capability).to_string())
19009                .collect(),
19010            requires: Vec::new(),
19011            must_never_reach: must_never_reach
19012                .iter()
19013                .map(|capability| (*capability).to_string())
19014                .collect(),
19015        });
19016        manifest
19017    }
19018
19019    fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
19020        Frame::build(
19021            FrameType::Hello,
19022            control_flags(),
19023            0,
19024            0,
19025            corr,
19026            serde_json::to_vec(&ModuleHelloBody {
19027                protocol_ver: manifest.protocol_ver,
19028                manifest,
19029                control_ops: None,
19030                launch_nonce: None,
19031            })
19032            .expect("capability test HELLO serializes"),
19033        )
19034        .expect("capability test HELLO frame builds")
19035    }
19036
19037    fn catalog_update_with_capabilities_frame(
19038        corr: u64,
19039        capabilities: CapabilityDeclarations,
19040    ) -> Frame {
19041        Frame::build(
19042            FrameType::Request,
19043            control_flags(),
19044            0,
19045            0,
19046            corr,
19047            serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
19048                provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
19049                capabilities: Some(capabilities),
19050                ready: None,
19051            })
19052            .expect("capability catalog.update serializes"),
19053        )
19054        .expect("capability catalog.update frame builds")
19055    }
19056
19057    async fn register_capability_manifest(
19058        handler: &ControlHandler,
19059        ctx: &RouteCtx,
19060        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
19061        manifest: ModuleManifest,
19062        corr: u64,
19063    ) {
19064        hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
19065    }
19066
19067    async fn open_route_for_capability_test(
19068        handler: &ControlHandler,
19069        target_ctx: &RouteCtx,
19070        target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
19071        client_connection_id: u64,
19072        corr: u64,
19073        target_module_id: &str,
19074        consumer_identity: Option<ConsumerIdentity>,
19075    ) -> (
19076        mpsc::Receiver<crate::router::OutboundFrame>,
19077        ModuleControlRequest,
19078    ) {
19079        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
19080        let route_handler = handler.clone();
19081        let target_module_id = target_module_id.to_string();
19082        let route_task = tokio::spawn(async move {
19083            route_handler
19084                .handle_control_frame(
19085                    &client_ctx,
19086                    route_open_frame_with_admission_facts(
19087                        corr,
19088                        &target_module_id,
19089                        unique_project_root("admission-facts"),
19090                        consumer_identity,
19091                        None,
19092                    ),
19093                )
19094                .await
19095                .expect("capability test route.open succeeds")
19096        });
19097        let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
19098            .await
19099            .expect("capability test route.open must reach route.bind")
19100            .expect("target control receiver stays open");
19101        let bind_request: ModuleControlRequest =
19102            serde_json::from_slice(&bind.body).expect("route.bind decodes");
19103        handler
19104            .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
19105            .await
19106            .expect("capability test route.bind ACK succeeds");
19107        assert!(route_task.await.expect("route.open task joins").is_empty());
19108        let opened = client_rx
19109            .recv()
19110            .await
19111            .expect("successful route.open publishes a response");
19112        assert!(matches!(
19113            serde_json::from_slice::<ClientControlResponse>(&opened.body),
19114            Ok(ClientControlResponse::RouteOpen { .. })
19115        ));
19116        (client_rx, bind_request)
19117    }
19118
19119    fn assert_capability_denied_push(frame: Frame, target_module_id: &str) -> u16 {
19120        assert_eq!(frame.header.ty, FrameType::Push);
19121        assert_eq!(frame.header.channel, 0);
19122        let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
19123            .expect("route.closed control push decodes");
19124        let ClientControlPush::RouteClosed { channels, .. } = &push else {
19125            panic!("expected route.closed");
19126        };
19127        assert_eq!(channels.len(), 1, "exactly one violating route closed");
19128        let closed_channel = channels[0];
19129        let channels = channels.clone();
19130        assert_eq!(
19131            push,
19132            ClientControlPush::RouteClosed {
19133                module_id: target_module_id.to_string(),
19134                channels,
19135                reason: RouteCloseReason::CapabilityDenied,
19136                drained: false,
19137                abandoned: 0,
19138                excluded_subscriptions: 0,
19139                terminal: Some(false),
19140            }
19141        );
19142        closed_channel
19143    }
19144
19145    #[tokio::test]
19146    async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
19147        let registry = Arc::new(Registry::default());
19148        let forwarding = Arc::new(ForwardingTable::default());
19149        let supervisor = SupervisorHandle::new();
19150        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
19151        let handler =
19152            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
19153                .with_supervisor(supervisor);
19154        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
19155        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
19156        register_capability_manifest(
19157            &handler,
19158            &target_ctx,
19159            &mut target_rx,
19160            capability_manifest("target", &["credentials-provider/v1"], &[]),
19161            1,
19162        )
19163        .await;
19164        register_capability_manifest(
19165            &handler,
19166            &opener_ctx,
19167            &mut opener_rx,
19168            capability_manifest("opener", &[], &["credentials-provider/v1"]),
19169            2,
19170        )
19171        .await;
19172
19173        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
19174        let replies = handler
19175            .handle_control_frame(
19176                &client_ctx,
19177                route_open_frame_with_admission_facts(
19178                    3,
19179                    "target",
19180                    unique_project_root("admission-facts"),
19181                    Some(ConsumerIdentity {
19182                        module_id: "opener".to_string(),
19183                        launch_nonce: "opener-nonce".to_string(),
19184                    }),
19185                    None,
19186                ),
19187            )
19188            .await
19189            .expect("denied route.open returns a typed frame");
19190        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
19191        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
19192        assert!(
19193            target_rx.try_recv().is_err(),
19194            "forbidden route.open must not relay route.bind"
19195        );
19196    }
19197
19198    #[tokio::test]
19199    async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
19200        let registry = Arc::new(Registry::default());
19201        let forwarding = Arc::new(ForwardingTable::default());
19202        let supervisor = SupervisorHandle::new();
19203        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
19204        let handler =
19205            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
19206                .with_supervisor(supervisor);
19207        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
19208        let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
19209        register_capability_manifest(
19210            &handler,
19211            &target_ctx,
19212            &mut target_rx,
19213            capability_manifest("target", &["credentials-provider/v1"], &[]),
19214            1,
19215        )
19216        .await;
19217        register_capability_manifest(
19218            &handler,
19219            &old_opener_ctx,
19220            &mut old_opener_rx,
19221            capability_manifest("opener", &[], &[]),
19222            2,
19223        )
19224        .await;
19225        let (mut client_rx, _) = open_route_for_capability_test(
19226            &handler,
19227            &target_ctx,
19228            &mut target_rx,
19229            712,
19230            3,
19231            "target",
19232            Some(ConsumerIdentity {
19233                module_id: "opener".to_string(),
19234                launch_nonce: "opener-nonce".to_string(),
19235            }),
19236        )
19237        .await;
19238        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
19239
19240        handler
19241            .cleanup_connection(old_opener_ctx.connection_id)
19242            .expect("old opener registration cleans up");
19243        let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
19244        register_capability_manifest(
19245            &handler,
19246            &new_opener_ctx,
19247            &mut new_opener_rx,
19248            capability_manifest("opener", &[], &["credentials-provider/v1"]),
19249            4,
19250        )
19251        .await;
19252
19253        let closed_channel = assert_capability_denied_push(
19254            client_rx
19255                .try_recv()
19256                .expect("HELLO deny addition must emit route.closed")
19257                .frame,
19258            "target",
19259        );
19260        let goodbye = client_rx
19261            .try_recv()
19262            .expect("HELLO deny addition must send the client a route GOODBYE")
19263            .frame;
19264        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
19265        assert_eq!(goodbye.header.channel, closed_channel);
19266        assert_ne!(goodbye.header.channel, 0);
19267        assert_ne!(goodbye.header.epoch, 0);
19268        assert!(client_rx.try_recv().is_err());
19269        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
19270        assert!(matches!(
19271            target_rx.try_recv(),
19272            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
19273        ));
19274    }
19275
19276    #[tokio::test]
19277    async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
19278        let registry = Arc::new(Registry::default());
19279        let forwarding = Arc::new(ForwardingTable::default());
19280        let supervisor = SupervisorHandle::new();
19281        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
19282        let handler =
19283            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
19284                .with_supervisor(supervisor);
19285        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
19286        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
19287        register_capability_manifest(
19288            &handler,
19289            &target_ctx,
19290            &mut target_rx,
19291            capability_manifest("target", &[], &[]),
19292            1,
19293        )
19294        .await;
19295        register_capability_manifest(
19296            &handler,
19297            &opener_ctx,
19298            &mut opener_rx,
19299            capability_manifest("opener", &[], &["credentials-provider/v1"]),
19300            2,
19301        )
19302        .await;
19303        let (mut client_rx, _) = open_route_for_capability_test(
19304            &handler,
19305            &target_ctx,
19306            &mut target_rx,
19307            722,
19308            3,
19309            "target",
19310            Some(ConsumerIdentity {
19311                module_id: "opener".to_string(),
19312                launch_nonce: "opener-nonce".to_string(),
19313            }),
19314        )
19315        .await;
19316        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
19317
19318        let replies = handler
19319            .handle_control_frame(
19320                &target_ctx,
19321                catalog_update_with_capabilities_frame(
19322                    4,
19323                    CapabilityDeclarations {
19324                        provides: vec!["credentials-provider/v1".to_string()],
19325                        requires: Vec::new(),
19326                        must_never_reach: Vec::new(),
19327                    },
19328                ),
19329            )
19330            .await
19331            .expect("claim catalog.update succeeds");
19332        assert!(matches!(
19333            serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
19334            Ok(ModuleControlResponseToModule::CatalogUpdate {})
19335        ));
19336        let closed_channel = assert_capability_denied_push(
19337            client_rx
19338                .try_recv()
19339                .expect("claim addition must emit route.closed")
19340                .frame,
19341            "target",
19342        );
19343        let goodbye = client_rx
19344            .try_recv()
19345            .expect("claim addition must send the client a route GOODBYE")
19346            .frame;
19347        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
19348        assert_eq!(goodbye.header.channel, closed_channel);
19349        assert_ne!(goodbye.header.channel, 0);
19350        assert_ne!(goodbye.header.epoch, 0);
19351        assert!(client_rx.try_recv().is_err());
19352        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
19353        assert!(matches!(
19354            target_rx.try_recv(),
19355            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
19356        ));
19357    }
19358
19359    #[tokio::test]
19360    async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
19361        let registry = Arc::new(Registry::default());
19362        let forwarding = Arc::new(ForwardingTable::default());
19363        let supervisor = SupervisorHandle::new();
19364        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
19365        let handler =
19366            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
19367                .with_supervisor(supervisor);
19368        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
19369        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
19370        register_capability_manifest(
19371            &handler,
19372            &target_ctx,
19373            &mut target_rx,
19374            capability_manifest("target", &["credentials-provider/v1"], &[]),
19375            1,
19376        )
19377        .await;
19378        register_capability_manifest(
19379            &handler,
19380            &opener_ctx,
19381            &mut opener_rx,
19382            capability_manifest("opener", &[], &[]),
19383            2,
19384        )
19385        .await;
19386        let (mut client_rx, _) = open_route_for_capability_test(
19387            &handler,
19388            &target_ctx,
19389            &mut target_rx,
19390            732,
19391            3,
19392            "target",
19393            Some(ConsumerIdentity {
19394                module_id: "opener".to_string(),
19395                launch_nonce: "opener-nonce".to_string(),
19396            }),
19397        )
19398        .await;
19399        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
19400
19401        handler
19402            .handle_control_frame(
19403                &target_ctx,
19404                catalog_update_with_capabilities_frame(
19405                    4,
19406                    CapabilityDeclarations {
19407                        provides: Vec::new(),
19408                        requires: Vec::new(),
19409                        must_never_reach: Vec::new(),
19410                    },
19411                ),
19412            )
19413            .await
19414            .expect("claim removal catalog.update succeeds");
19415        assert_eq!(
19416            forwarding.active_binding_count().unwrap(),
19417            1,
19418            "removing an attested target claim must leave the route census unchanged"
19419        );
19420        assert!(
19421            client_rx.try_recv().is_err(),
19422            "claim removal must not emit route.closed or a client route GOODBYE"
19423        );
19424        assert!(
19425            target_rx.try_recv().is_err(),
19426            "claim removal must not send the target a route GOODBYE"
19427        );
19428    }
19429
19430    /// A direct client may open a route to a denied capability provider; this
19431    /// policy applies only to attested supervised module origins, not to direct clients.
19432    #[tokio::test]
19433    async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
19434        let registry = Arc::new(Registry::default());
19435        let forwarding = Arc::new(ForwardingTable::default());
19436        let supervisor = SupervisorHandle::new();
19437        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
19438        let handler =
19439            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
19440                .with_supervisor(supervisor);
19441        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
19442        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
19443        register_capability_manifest(
19444            &handler,
19445            &target_ctx,
19446            &mut target_rx,
19447            capability_manifest("target", &["credentials-provider/v1"], &[]),
19448            1,
19449        )
19450        .await;
19451        register_capability_manifest(
19452            &handler,
19453            &opener_ctx,
19454            &mut opener_rx,
19455            capability_manifest("opener", &[], &["credentials-provider/v1"]),
19456            2,
19457        )
19458        .await;
19459
19460        let (_client_rx, bind) = open_route_for_capability_test(
19461            &handler,
19462            &target_ctx,
19463            &mut target_rx,
19464            742,
19465            3,
19466            "target",
19467            None,
19468        )
19469        .await;
19470        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
19471            panic!("direct scope-honesty route must bind");
19472        };
19473        assert_eq!(principal, Some(Principal::Direct));
19474        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
19475    }
19476
19477    /// A module that denies a capability receives no self-route exemption when it
19478    /// also attestedly provides that capability.
19479    #[tokio::test]
19480    async fn must_never_reach_self_route_is_capability_forbidden() {
19481        let registry = Arc::new(Registry::default());
19482        let forwarding = Arc::new(ForwardingTable::default());
19483        let supervisor = SupervisorHandle::new();
19484        supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
19485        let handler =
19486            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
19487                .with_supervisor(supervisor);
19488        let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
19489        register_capability_manifest(
19490            &handler,
19491            &self_ctx,
19492            &mut self_rx,
19493            capability_manifest(
19494                "self-provider",
19495                &["credentials-provider/v1"],
19496                &["credentials-provider/v1"],
19497            ),
19498            1,
19499        )
19500        .await;
19501
19502        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
19503        let replies = handler
19504            .handle_control_frame(
19505                &client_ctx,
19506                route_open_frame_with_admission_facts(
19507                    2,
19508                    "self-provider",
19509                    unique_project_root("admission-facts"),
19510                    Some(ConsumerIdentity {
19511                        module_id: "self-provider".to_string(),
19512                        launch_nonce: "self-nonce".to_string(),
19513                    }),
19514                    None,
19515                ),
19516            )
19517            .await
19518            .expect("self-route refusal returns a typed frame");
19519        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
19520        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
19521        assert!(
19522            self_rx.try_recv().is_err(),
19523            "self denial must not relay route.bind"
19524        );
19525    }
19526
19527    #[test]
19528    fn unsupported_channel_zero_frame_returns_error() {
19529        let handler = ControlHandler::default();
19530        let request = Frame::build(
19531            FrameType::Request,
19532            control_flags(),
19533            0,
19534            0,
19535            21,
19536            b"opaque".to_vec(),
19537        )
19538        .unwrap();
19539
19540        let response = handler
19541            .handle_control(ConnectionId::new(1), request)
19542            .unwrap();
19543
19544        assert_eq!(response[0].header.ty, FrameType::Error);
19545        assert_eq!(
19546            parse_error(&response[0])["code"],
19547            "unsupported_control_frame"
19548        );
19549    }
19550
19551    /// Blue/green swap at the control-plane boundary. The supervisor that opens
19552    /// a swap is not wired yet, so the candidate is registered here directly
19553    /// into the registry and forwarding candidate slots, the way the swap's
19554    /// HELLO admission will.
19555    mod swap {
19556        use super::*;
19557
19558        const INCUMBENT: ConnectionId = ConnectionId::new(30);
19559        const CANDIDATE: ConnectionId = ConnectionId::new(40);
19560
19561        struct Swap {
19562            registry: Arc<Registry>,
19563            forwarding: Arc<ForwardingTable>,
19564            handler: ControlHandler,
19565            incumbent_ctx: RouteCtx,
19566            incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
19567            candidate_ctx: RouteCtx,
19568            candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
19569        }
19570
19571        async fn swap_with_incumbent() -> Swap {
19572            let registry = Arc::new(Registry::default());
19573            let forwarding = Arc::new(ForwardingTable::default());
19574            let handler =
19575                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
19576            let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
19577            hello_via_sink(
19578                &handler,
19579                &incumbent_ctx,
19580                &mut incumbent_rx,
19581                hello_frame("aft", PROTOCOL_VERSION, 7),
19582            )
19583            .await;
19584            let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
19585            Swap {
19586                registry,
19587                forwarding,
19588                handler,
19589                incumbent_ctx,
19590                incumbent_rx,
19591                candidate_ctx,
19592                candidate_rx,
19593            }
19594        }
19595
19596        fn register_candidate(swap: &Swap, ready: Option<bool>) {
19597            let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
19598            candidate_manifest.ready = ready;
19599            let registration = swap
19600                .registry
19601                .register_candidate_with_control_ops(
19602                    candidate_manifest,
19603                    PROTOCOL_VERSION,
19604                    CANDIDATE,
19605                    module_baseline_control_ops(),
19606                )
19607                .unwrap();
19608            swap.forwarding
19609                .register_candidate_module_connection(
19610                    CANDIDATE,
19611                    "aft".to_string(),
19612                    PROTOCOL_VERSION,
19613                    manifest_concurrency(&registration.manifest),
19614                    swap.candidate_ctx.egress.clone(),
19615                )
19616                .unwrap();
19617        }
19618
19619        fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
19620            let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
19621            swap.registry.promote_candidate("aft").unwrap().unwrap();
19622            cutover.incumbent.unwrap()
19623        }
19624
19625        /// A swap candidate's HELLO reserves its candidate registry entry
19626        /// before installing its candidate forwarding endpoint. The swap
19627        /// supervisor promotes a ready candidate as soon as it reads it as
19628        /// registered, and promotion cuts forwarding over to the candidate
19629        /// endpoint, so the lookup the supervisor reads
19630        /// (`Registry::published_registration`) must not return the candidate
19631        /// until that endpoint exists. Otherwise the cutover finds no candidate
19632        /// endpoint and the swap fails as if the candidate had disconnected.
19633        #[tokio::test]
19634        async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
19635            let mut swap = swap_with_incumbent().await;
19636            let hold = Arc::new(std::sync::Barrier::new(2));
19637            let handler = swap
19638                .handler
19639                .clone()
19640                .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
19641            handler
19642                .supervisor
19643                .open_swap("aft", "candidate-nonce".to_string());
19644            let hello = {
19645                let handler = handler.clone();
19646                let candidate_ctx = swap.candidate_ctx.clone();
19647                std::thread::spawn(move || {
19648                    handler
19649                        .handle_hello(
19650                            candidate_ctx.connection_id,
19651                            Some(candidate_ctx.egress.clone()),
19652                            hello_frame_with_nonce(
19653                                "aft",
19654                                PROTOCOL_VERSION,
19655                                8,
19656                                Some("candidate-nonce"),
19657                            ),
19658                        )
19659                        .unwrap()
19660                })
19661            };
19662
19663            hold.wait();
19664            // Parked: the candidate slot is claimed but not published, and
19665            // forwarding has no candidate endpoint.
19666            assert!(swap
19667                .registry
19668                .registration(crate::registry::RegistrationSlot::Candidate("aft"))
19669                .unwrap()
19670                .is_some());
19671            assert!(swap
19672                .registry
19673                .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
19674                .unwrap()
19675                .is_none());
19676            assert!(swap
19677                .forwarding
19678                .module_endpoint_for_connection(CANDIDATE)
19679                .unwrap()
19680                .is_none());
19681
19682            hold.wait();
19683            assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
19684            assert_eq!(
19685                swap.candidate_rx.try_recv().unwrap().frame.header.ty,
19686                FrameType::HelloAck
19687            );
19688            let published = swap
19689                .registry
19690                .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
19691                .unwrap()
19692                .expect("candidate is published once its HELLO completes");
19693            assert_eq!(published.connection_id, CANDIDATE);
19694            assert!(swap
19695                .forwarding
19696                .module_endpoint_for_connection(CANDIDATE)
19697                .unwrap()
19698                .is_some());
19699        }
19700
19701        #[tokio::test]
19702        async fn candidate_lookup_after_hello_ack_waits_for_publication() {
19703            let mut swap = swap_with_incumbent().await;
19704            let hold = Arc::new(std::sync::Barrier::new(2));
19705            swap.handler.hello_ack_to_publication_hold = Some(Arc::clone(&hold));
19706            swap.handler
19707                .supervisor
19708                .open_swap("aft", "candidate-nonce".to_string());
19709            let hello = {
19710                let handler = swap.handler.clone();
19711                let candidate_ctx = swap.candidate_ctx.clone();
19712                std::thread::spawn(move || {
19713                    handler
19714                        .handle_hello(
19715                            CANDIDATE,
19716                            Some(candidate_ctx.egress),
19717                            hello_frame_with_nonce(
19718                                "aft",
19719                                PROTOCOL_VERSION,
19720                                8,
19721                                Some("candidate-nonce"),
19722                            ),
19723                        )
19724                        .unwrap()
19725                })
19726            };
19727            hold.wait();
19728            assert_eq!(
19729                swap.candidate_rx.try_recv().unwrap().frame.header.ty,
19730                FrameType::HelloAck
19731            );
19732            let reader_registry = Arc::clone(&swap.registry);
19733            let published = lookup_during_hello_publication(&swap.registry, &hold, move || {
19734                reader_registry
19735                    .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
19736                    .unwrap()
19737            });
19738            assert!(hello.join().unwrap().is_empty());
19739            assert_eq!(
19740                published
19741                    .expect("ACK makes candidate publication visible")
19742                    .connection_id,
19743                CANDIDATE
19744            );
19745            assert_eq!(
19746                swap.registry.generation().unwrap(),
19747                1,
19748                "candidate is not a second catalog entry"
19749            );
19750        }
19751
19752        #[tokio::test]
19753        async fn audit4_ungated_removed_candidate_publication_rolls_back_admission() {
19754            let mut swap = swap_with_incumbent().await;
19755            let hold = Arc::new(std::sync::Barrier::new(2));
19756            let handler = swap
19757                .handler
19758                .clone()
19759                .with_hello_registry_to_forwarding_hold(hold.clone());
19760            handler
19761                .supervisor
19762                .open_swap("aft", "candidate-nonce".into());
19763            let hello = {
19764                let handler = handler.clone();
19765                let ctx = swap.candidate_ctx.clone();
19766                std::thread::spawn(move || {
19767                    handler
19768                        .handle_hello(
19769                            CANDIDATE,
19770                            Some(ctx.egress),
19771                            hello_frame_with_nonce(
19772                                "aft",
19773                                PROTOCOL_VERSION,
19774                                8,
19775                                Some("candidate-nonce"),
19776                            ),
19777                        )
19778                        .unwrap()
19779                })
19780            };
19781            hold.wait();
19782            assert_eq!(
19783                swap.registry
19784                    .deregister_connection(CANDIDATE)
19785                    .unwrap()
19786                    .len(),
19787                1
19788            );
19789            hold.wait();
19790            assert!(hello.join().unwrap().is_empty());
19791            assert_eq!(
19792                swap.candidate_rx.try_recv().unwrap().frame.header.ty,
19793                FrameType::HelloAck
19794            );
19795            assert!(swap.registry.get_candidate("aft").unwrap().is_none());
19796            assert_eq!(
19797                handler
19798                    .supervisor
19799                    .swap_hello_admission("aft", Some("candidate-nonce")),
19800                SwapHelloAdmission::Candidate,
19801                "failed publication must roll back candidate_admitted for an ungated nonce swap"
19802            );
19803            handler.supervisor.close_swap("aft");
19804        }
19805
19806        fn keyed_total(counters: &Value, key: &str) -> u64 {
19807            counters[key]
19808                .as_object()
19809                .map(|counts| counts.values().filter_map(Value::as_u64).sum())
19810                .unwrap_or(0)
19811        }
19812
19813        #[tokio::test]
19814        async fn replacement_logs_old_end_and_new_admission_once() {
19815            let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
19816            let mut swap = swap_with_incumbent().await;
19817            swap.handler
19818                .supervisor
19819                .open_swap("aft", "candidate-nonce".to_string());
19820            hello_via_sink(
19821                &swap.handler,
19822                &swap.candidate_ctx,
19823                &mut swap.candidate_rx,
19824                hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
19825            )
19826            .await;
19827            cutover(&swap);
19828            swap.handler.cleanup_connection(INCUMBENT).unwrap();
19829
19830            let captured = crate::router::test_log::captured_logs(&logs);
19831            println!("captured replacement registry logs:\n{captured}");
19832            let new_admissions: Vec<_> = captured
19833                .lines()
19834                .filter(|line| {
19835                    line.contains("connection_id=40")
19836                        && line.contains("swap candidate registered; not routable until cutover")
19837                })
19838                .collect();
19839            assert_eq!(
19840                new_admissions.len(),
19841                1,
19842                "unexpected admission logs: {captured}"
19843            );
19844            assert!(new_admissions[0].contains("module_id=aft"));
19845            assert!(new_admissions[0].contains("connection_id=40"));
19846            assert!(
19847                new_admissions[0].contains("swap candidate registered; not routable until cutover")
19848            );
19849            assert!(new_admissions[0].contains("ready=true"));
19850
19851            let old_admissions: Vec<_> = captured
19852                .lines()
19853                .filter(|line| {
19854                    line.contains("module registered module_id=aft ")
19855                        && line.contains("connection_id=30")
19856                })
19857                .collect();
19858            assert_eq!(
19859                old_admissions.len(),
19860                1,
19861                "unexpected incumbent admission: {captured}"
19862            );
19863
19864            let old_ends: Vec<_> = captured
19865                .lines()
19866                .filter(|line| {
19867                    line.contains("connection_id=30") && line.contains("module registration ended")
19868                })
19869                .collect();
19870            assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
19871            assert!(old_ends[0].contains("module_id=aft"));
19872            assert!(old_ends[0].contains("connection_id=30"));
19873            assert!(old_ends[0].contains("reason=replaced"));
19874            assert!(old_ends[0].contains("replaced_by_connection_id=40"));
19875            assert_eq!(
19876                captured
19877                    .lines()
19878                    .filter(|line| line.contains("module registration promoted"))
19879                    .count(),
19880                1,
19881                "unexpected promotion logs: {captured}"
19882            );
19883        }
19884
19885        /// An ack from the incumbent for a bind it was sent before cutover,
19886        /// arriving before the incumbent is drained. The incumbent is the live
19887        /// connection carrying every other client's routes, so the ack must
19888        /// not end it: the waiting client is told to retry, the reservation is
19889        /// given back, and the incumbent is told to drop just that binding.
19890        #[tokio::test]
19891        async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
19892            let mut swap = swap_with_incumbent().await;
19893            let handler = swap.handler.clone();
19894
19895            // A co-tenant route, bound on the incumbent before the swap.
19896            let cotenant = ConnectionId::new(31);
19897            let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
19898            let (cotenant_task, cotenant_bind) = relay_route_open(
19899                &handler,
19900                cotenant,
19901                &cotenant_ctx.egress,
19902                &mut swap.incumbent_rx,
19903                100,
19904                "aft",
19905                "swap-cotenant",
19906            )
19907            .await;
19908            handler
19909                .handle_control_frame(
19910                    &swap.incumbent_ctx,
19911                    route_bind_ack(cotenant_bind.header.corr),
19912                )
19913                .await
19914                .unwrap();
19915            assert!(cotenant_task.await.unwrap().is_empty());
19916            let (cotenant_channel, cotenant_epoch) =
19917                published_route(&cotenant_rx.recv().await.unwrap());
19918
19919            // A second route.open, relayed to the incumbent and not yet acked.
19920            let caller = ConnectionId::new(32);
19921            let (caller_ctx, mut caller_rx) = route_ctx(caller);
19922            let (caller_task, caller_bind) = relay_route_open(
19923                &handler,
19924                caller,
19925                &caller_ctx.egress,
19926                &mut swap.incumbent_rx,
19927                101,
19928                "aft",
19929                "swap-caller",
19930            )
19931            .await;
19932            let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
19933
19934            register_candidate(&swap, None);
19935            cutover(&swap);
19936
19937            // The incumbent acks after promotion and before any drain.
19938            let ack = handler
19939                .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
19940                .await;
19941            let module_loop_error = ack.as_ref().err().map(ToString::to_string);
19942            if module_loop_error.is_some() {
19943                // What the connection loop does with an untranslated router
19944                // error: end the connection, releasing every route on it.
19945                handler.cleanup_connection(INCUMBENT).unwrap();
19946            }
19947
19948            // 1. The incumbent's other routes survive.
19949            assert!(
19950                cotenant_rx.try_recv().is_err(),
19951                "the co-tenant route on the incumbent was torn down by one late ack: \
19952                 {module_loop_error:?}"
19953            );
19954            assert!(matches!(
19955                swap.forwarding
19956                    .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
19957                    .unwrap(),
19958                DataRoute::Client(DataRouteState::Bound(_))
19959            ));
19960            assert_eq!(module_loop_error, None);
19961            assert!(swap
19962                .registry
19963                .get_module_by_connection(INCUMBENT)
19964                .unwrap()
19965                .is_some());
19966
19967            // 2. Exactly one channel-scoped GOODBYE to the incumbent.
19968            let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
19969                .await
19970                .expect("the incumbent is told to drop the abandoned binding")
19971                .unwrap()
19972                .frame;
19973            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
19974            assert_eq!(goodbye.header.channel, abandoned_channel);
19975            assert_eq!(goodbye.header.epoch, abandoned_epoch);
19976            assert!(swap.incumbent_rx.try_recv().is_err());
19977
19978            // 3. The waiting client gets a retryable refusal and no route.
19979            let response = caller_task.await.unwrap();
19980            assert_eq!(response.len(), 1);
19981            assert_eq!(parse_error(&response[0])["code"], "module_reloading");
19982            assert!(caller_rx.try_recv().is_err());
19983
19984            // 4. The reservation pair is given back, and the pending bind
19985            //    settled exactly once: one accepted open (the co-tenant) and one
19986            //    refused open (the caller), nothing counted twice.
19987            assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
19988            let counters = handler.counters().snapshot();
19989            assert_eq!(
19990                keyed_total(&counters, "route_open_accepted_by_principal"),
19991                1
19992            );
19993            assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
19994            assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
19995        }
19996
19997        /// After cutover the incumbent is drained BY ENDPOINT. Draining by module
19998        /// id would resolve to the promoted candidate and every new route.open
19999        /// would be refused as reloading, leaving neither process routable.
20000        #[tokio::test]
20001        async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
20002            let mut swap = swap_with_incumbent().await;
20003            register_candidate(&swap, None);
20004            let incumbent = cutover(&swap);
20005            swap.forwarding
20006                .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
20007                .unwrap()
20008                .expect("the incumbent is still registered");
20009
20010            let client = ConnectionId::new(33);
20011            let (client_ctx, mut client_rx) = route_ctx(client);
20012            let route_handler = swap.handler.clone();
20013            let open_ctx = RouteCtx {
20014                connection_id: client,
20015                egress: client_ctx.egress.clone(),
20016            };
20017            let mut route_task = tokio::spawn(async move {
20018                route_handler
20019                    .handle_control_frame(
20020                        &open_ctx,
20021                        route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
20022                    )
20023                    .await
20024                    .unwrap()
20025            });
20026            let bind = tokio::select! {
20027                bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
20028                response = &mut route_task => {
20029                    let response = response.unwrap();
20030                    panic!(
20031                        "post-cutover route.open was refused instead of relayed to the candidate: {}",
20032                        parse_error(&response[0])["code"]
20033                    );
20034                }
20035            };
20036            swap.handler
20037                .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
20038                .await
20039                .unwrap();
20040            assert!(route_task.await.unwrap().is_empty());
20041            let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
20042            match swap
20043                .forwarding
20044                .lookup_data_route(client, channel, epoch)
20045                .unwrap()
20046            {
20047                DataRoute::Client(DataRouteState::Bound(route)) => {
20048                    assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
20049                }
20050                other => panic!("expected a bound route on the candidate, got {other:?}"),
20051            }
20052            assert!(swap.incumbent_rx.try_recv().is_err());
20053        }
20054
20055        /// A candidate declares itself ready with `catalog.update` on its own
20056        /// connection. If the connection-keyed registry lookups searched only the
20057        /// active slot, this would answer `not_registered` and the candidate
20058        /// would never become ready.
20059        #[tokio::test]
20060        async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
20061            let swap = swap_with_incumbent().await;
20062            register_candidate(&swap, Some(false));
20063            let update = Frame::build(
20064                FrameType::Request,
20065                control_flags(),
20066                0,
20067                0,
20068                55,
20069                serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
20070                    provides: manifest("aft", PROTOCOL_VERSION).provides,
20071                    capabilities: None,
20072                    ready: Some(true),
20073                })
20074                .unwrap(),
20075            )
20076            .unwrap();
20077
20078            let replies = swap
20079                .handler
20080                .handle_control_frame(&swap.candidate_ctx, update)
20081                .await
20082                .unwrap();
20083
20084            assert_eq!(replies.len(), 1);
20085            assert_eq!(
20086                replies[0].header.ty,
20087                FrameType::Response,
20088                "candidate catalog.update was refused: {:?}",
20089                serde_json::from_slice::<Value>(&replies[0].body).ok()
20090            );
20091            assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
20092            assert_eq!(
20093                swap.registry
20094                    .get_module("aft")
20095                    .unwrap()
20096                    .unwrap()
20097                    .connection_id,
20098                INCUMBENT
20099            );
20100        }
20101    }
20102
20103    /// The HELLO gate while the supervisor has a swap open: only the nonce it
20104    /// minted for the candidate admits a second process, into the candidate
20105    /// slot, and that check runs ahead of the reserved-module gate.
20106    mod swap_admission {
20107        use super::*;
20108
20109        const INCUMBENT_NONCE: &str = "incumbent-nonce";
20110        const CANDIDATE_NONCE: &str = "candidate-nonce";
20111
20112        fn handler_with_incumbent(
20113            module_id: &str,
20114            reserved: bool,
20115        ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
20116            let registry = Arc::new(Registry::default());
20117            let supervisor = SupervisorHandle::new();
20118            supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
20119            if reserved {
20120                supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
20121            }
20122            let handler =
20123                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor.clone());
20124            let incumbent = handler
20125                .handle_control(
20126                    ConnectionId::new(1),
20127                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
20128                )
20129                .unwrap();
20130            assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
20131            supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
20132            (registry, supervisor, handler)
20133        }
20134
20135        /// Design mutation arm (ii). On an UNRESERVED id the reserved gate
20136        /// admits every nonce, so while a swap is open the swap gate is the only
20137        /// thing between a key-holder and the candidate slot. A nonce the
20138        /// supervisor did not mint, or none at all, is refused, and neither the
20139        /// incumbent's registration nor the candidate slot moves.
20140        #[test]
20141        fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
20142            let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
20143
20144            for (connection, nonce) in [(2, Some("forged")), (3, None)] {
20145                let replies = handler
20146                    .handle_control(
20147                        ConnectionId::new(connection),
20148                        hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
20149                    )
20150                    .unwrap();
20151                assert_eq!(replies[0].header.ty, FrameType::Error);
20152                assert_eq!(
20153                    parse_error(&replies[0])["code"],
20154                    "swap_token_invalid",
20155                    "nonce {nonce:?}"
20156                );
20157            }
20158            assert!(registry.get_candidate("aft").unwrap().is_none());
20159            assert_eq!(
20160                registry.get_module("aft").unwrap().unwrap().connection_id,
20161                ConnectionId::new(1)
20162            );
20163
20164            // Control: the minted token is admitted, into the candidate slot,
20165            // and only once.
20166            let admitted = handler
20167                .handle_control(
20168                    ConnectionId::new(4),
20169                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
20170                )
20171                .unwrap();
20172            assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
20173            assert_eq!(
20174                registry
20175                    .get_candidate("aft")
20176                    .unwrap()
20177                    .unwrap()
20178                    .connection_id,
20179                ConnectionId::new(4)
20180            );
20181            assert_eq!(
20182                registry.get_module("aft").unwrap().unwrap().connection_id,
20183                ConnectionId::new(1),
20184                "the candidate must not take the active slot"
20185            );
20186            let replayed = handler
20187                .handle_control(
20188                    ConnectionId::new(5),
20189                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
20190                )
20191                .unwrap();
20192            assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
20193
20194            // The case only this gate covers: the incumbent has died mid-swap,
20195            // so its duplicate refusal is gone too, and without the gate a
20196            // key-holder would take the id's ACTIVE slot.
20197            handler.cleanup_connection(ConnectionId::new(1)).unwrap();
20198            let squatter = handler
20199                .handle_control(
20200                    ConnectionId::new(6),
20201                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
20202                )
20203                .unwrap();
20204            assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
20205            assert!(
20206                registry.get_module("aft").unwrap().is_none(),
20207                "a squatter took the active slot of an id being swapped"
20208            );
20209        }
20210
20211        /// Design mutation arm (iii). A reserved module's candidate presents a
20212        /// nonce the reserved gate has never seen (that gate holds the
20213        /// incumbent's), so the swap gate must run first or the candidate is
20214        /// refused `reserved_module` and a reserved module can never be swapped.
20215        #[test]
20216        fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
20217            let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
20218
20219            let replies = handler
20220                .handle_control(
20221                    ConnectionId::new(2),
20222                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
20223                )
20224                .unwrap();
20225
20226            assert_eq!(
20227                replies[0].header.ty,
20228                FrameType::HelloAck,
20229                "reserved candidate refused: {:?}",
20230                serde_json::from_slice::<Value>(&replies[0].body).ok()
20231            );
20232            assert_eq!(
20233                registry
20234                    .get_candidate("vault")
20235                    .unwrap()
20236                    .unwrap()
20237                    .connection_id,
20238                ConnectionId::new(2)
20239            );
20240        }
20241
20242        /// With no swap open the gate is inert: the incumbent's reserved gate
20243        /// and duplicate refusal behave exactly as before.
20244        #[test]
20245        fn without_an_open_swap_the_ordinary_gates_decide() {
20246            let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
20247            supervisor.close_swap("vault");
20248
20249            let candidate = handler
20250                .handle_control(
20251                    ConnectionId::new(2),
20252                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
20253                )
20254                .unwrap();
20255            assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
20256            let duplicate = handler
20257                .handle_control(
20258                    ConnectionId::new(3),
20259                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
20260                )
20261                .unwrap();
20262            assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
20263            assert!(registry.get_candidate("vault").unwrap().is_none());
20264        }
20265    }
20266
20267    /// `scope.sync` and `scope.describe` through the real control handler: who
20268    /// may sync is decided by the registration and launch nonce of the module
20269    /// connection, never by the request body.
20270    mod scopes {
20271        mod close_reasons {
20272            include!("route_close_reason_tests.rs");
20273        }
20274        mod idle_tests {
20275            use super::*;
20276            include!("scope_idle_tests.rs");
20277        }
20278        mod route_open_timings {
20279            use super::*;
20280            include!("route_open_timing_tests.rs");
20281        }
20282        mod route_open_refusal_log {
20283            use super::*;
20284            include!("route_open_refusal_log_tests.rs");
20285        }
20286        mod activity {
20287            include!("activity_scope_tests.rs");
20288        }
20289        mod delta {
20290            include!("scope_handler_delta_tests.rs");
20291        }
20292        mod call_modes {
20293            include!("scope_call_mode_tests.rs");
20294        }
20295        mod federated {
20296            include!("scope_federated_tests.rs");
20297        }
20298        mod authorize {
20299            include!("route_authorize_tests.rs");
20300        }
20301        #[tokio::test]
20302        async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
20303            let rig = rig().await;
20304            let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
20305                "s".into(),
20306                1,
20307                "tool".into(),
20308            ))
20309            .unwrap();
20310            assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
20311            let (direct, _rx) = wide_ctx(999);
20312            for (ctx, expected) in [
20313                (&rig.owner, FrameType::Response),
20314                (&direct, FrameType::Error),
20315            ] {
20316                let frame =
20317                    Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
20318                        .unwrap();
20319                let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
20320                assert_eq!(replies[0].header.ty, expected);
20321                if expected == FrameType::Response {
20322                    let reply: subc_control::ScopeServedByReply =
20323                        serde_json::from_slice(&replies[0].body).unwrap();
20324                    assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
20325                } else {
20326                    assert_eq!(parse_error(&replies[0])["code"], "not_registered");
20327                }
20328            }
20329            let reply = call(
20330                &rig.handler,
20331                &rig.owner,
20332                &ModuleControlRequestFromModule::ScopeDescribe {
20333                    owner: Principal::Reserved {
20334                        module_id: OWNER.into(),
20335                    },
20336                    scope_ref: "s".into(),
20337                },
20338            )
20339            .await;
20340            assert!(matches!(
20341                serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
20342                ModuleControlResponseToModule::ScopeDescribe { .. }
20343            ));
20344        }
20345
20346        use subc_protocol::scope::{
20347            ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
20348            ScopeStatus,
20349        };
20350
20351        use super::*;
20352
20353        const OWNER: &str = "prefrontal-core";
20354
20355        fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
20356            ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
20357        }
20358
20359        async fn call(
20360            handler: &ControlHandler,
20361            ctx: &RouteCtx,
20362            request: &ModuleControlRequestFromModule,
20363        ) -> Frame {
20364            let body = serde_json::to_vec(request).unwrap();
20365            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
20366            let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
20367            assert_eq!(replies.len(), 1, "{replies:?}");
20368            replies.pop().unwrap()
20369        }
20370
20371        async fn sync(
20372            handler: &ControlHandler,
20373            ctx: &RouteCtx,
20374            generation: u64,
20375            scopes: Vec<ScopeRecord>,
20376        ) -> Result<ModuleControlResponseToModule, String> {
20377            let reply = call(
20378                handler,
20379                ctx,
20380                &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
20381            )
20382            .await;
20383            match reply.header.ty {
20384                FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
20385                _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
20386            }
20387        }
20388
20389        async fn describe(
20390            handler: &ControlHandler,
20391            ctx: &RouteCtx,
20392            owner: &str,
20393            scope_ref: &str,
20394        ) -> ModuleControlResponseToModule {
20395            let reply = call(
20396                handler,
20397                ctx,
20398                &ModuleControlRequestFromModule::ScopeDescribe {
20399                    owner: Principal::Reserved {
20400                        module_id: owner.to_string(),
20401                    },
20402                    scope_ref: scope_ref.to_string(),
20403                },
20404            )
20405            .await;
20406            assert_eq!(
20407                reply.header.ty,
20408                FrameType::Response,
20409                "{:?}",
20410                parse_error(&reply)
20411            );
20412            serde_json::from_slice(&reply.body).unwrap()
20413        }
20414
20415        #[tokio::test]
20416        async fn scope_describe_never_echoes_bind_proof_or_hash() {
20417            const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
20418            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
20419            let mut rig = rig().await;
20420            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
20421                .await;
20422            let _bound = rig
20423                .bound(
20424                    Some(AFT),
20425                    PLEXUS,
20426                    Some(
20427                        rig.selector("s", Some(1))
20428                            .with_bind_proof(Some(PROOF.into())),
20429                    ),
20430                )
20431                .await;
20432            let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
20433            let output = serde_json::to_string(&reply).unwrap();
20434            assert!(output.contains("\"status\":\"live\""));
20435            assert!(!output.contains(PROOF));
20436            assert!(!output.contains(HASH));
20437        }
20438
20439        /// Register `module_id` on `connection` with `nonce`, returning its ctx.
20440        async fn module(
20441            handler: &ControlHandler,
20442            connection: u64,
20443            module_id: &str,
20444            nonce: Option<&str>,
20445        ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
20446            let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
20447            hello_via_sink(
20448                handler,
20449                &ctx,
20450                &mut rx,
20451                hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
20452            )
20453            .await;
20454            (ctx, rx)
20455        }
20456
20457        /// `direct` and every other client connection has no registration, so
20458        /// it can neither sync nor own a scope.
20459        #[tokio::test]
20460        async fn a_client_connection_cannot_sync_or_describe() {
20461            let handler = ControlHandler::new(Arc::new(Registry::default()));
20462            let (ctx, _rx) = route_ctx(ConnectionId::new(9));
20463            for request in [
20464                ModuleControlRequestFromModule::ScopeSync {
20465                    generation: 1,
20466                    scopes: vec![head("s", 1)],
20467                },
20468                ModuleControlRequestFromModule::ScopeDescribe {
20469                    owner: Principal::Direct,
20470                    scope_ref: "s".to_string(),
20471                },
20472            ] {
20473                let reply = call(&handler, &ctx, &request).await;
20474                assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
20475            }
20476            assert!(
20477                !handler
20478                    .scopes
20479                    .read()
20480                    .unwrap()
20481                    .describe(
20482                        &Principal::Reserved {
20483                            module_id: OWNER.to_string()
20484                        },
20485                        "s"
20486                    )
20487                    .owner_synced
20488            );
20489        }
20490
20491        /// A module the supervisor did not spawn registers without a launch
20492        /// nonce, so it is never an owner's current launch.
20493        #[tokio::test]
20494        async fn a_module_without_a_supervised_launch_cannot_sync() {
20495            let handler = ControlHandler::new(Arc::new(Registry::default()));
20496            let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
20497            assert_eq!(
20498                sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
20499                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
20500            );
20501        }
20502
20503        #[tokio::test]
20504        async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
20505            let supervisor = SupervisorHandle::new();
20506            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
20507            let handler = ControlHandler::new(Arc::new(Registry::default()))
20508                .with_supervisor(supervisor.clone());
20509            let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
20510            sync(&handler, &incumbent, 1, vec![head("s", 1)])
20511                .await
20512                .expect("the current launch syncs");
20513
20514            // A swap candidate registers with the swap token and is refused
20515            // while the incumbent keeps syncing.
20516            supervisor.open_swap(OWNER, "n2".to_string());
20517            let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
20518            assert_eq!(
20519                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
20520                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
20521            );
20522            sync(&handler, &incumbent, 2, vec![head("s", 1)])
20523                .await
20524                .expect("the serving owner syncs during the swap");
20525
20526            // The swap fails and is rolled back. The candidate never held sync
20527            // authority, and still cannot sync.
20528            supervisor.close_swap(OWNER);
20529            assert_eq!(
20530                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
20531                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
20532            );
20533            sync(&handler, &incumbent, 3, vec![head("s", 1)])
20534                .await
20535                .expect("the serving owner syncs after the rollback");
20536            handler.cleanup_connection(candidate.connection_id).unwrap();
20537
20538            // A swap that cuts over. Promotion records the candidate's nonce as
20539            // the module's spawn nonce, which is what `set_spawn_nonce` does
20540            // here; the promoted connection then takes authority at any
20541            // generation and the superseded incumbent is refused.
20542            supervisor.open_swap(OWNER, "n3".to_string());
20543            let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
20544            supervisor.set_spawn_nonce(OWNER, "n3".to_string());
20545            let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
20546                .await
20547                .expect("the promoted launch takes authority");
20548            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
20549                panic!("unexpected reply {reply:?}");
20550            };
20551            assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
20552            assert_eq!(
20553                sync(&handler, &incumbent, 4, Vec::new()).await,
20554                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
20555            );
20556        }
20557
20558        /// Authority dies with its connection: the cleanup path releases it,
20559        /// so the owner's next connection takes it at any generation.
20560        #[tokio::test]
20561        async fn closing_the_authority_connection_frees_sync_authority() {
20562            let supervisor = SupervisorHandle::new();
20563            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
20564            let handler = ControlHandler::new(Arc::new(Registry::default()))
20565                .with_supervisor(supervisor.clone());
20566            let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
20567            sync(&handler, &first, 10, vec![head("s", 1)])
20568                .await
20569                .unwrap();
20570            handler.cleanup_connection(first.connection_id).unwrap();
20571
20572            let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
20573            sync(&handler, &second, 1, vec![head("s", 1)])
20574                .await
20575                .expect("the next connection takes the released authority");
20576        }
20577
20578        #[tokio::test]
20579        async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
20580            let supervisor = SupervisorHandle::new();
20581            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
20582            let handler =
20583                ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
20584            let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
20585            sync(&handler, &first, 10, vec![head("s", 1)])
20586                .await
20587                .unwrap();
20588            handler
20589                .handle_control_frame(
20590                    &first,
20591                    Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
20592                )
20593                .await
20594                .unwrap();
20595            let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
20596            sync(&handler, &second, 1, vec![head("s", 1)])
20597                .await
20598                .expect("GOODBYE releases authority even if the old socket remains open");
20599        }
20600
20601        #[tokio::test]
20602        async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
20603            let registry = Arc::new(Registry::default());
20604            let supervisor_handle = SupervisorHandle::new();
20605            let supervisor =
20606                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
20607                    .with_handle(supervisor_handle.clone())
20608                    .with_daemon_incarnation("incarnation-7".to_string());
20609            // Configured with enabled: false, so the supervisor lists the
20610            // module without spawning a process for it.
20611            supervisor
20612                .supervise_configured(
20613                    ModuleSpec {
20614                        script: None,
20615                        carry_containment: false,
20616                        module_id: OWNER.to_string(),
20617                        program: PathBuf::from("/nonexistent/prefrontal-core"),
20618                        args: Vec::new(),
20619                        env: Vec::new(),
20620                        reserved: false,
20621                        reserved_prefixes: Vec::new(),
20622                        protocol: ModuleProtocol::Subc,
20623                        overlap: Default::default(),
20624                    },
20625                    false,
20626                )
20627                .unwrap();
20628            supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
20629            let handler =
20630                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
20631            let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
20632
20633            // Configured but not yet synced: a reader waits for the owner.
20634            let ModuleControlResponseToModule::ScopeDescribe {
20635                status,
20636                daemon_incarnation,
20637                owner_synced,
20638                owner_configured,
20639                scope,
20640                ..
20641            } = describe(&handler, &reader, OWNER, "s").await
20642            else {
20643                panic!("not a describe reply");
20644            };
20645            assert_eq!(status, ScopeStatus::NotLive);
20646            assert_eq!(daemon_incarnation, "incarnation-7");
20647            assert!(!owner_synced);
20648            assert!(owner_configured);
20649            assert!(scope.is_none());
20650
20651            // Not a supervised module: the owner will never sync, and a reader
20652            // refuses rather than waits.
20653            let ModuleControlResponseToModule::ScopeDescribe {
20654                status,
20655                owner_configured,
20656                ..
20657            } = describe(&handler, &reader, "ghost", "s").await
20658            else {
20659                panic!("not a describe reply");
20660            };
20661            assert_eq!(status, ScopeStatus::NotLive);
20662            assert!(!owner_configured);
20663
20664            // Live, with the stamp fields and the computed owner_authorized.
20665            let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
20666            sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
20667            let ModuleControlResponseToModule::ScopeDescribe {
20668                status,
20669                scope_epoch,
20670                owner_synced,
20671                scope,
20672                ..
20673            } = describe(&handler, &reader, OWNER, "s").await
20674            else {
20675                panic!("not a describe reply");
20676            };
20677            assert_eq!(status, ScopeStatus::Live);
20678            assert_eq!(scope_epoch, Some(4));
20679            assert!(owner_synced);
20680            let stamp = scope.expect("a live scope carries its stamp");
20681            assert!(
20682                stamp.owner_authorized,
20683                "prefrontal-core is the default authority"
20684            );
20685            assert_eq!(stamp.kind, ScopeKind::Head);
20686        }
20687
20688        #[tokio::test]
20689        async fn scope_authority_owners_decides_owner_authorized() {
20690            let supervisor = SupervisorHandle::new();
20691            supervisor.set_spawn_nonce("broca", "b1".to_string());
20692            let handler = ControlHandler::new(Arc::new(Registry::default()))
20693                .with_supervisor(supervisor)
20694                .with_scope_authority_owners(vec!["broca".to_string()]);
20695            let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
20696            let mut gated = head("s", 1);
20697            gated.attributes.agent_id = Some("agent".to_string());
20698            sync(&handler, &broca, 1, vec![gated]).await.unwrap();
20699            let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
20700                describe(&handler, &broca, "broca", "s").await
20701            else {
20702                panic!("not a describe reply");
20703            };
20704            assert!(scope.unwrap().owner_authorized);
20705        }
20706
20707        /// With route admission, the stamp, the commit re-check and drains in
20708        /// place, the feature is advertised: the module ops in HELLO_ACK, and
20709        /// `scopes/v1` in HELLO_ACK and `server.describe`.
20710        #[tokio::test]
20711        async fn scope_ops_and_the_scopes_capability_are_advertised() {
20712            let handler = ControlHandler::new(Arc::new(Registry::default()));
20713            let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
20714            let ack = hello_via_sink(
20715                &handler,
20716                &ctx,
20717                &mut rx,
20718                hello_frame("m", PROTOCOL_VERSION, 1),
20719            )
20720            .await;
20721            let ack = parse_ack(&ack);
20722            for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
20723                assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
20724            }
20725            assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
20726
20727            let (client, _client_rx) = route_ctx(ConnectionId::new(2));
20728            let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
20729            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
20730            let reply = handler
20731                .handle_control_frame(&client, frame)
20732                .await
20733                .unwrap()
20734                .pop()
20735                .unwrap();
20736            let ClientControlResponse::ServerDescribe { capabilities, .. } =
20737                serde_json::from_slice(&reply.body).unwrap()
20738            else {
20739                panic!("not a server.describe reply");
20740            };
20741            assert!(
20742                capabilities.iter().any(|c| c == CAP_SCOPES_V1),
20743                "{capabilities:?}"
20744            );
20745        }
20746
20747        // ---- route admission, stamps, commit re-check and drains ----------
20748
20749        const PLEXUS: &str = "plexus";
20750        const OTHER: &str = "other";
20751        const AFT: &str = "aft";
20752        const BROCA: &str = "broca";
20753        const MAGIC: &str = "magic-context";
20754
20755        fn nonce(module_id: &str) -> String {
20756            format!("nonce-{module_id}")
20757        }
20758
20759        fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
20760            let (tx, rx) = mpsc::channel(64);
20761            (
20762                RouteCtx {
20763                    connection_id: ConnectionId::new(connection),
20764                    egress: FrameSink::new(tx),
20765                },
20766                rx,
20767            )
20768        }
20769
20770        /// A daemon with a configured owner (prefrontal-core) registered on its
20771        /// own module connection, two routable targets (plexus, other), and
20772        /// launch nonces minted for the modules that open routes as carriers.
20773        struct Rig {
20774            handler: ControlHandler,
20775            forwarding: Arc<ForwardingTable>,
20776            owner: RouteCtx,
20777            _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
20778            modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
20779            generation: u64,
20780            next_connection: u64,
20781            _supervisor: Supervisor,
20782        }
20783
20784        async fn rig() -> Rig {
20785            rig_with_flow_support(true).await
20786        }
20787
20788        async fn rig_with_flow_support(flow_support: bool) -> Rig {
20789            rig_with_scope_capabilities(flow_support, false).await
20790        }
20791
20792        async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
20793            let mut provides = Vec::new();
20794            if flow_support {
20795                provides.push("flow-scopes/v1");
20796            }
20797            if run_support {
20798                provides.push("agent-run-scopes/v1");
20799            }
20800            rig_with_target_capabilities(&provides).await
20801        }
20802
20803        fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
20804            let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
20805            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
20806            // Decoder versions alone cannot admit scoped calls: only a capability
20807            // declaration promises the required run or flow behaviour.
20808            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
20809            body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
20810            Frame::build(
20811                FrameType::Hello,
20812                control_flags(),
20813                0,
20814                0,
20815                connection,
20816                serde_json::to_vec(&body).unwrap(),
20817            )
20818            .unwrap()
20819        }
20820
20821        async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
20822            let registry = Arc::new(Registry::default());
20823            let forwarding = Arc::new(ForwardingTable::default());
20824            let supervisor_handle = SupervisorHandle::new();
20825            let supervisor =
20826                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
20827                    .with_handle(supervisor_handle.clone());
20828            supervisor
20829                .supervise_configured(
20830                    ModuleSpec {
20831                        script: None,
20832                        carry_containment: false,
20833                        module_id: OWNER.to_string(),
20834                        program: PathBuf::from("/nonexistent/prefrontal-core"),
20835                        args: Vec::new(),
20836                        env: Vec::new(),
20837                        reserved: false,
20838                        reserved_prefixes: Vec::new(),
20839                        protocol: ModuleProtocol::Subc,
20840                        overlap: Default::default(),
20841                    },
20842                    false,
20843                )
20844                .unwrap();
20845            for module_id in [OWNER, AFT, BROCA, MAGIC] {
20846                supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
20847            }
20848            let handler =
20849                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
20850                    .with_supervisor(supervisor_handle);
20851            let (owner, mut owner_rx) = wide_ctx(1);
20852            hello_via_sink(
20853                &handler,
20854                &owner,
20855                &mut owner_rx,
20856                hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
20857            )
20858            .await;
20859            let mut modules = BTreeMap::new();
20860            for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
20861                let (ctx, mut rx) = wide_ctx(connection);
20862                let hello = hello_with_capabilities(module_id, connection, provides);
20863                hello_via_sink(&handler, &ctx, &mut rx, hello).await;
20864                modules.insert(module_id.to_string(), (ctx, rx));
20865            }
20866            Rig {
20867                handler,
20868                forwarding,
20869                owner,
20870                _owner_rx: owner_rx,
20871                modules,
20872                generation: 0,
20873                next_connection: 100,
20874                _supervisor: supervisor,
20875            }
20876        }
20877
20878        fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
20879            ScopeCarrier::new(Principal::Reserved {
20880                module_id: module_id.to_string(),
20881            })
20882            .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
20883        }
20884
20885        /// The scope most tests open under: aft carries to any module, broca
20886        /// only to plexus and other, and the owner delegates as agent-1.
20887        fn session(scope_epoch: u64) -> ScopeRecord {
20888            let mut record = head("s", scope_epoch);
20889            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
20890            record.attributes.agent_id = Some("agent-1".to_string());
20891            record.attributes.delegates = true;
20892            record
20893        }
20894
20895        impl Rig {
20896            async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
20897                self.generation += 1;
20898                sync(&self.handler, &self.owner, self.generation, scopes)
20899                    .await
20900                    .expect("the owner's sync is accepted");
20901            }
20902
20903            fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
20904                ScopeSelector::new(
20905                    Principal::Reserved {
20906                        module_id: OWNER.to_string(),
20907                    },
20908                    scope_ref.to_string(),
20909                    scope_epoch,
20910                )
20911            }
20912
20913            fn open_frame(
20914                &mut self,
20915                opener: Option<&str>,
20916                target: &str,
20917                scope: Option<ScopeSelector>,
20918            ) -> (
20919                RouteCtx,
20920                mpsc::Receiver<crate::router::OutboundFrame>,
20921                Frame,
20922            ) {
20923                self.next_connection += 1;
20924                let (ctx, rx) = wide_ctx(self.next_connection);
20925                let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
20926                    busy_refusals: None,
20927                    target: RouteTarget::ToolProvider {
20928                        module_id: target.to_string(),
20929                    },
20930                    identity: BindIdentity::new("unit".to_string(), "session".to_string()),
20931                    consumer_identity: opener.map(|module_id| ConsumerIdentity {
20932                        module_id: module_id.to_string(),
20933                        launch_nonce: nonce(module_id),
20934                    }),
20935                    consumer_capabilities: None,
20936                    role_versions: None,
20937                    admission_facts: None,
20938                    scope,
20939                })
20940                .unwrap();
20941                let frame = Frame::build(
20942                    FrameType::Request,
20943                    control_flags(),
20944                    0,
20945                    0,
20946                    self.next_connection,
20947                    body,
20948                )
20949                .unwrap();
20950                (ctx, rx, frame)
20951            }
20952
20953            /// Open and expect a refusal before anything is relayed.
20954            async fn refused(
20955                &mut self,
20956                opener: Option<&str>,
20957                target: &str,
20958                scope: Option<ScopeSelector>,
20959            ) -> String {
20960                self.refusal_body(opener, target, scope).await["code"]
20961                    .as_str()
20962                    .unwrap()
20963                    .to_string()
20964            }
20965
20966            async fn refusal_body(
20967                &mut self,
20968                opener: Option<&str>,
20969                target: &str,
20970                scope: Option<ScopeSelector>,
20971            ) -> Value {
20972                let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
20973                let replies = tokio::time::timeout(
20974                    Duration::from_secs(2),
20975                    self.handler.handle_control_frame(&ctx, frame),
20976                )
20977                .await
20978                .expect("the open must be refused before waiting for a bind ack")
20979                .unwrap();
20980                assert_eq!(replies.len(), 1, "{replies:?}");
20981                assert_eq!(replies[0].header.ty, FrameType::Error);
20982                let (_, module_rx) = self.modules.get_mut(target).unwrap();
20983                assert!(
20984                    module_rx.try_recv().is_err(),
20985                    "a refused open relays nothing"
20986                );
20987                assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
20988                parse_error(&replies[0])
20989            }
20990
20991            /// Start an open and return its task and the bind the target got.
20992            async fn relayed(
20993                &mut self,
20994                opener: Option<&str>,
20995                target: &str,
20996                scope: Option<ScopeSelector>,
20997            ) -> Relayed {
20998                self.relayed_with_legacy_root(opener, target, scope, None)
20999                    .await
21000            }
21001
21002            async fn relayed_with_legacy_root(
21003                &mut self,
21004                opener: Option<&str>,
21005                target: &str,
21006                scope: Option<ScopeSelector>,
21007                legacy_root: Option<&std::path::Path>,
21008            ) -> Relayed {
21009                let (ctx, rx, mut frame) = self.open_frame(opener, target, scope);
21010                if let Some(root) = legacy_root {
21011                    let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
21012                    body["identity"]["project_root"] = serde_json::json!(root);
21013                    frame.body = serde_json::to_vec(&body).unwrap();
21014                }
21015                let handler = self.handler.clone();
21016                let task_ctx = ctx.clone();
21017                let task = tokio::spawn(async move {
21018                    handler
21019                        .handle_control_frame(&task_ctx, frame)
21020                        .await
21021                        .unwrap()
21022                });
21023                let (_, module_rx) = self.modules.get_mut(target).unwrap();
21024                let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
21025                    .await
21026                    .expect("the target receives the relayed route.bind")
21027                    .unwrap()
21028                    .frame;
21029                Relayed {
21030                    target: target.to_string(),
21031                    client: ctx,
21032                    client_rx: rx,
21033                    task,
21034                    bind,
21035                }
21036            }
21037
21038            async fn ack(&self, relayed: &Relayed) {
21039                let (module, _) = &self.modules[&relayed.target];
21040                self.handler
21041                    .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
21042                    .await
21043                    .unwrap();
21044            }
21045
21046            /// Open, ack and return the bound route.
21047            async fn bound(
21048                &mut self,
21049                opener: Option<&str>,
21050                target: &str,
21051                scope: Option<ScopeSelector>,
21052            ) -> Bound {
21053                let relayed = self.relayed(opener, target, scope).await;
21054                self.commit_relayed(relayed).await
21055            }
21056
21057            async fn bound_with_legacy_root(
21058                &mut self,
21059                opener: Option<&str>,
21060                target: &str,
21061                scope: Option<ScopeSelector>,
21062                root: &std::path::Path,
21063            ) -> Bound {
21064                let relayed = self
21065                    .relayed_with_legacy_root(opener, target, scope, Some(root))
21066                    .await;
21067                self.commit_relayed(relayed).await
21068            }
21069
21070            async fn commit_relayed(&self, relayed: Relayed) -> Bound {
21071                self.ack(&relayed).await;
21072                let Relayed {
21073                    target,
21074                    client,
21075                    mut client_rx,
21076                    task,
21077                    bind,
21078                } = relayed;
21079                assert!(
21080                    task.await.unwrap().is_empty(),
21081                    "the open is answered by commit"
21082                );
21083                let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
21084                Bound {
21085                    target,
21086                    client,
21087                    client_rx,
21088                    channel,
21089                    epoch,
21090                    bind,
21091                }
21092            }
21093
21094            fn live(&self, route: &Bound) -> bool {
21095                matches!(
21096                    self.forwarding
21097                        .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
21098                        .unwrap(),
21099                    DataRoute::Client(DataRouteState::Bound(_))
21100                )
21101            }
21102        }
21103
21104        struct Relayed {
21105            target: String,
21106            client: RouteCtx,
21107            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
21108            task: tokio::task::JoinHandle<Vec<Frame>>,
21109            bind: Frame,
21110        }
21111
21112        struct Bound {
21113            target: String,
21114            client: RouteCtx,
21115            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
21116            channel: u16,
21117            epoch: u32,
21118            bind: Frame,
21119        }
21120
21121        impl Bound {
21122            /// The reason of the `route.closed` this client was sent, after
21123            /// checking it also got a GOODBYE on exactly this route.
21124            fn closed_reason(&mut self) -> RouteCloseReason {
21125                let mut reason = None;
21126                let mut goodbye = false;
21127                while let Ok(outbound) = self.client_rx.try_recv() {
21128                    let frame = outbound.frame;
21129                    match frame.header.ty {
21130                        FrameType::Goodbye => {
21131                            assert_eq!(
21132                                (frame.header.channel, frame.header.epoch),
21133                                (self.channel, self.epoch)
21134                            );
21135                            goodbye = true;
21136                        }
21137                        FrameType::Push => {
21138                            let ClientControlPush::RouteClosed {
21139                                reason: r,
21140                                module_id,
21141                                ..
21142                            } = serde_json::from_slice(&frame.body).unwrap()
21143                            else {
21144                                panic!("unexpected push");
21145                            };
21146                            assert_eq!(module_id, self.target);
21147                            reason = Some(r);
21148                        }
21149                        other => panic!("unexpected frame {other:?}"),
21150                    }
21151                }
21152                assert!(goodbye, "the client is sent a GOODBYE for the closed route");
21153                reason.expect("the client is told why the route closed")
21154            }
21155
21156            fn untouched(&mut self) -> bool {
21157                self.client_rx.try_recv().is_err()
21158            }
21159
21160            fn stamp(&self) -> Option<ScopeStamp> {
21161                match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
21162                    ModuleControlRequest::RouteBind { scope, .. } => scope,
21163                    other => panic!("expected a route.bind, got {other:?}"),
21164                }
21165            }
21166        }
21167
21168        #[tokio::test]
21169        async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
21170        ) {
21171            let mut rig = rig().await;
21172            let mut record = session(1);
21173            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
21174            record.child_owners = vec![Principal::Reserved {
21175                module_id: MAGIC.to_string(),
21176            }];
21177            rig.sync(vec![record]).await;
21178            let scope = || Some(rig_selector("s", Some(1)));
21179
21180            // Admitted: the owner, a bare carrier to any module, a targeted
21181            // carrier to its listed module.
21182            rig.bound(Some(OWNER), PLEXUS, scope()).await;
21183            rig.bound(Some(AFT), OTHER, scope()).await;
21184            rig.bound(Some(BROCA), PLEXUS, scope()).await;
21185
21186            // Refused scope_not_carrier: a targeted carrier to an unlisted
21187            // module, a module that is not listed at all (a child owner is not
21188            // a carrier), and a direct key-holder.
21189            for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
21190                assert_eq!(
21191                    rig.refused(opener, target, scope()).await,
21192                    error_codes::SCOPE_NOT_CARRIER,
21193                    "{opener:?} -> {target}"
21194                );
21195            }
21196        }
21197
21198        fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
21199            ScopeSelector::new(
21200                Principal::Reserved {
21201                    module_id: OWNER.to_string(),
21202                },
21203                scope_ref.to_string(),
21204                scope_epoch,
21205            )
21206        }
21207
21208        #[tokio::test]
21209        async fn an_open_without_an_epoch_is_refused_the_owners_included() {
21210            let mut rig = rig().await;
21211            rig.sync(vec![session(1)]).await;
21212            for opener in [OWNER, AFT] {
21213                assert_eq!(
21214                    rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
21215                        .await,
21216                    error_codes::SCOPE_EPOCH_REQUIRED,
21217                    "{opener}"
21218                );
21219            }
21220        }
21221
21222        #[tokio::test]
21223        async fn admission_separates_not_synced_not_live_and_ended() {
21224            let mut rig = rig().await;
21225            // Before the configured owner's first sync: retryable.
21226            let code = rig
21227                .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21228                .await;
21229            assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
21230            assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
21231
21232            // An owner that is not configured will never sync: terminal.
21233            let ghost = ScopeSelector::new(
21234                Principal::Reserved {
21235                    module_id: "ghost".to_string(),
21236                },
21237                "s".to_string(),
21238                Some(1),
21239            );
21240            assert_eq!(
21241                rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
21242                error_codes::SCOPE_NOT_LIVE
21243            );
21244
21245            rig.sync(vec![session(2)]).await;
21246            assert_eq!(
21247                rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
21248                    .await,
21249                error_codes::SCOPE_NOT_LIVE
21250            );
21251            for epoch in [1, 3] {
21252                assert_eq!(
21253                    rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
21254                        .await,
21255                    error_codes::SCOPE_ENDED,
21256                    "epoch {epoch}"
21257                );
21258            }
21259            // Control: the live epoch is admitted.
21260            rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
21261                .await;
21262        }
21263
21264        #[tokio::test]
21265        async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
21266            let mut rig = rig().await;
21267            rig.sync(vec![session(1)]).await;
21268            let route = rig
21269                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21270                .await;
21271            let stamp = route.stamp().expect("a scoped bind carries the stamp");
21272            assert_eq!(stamp.scope_ref, "s");
21273            assert_eq!(stamp.scope_epoch, 1);
21274            assert_eq!(stamp.kind, ScopeKind::Head);
21275            assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
21276            assert!(stamp.attributes.delegates);
21277            assert!(stamp.owner_authorized);
21278            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
21279            assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
21280
21281            // broca owns a scope of its own on its own module connection; it is
21282            // not in scope_authority_owners, so its stamp is not authorized.
21283            let (broca, mut broca_rx) = wide_ctx(50);
21284            hello_via_sink(
21285                &rig.handler,
21286                &broca,
21287                &mut broca_rx,
21288                hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
21289            )
21290            .await;
21291            sync(&rig.handler, &broca, 1, vec![head("b", 1)])
21292                .await
21293                .unwrap();
21294            let own = ScopeSelector::new(
21295                Principal::Reserved {
21296                    module_id: BROCA.to_string(),
21297                },
21298                "b".to_string(),
21299                Some(1),
21300            );
21301            let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
21302            assert!(!route.stamp().unwrap().owner_authorized);
21303        }
21304
21305        #[tokio::test]
21306        async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
21307            let mut rig = rig().await;
21308            let mut record = head("s", 1);
21309            record.carriers = vec![carrier(AFT, None)];
21310            let flow_id = "Flow:run-7/step_2!~";
21311            record.attributes.flow_id = Some(flow_id.to_string());
21312            let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
21313                .await
21314                .unwrap();
21315            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
21316                panic!("not a sync reply");
21317            };
21318            assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
21319            let route = rig
21320                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21321                .await;
21322            assert!(rig.live(&route), "the stamped bind committed");
21323            let stamp = route.stamp().expect("a flow scope carries a stamp");
21324            assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
21325            assert_eq!(stamp.attributes.agent_id, None);
21326            assert!(!stamp.attributes.delegates);
21327            assert!(stamp.owner_authorized);
21328            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
21329            assert_eq!(unscoped.stamp(), None);
21330        }
21331
21332        #[tokio::test]
21333        async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
21334            let mut rig = rig_with_flow_support(false).await;
21335            let mut record = session(1);
21336            record.attributes.flow_id = Some("flow:7".to_string());
21337            rig.sync(vec![record]).await;
21338            for opener in [OWNER, AFT] {
21339                let body = rig
21340                    .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
21341                    .await;
21342                assert_eq!(body["code"], "target_flow_unsupported");
21343                let message = body["message"].as_str().unwrap();
21344                for required in [PLEXUS, "flow-scopes/v1"] {
21345                    assert!(message.contains(required), "{message}");
21346                }
21347            }
21348        }
21349
21350        #[tokio::test]
21351        async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
21352            let mut rig = rig_with_flow_support(true).await;
21353            let mut record = session(1);
21354            record.attributes.flow_id = Some("flow:7".to_string());
21355            rig.sync(vec![record]).await;
21356            for opener in [OWNER, AFT] {
21357                let route = rig
21358                    .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
21359                    .await;
21360                assert!(rig.live(&route));
21361                assert_eq!(
21362                    route.stamp().unwrap().attributes.flow_id.as_deref(),
21363                    Some("flow:7")
21364                );
21365            }
21366        }
21367
21368        #[tokio::test]
21369        async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
21370            use std::future::Future;
21371
21372            let mut rig = rig_with_flow_support(true).await;
21373            let mut record = session(1);
21374            record.attributes.flow_id = Some("flow:7".to_string());
21375            rig.sync(vec![record]).await;
21376            let (client, mut client_rx, frame) =
21377                rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
21378            // Hold the route-open response permit so admission sees the first
21379            // target but relay reservation cannot capture an endpoint yet.
21380            for _ in 0..64 {
21381                client.egress.try_send(route_bind_ack(1)).unwrap();
21382            }
21383            let handler = rig.handler.clone();
21384            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
21385            std::future::poll_fn(|cx| {
21386                assert!(open.as_mut().poll(cx).is_pending());
21387                std::task::Poll::Ready(())
21388            })
21389            .await;
21390            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21391
21392            let old_connection = rig.modules[PLEXUS].0.connection_id;
21393            rig.handler.cleanup_connection(old_connection).unwrap();
21394            let (replacement, mut replacement_rx) = wide_ctx(200);
21395            let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
21396            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
21397            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
21398            let hello = Frame::build(
21399                FrameType::Hello,
21400                control_flags(),
21401                0,
21402                0,
21403                200,
21404                serde_json::to_vec(&body).unwrap(),
21405            )
21406            .unwrap();
21407            hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
21408
21409            client_rx.try_recv().unwrap();
21410            let replies = tokio::time::timeout(Duration::from_secs(2), open)
21411                .await
21412                .expect("the replacement is refused without waiting for a bind ack")
21413                .unwrap();
21414            assert_eq!(replies.len(), 1);
21415            let body = parse_error(&replies[0]);
21416            assert_eq!(body["code"], "target_flow_unsupported");
21417            for required in [PLEXUS, "flow-scopes/v1"] {
21418                assert!(body["message"].as_str().unwrap().contains(required));
21419            }
21420            assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
21421            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
21422            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21423            assert!(rig
21424                .handler
21425                .registry
21426                .get_module_by_connection(replacement.connection_id)
21427                .unwrap()
21428                .is_some());
21429        }
21430
21431        #[tokio::test]
21432        async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
21433            use std::future::Future;
21434            const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
21435            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
21436            const NEXT_HASH: &str =
21437                "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
21438            let mut rig = rig().await;
21439            let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
21440            record.carriers.push(ScopeCarrier::new(Principal::Direct));
21441            rig.sync(vec![record.clone()]).await;
21442            let (client, mut rx, frame) = rig.open_frame(
21443                None,
21444                PLEXUS,
21445                Some(
21446                    rig.selector("s", Some(1))
21447                        .with_bind_proof(Some(PROOF.into())),
21448                ),
21449            );
21450            // Hold egress reservation while the scope owner changes its bind-proof
21451            // hash, so a proof valid at admission must be checked again at relay.
21452            for _ in 0..64 {
21453                client.egress.try_send(route_bind_ack(1)).unwrap();
21454            }
21455            let handler = rig.handler.clone();
21456            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
21457            std::future::poll_fn(|cx| {
21458                assert!(open.as_mut().poll(cx).is_pending());
21459                std::task::Poll::Ready(())
21460            })
21461            .await;
21462            record.bind_proof_sha256 = Some(NEXT_HASH.into());
21463            rig.sync(vec![record]).await;
21464            rx.try_recv().unwrap();
21465            let replies = tokio::time::timeout(Duration::from_secs(2), open)
21466                .await
21467                .unwrap()
21468                .unwrap();
21469            assert_eq!(
21470                parse_error(&replies[0])["code"],
21471                error_codes::SCOPE_BIND_PROOF_MISMATCH
21472            );
21473            assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
21474            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
21475            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21476        }
21477
21478        #[tokio::test]
21479        async fn scope_bind_proof_reserved_openers_ignore_proof() {
21480            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
21481            let mut rig = rig().await;
21482            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
21483                .await;
21484            for opener in [OWNER, AFT] {
21485                for proof in [None, Some("malformed+proof=")] {
21486                    let route = rig
21487                        .bound(
21488                            Some(opener),
21489                            PLEXUS,
21490                            Some(
21491                                rig.selector("s", Some(1))
21492                                    .with_bind_proof(proof.map(str::to_string)),
21493                            ),
21494                        )
21495                        .await;
21496                    assert!(rig.live(&route));
21497                }
21498            }
21499        }
21500
21501        #[tokio::test]
21502        async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
21503            use std::future::Future;
21504            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
21505            const NEXT_HASH: &str =
21506                "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
21507            let mut rig = rig().await;
21508            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
21509                .await;
21510            let (client, mut rx, frame) =
21511                rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
21512            for _ in 0..64 {
21513                client.egress.try_send(route_bind_ack(1)).unwrap();
21514            }
21515            let handler = rig.handler.clone();
21516            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
21517            std::future::poll_fn(|cx| {
21518                assert!(open.as_mut().poll(cx).is_pending());
21519                std::task::Poll::Ready(())
21520            })
21521            .await;
21522            rig.sync(vec![
21523                session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
21524            ])
21525            .await;
21526            rx.try_recv().unwrap();
21527            let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
21528            let bind = std::future::poll_fn(|cx| {
21529                assert!(open.as_mut().poll(cx).is_pending());
21530                module_rx.poll_recv(cx)
21531            })
21532            .await
21533            .unwrap()
21534            .frame;
21535            handler
21536                .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
21537                .await
21538                .unwrap();
21539            let replies = tokio::time::timeout(Duration::from_secs(2), open)
21540                .await
21541                .unwrap()
21542                .unwrap();
21543            assert_eq!(replies.len(), 1);
21544            // Proof rechecking allowed the relay. Independent tag settlement
21545            // still asks the opener to retry after any same-epoch content change.
21546            assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
21547            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21548            while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
21549            let route = rig
21550                .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
21551                .await;
21552            assert!(rig.live(&route));
21553        }
21554
21555        #[tokio::test]
21556        async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
21557        {
21558            let mut rig = rig_with_flow_support(false).await;
21559            rig.sync(vec![session(1)]).await;
21560            let route = rig
21561                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21562                .await;
21563            assert!(rig.live(&route));
21564            assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
21565            let mut record = session(1);
21566            record.attributes.flow_id = Some("flow:7".to_string());
21567            rig.sync(vec![record]).await;
21568            let unscoped = rig.bound(Some(AFT), OTHER, None).await;
21569            assert!(rig.live(&unscoped));
21570            assert_eq!(unscoped.stamp(), None);
21571        }
21572
21573        #[tokio::test]
21574        async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
21575            let mut rig = rig().await;
21576            let mut record = session(1);
21577            record.attributes.flow_id = Some("flow:7".to_string());
21578            rig.sync(vec![record.clone()]).await;
21579            let before = rig.forwarding.published_scope_tag(OWNER, "s");
21580            let mut owner_route = rig
21581                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
21582                .await;
21583            let mut carrier_route = rig
21584                .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
21585                .await;
21586            let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
21587            rig.sync(vec![record.clone()]).await;
21588            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
21589            assert!(rig.live(&owner_route) && owner_route.untouched());
21590            assert!(rig.live(&carrier_route) && carrier_route.untouched());
21591
21592            record.attributes.flow_id = Some("flow:8".to_string());
21593            rig.sync(vec![record]).await;
21594            let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
21595            let before = before.unwrap();
21596            assert_eq!(after.scope_epoch, before.scope_epoch);
21597            assert!(after.version > before.version);
21598            for route in [&mut owner_route, &mut carrier_route] {
21599                assert!(!rig.live(route));
21600                assert_eq!(
21601                    route.closed_reason(),
21602                    RouteCloseReason::ScopeDelegationChanged
21603                );
21604            }
21605            assert!(rig.live(&unscoped) && unscoped.untouched());
21606            // Each provider also receives a GOODBYE for its drained route;
21607            // consume it before expecting the next route.bind on that sink.
21608            for target in [PLEXUS, OTHER] {
21609                let (_, module_rx) = rig.modules.get_mut(target).unwrap();
21610                let goodbye = module_rx
21611                    .try_recv()
21612                    .expect("the provider sees the drain")
21613                    .frame;
21614                assert_eq!(goodbye.header.ty, FrameType::Goodbye);
21615                assert!(module_rx.try_recv().is_err());
21616            }
21617            let rebound = rig
21618                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21619                .await;
21620            assert_eq!(
21621                rebound.stamp().unwrap().attributes.flow_id.as_deref(),
21622                Some("flow:8")
21623            );
21624        }
21625
21626        /// The owner's sync lands between admission and the module's ack. The
21627        /// open is refused by name, the module's other routes stay up, and the
21628        /// reserved pair is released. Changed content is retryable; an ended
21629        /// scope is not.
21630        #[tokio::test]
21631        async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
21632            let mut rig = rig().await;
21633            rig.sync(vec![session(1)]).await;
21634            let mut cotenant = rig
21635                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
21636                .await;
21637
21638            let mut changed = session(1);
21639            changed.child_owners.push(Principal::Reserved {
21640                module_id: MAGIC.to_string(),
21641            });
21642            let mut ended = None;
21643            for (code, next) in [
21644                (error_codes::SCOPE_CHANGED, vec![changed]),
21645                (error_codes::SCOPE_ENDED, Vec::new()),
21646            ] {
21647                let relayed = rig
21648                    .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21649                    .await;
21650                let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
21651                ended = Some(next.is_empty());
21652                rig.sync(next).await;
21653                rig.ack(&relayed).await;
21654                let replies = relayed.task.await.unwrap();
21655                assert_eq!(replies.len(), 1, "{replies:?}");
21656                assert_eq!(parse_error(&replies[0])["code"], code);
21657                assert_eq!(
21658                    subc_protocol::error_codes::is_retryable_route_open(code),
21659                    code == error_codes::SCOPE_CHANGED
21660                );
21661                assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21662                // The module is told to drop just the binding it created.
21663                let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
21664                // Collected, because ending the scope also closes the co-tenant
21665                // route, whose GOODBYE comes first.
21666                let mut goodbyes = Vec::new();
21667                while let Ok(outbound) = plexus_rx.try_recv() {
21668                    assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
21669                    goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
21670                }
21671                assert!(
21672                    goodbyes.contains(&(bind_channel, bind_epoch)),
21673                    "{goodbyes:?}"
21674                );
21675                assert!(rig
21676                    .handler
21677                    .registry
21678                    .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
21679                    .unwrap()
21680                    .is_some());
21681            }
21682            assert_eq!(ended, Some(true));
21683            // The co-tenant stayed up through the change, and closed only when
21684            // the scope ended, by the drain rule rather than by the commit.
21685            assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
21686        }
21687
21688        /// Each row of the drain table on one set of routes: the owner's, a
21689        /// bare carrier's, and a targeted carrier's to each of its targets.
21690        #[tokio::test]
21691        async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
21692            struct Case {
21693                name: &'static str,
21694                change: fn(&mut ScopeRecord),
21695                /// Closed routes by index: owner->plexus, aft->plexus,
21696                /// broca->plexus, broca->other.
21697                closed: [Option<RouteCloseReason>; 4],
21698            }
21699            use RouteCloseReason::*;
21700            let cases = [
21701                Case {
21702                    name: "a carrier entry removed",
21703                    change: |r| {
21704                        r.carriers.retain(|c| {
21705                            c.principal
21706                                != Principal::Reserved {
21707                                    module_id: AFT.to_string(),
21708                                }
21709                        })
21710                    },
21711                    closed: [None, Some(ScopeCarrierRemoved), None, None],
21712                },
21713                Case {
21714                    name: "a target removed from a carrier",
21715                    change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
21716                    closed: [None, None, None, Some(ScopeCarrierRemoved)],
21717                },
21718                Case {
21719                    name: "a bare carrier narrowed to targets",
21720                    change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
21721                    closed: [None, Some(ScopeCarrierRemoved), None, None],
21722                },
21723                Case {
21724                    name: "delegates turned off",
21725                    change: |r| r.attributes.delegates = false,
21726                    closed: [Some(ScopeDelegationChanged); 4],
21727                },
21728                Case {
21729                    name: "agent_id changed",
21730                    change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
21731                    closed: [Some(ScopeDelegationChanged); 4],
21732                },
21733                Case {
21734                    name: "a carrier added, child owners changed, the record re-sent",
21735                    change: |r| {
21736                        r.carriers.push(carrier(MAGIC, None));
21737                        r.child_owners.push(Principal::Reserved {
21738                            module_id: MAGIC.to_string(),
21739                        });
21740                    },
21741                    closed: [None; 4],
21742                },
21743                Case {
21744                    name: "a target added",
21745                    change: |r| {
21746                        r.carriers[1]
21747                            .targets
21748                            .as_mut()
21749                            .unwrap()
21750                            .push("third".to_string())
21751                    },
21752                    closed: [None; 4],
21753                },
21754                Case {
21755                    name: "delegates turned on",
21756                    change: |r| r.attributes.delegates = true,
21757                    closed: [None; 4],
21758                },
21759            ];
21760            for case in cases {
21761                let mut rig = rig().await;
21762                rig.sync(vec![session(1)]).await;
21763                let scope = || Some(rig_selector("s", Some(1)));
21764                let mut routes = [
21765                    rig.bound(Some(OWNER), PLEXUS, scope()).await,
21766                    rig.bound(Some(AFT), PLEXUS, scope()).await,
21767                    rig.bound(Some(BROCA), PLEXUS, scope()).await,
21768                    rig.bound(Some(BROCA), OTHER, scope()).await,
21769                ];
21770                let mut record = session(1);
21771                (case.change)(&mut record);
21772                rig.sync(vec![record]).await;
21773                for (index, expected) in case.closed.iter().enumerate() {
21774                    let route = &mut routes[index];
21775                    match expected {
21776                        Some(reason) => {
21777                            assert!(!rig.live(route), "{}: route {index} still live", case.name);
21778                            assert_eq!(
21779                                route.closed_reason(),
21780                                *reason,
21781                                "{}: route {index}",
21782                                case.name
21783                            );
21784                        }
21785                        None => {
21786                            assert!(rig.live(route), "{}: route {index} closed", case.name);
21787                            assert!(
21788                                route.untouched(),
21789                                "{}: route {index} was told something",
21790                                case.name
21791                            );
21792                        }
21793                    }
21794                }
21795            }
21796        }
21797
21798        #[tokio::test]
21799        async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
21800            // Removed, and replaced by a higher epoch.
21801            for next in [Vec::new(), vec![session(2)]] {
21802                let mut rig = rig().await;
21803                rig.sync(vec![session(1)]).await;
21804                let mut route = rig
21805                    .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21806                    .await;
21807                rig.sync(next).await;
21808                assert!(!rig.live(&route));
21809                assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
21810            }
21811
21812            // A child whose parent ends: its routes close as parent-ended, the
21813            // child stays live, and routes under the parent close as ended.
21814            let mut rig = rig().await;
21815            let mut child = session(1);
21816            child.scope_ref = "child".to_string();
21817            child.kind = ScopeKind::Worker;
21818            child.parent = Some(ScopeParent::new(
21819                Principal::Reserved {
21820                    module_id: OWNER.to_string(),
21821                },
21822                "s".to_string(),
21823                1,
21824            ));
21825            rig.sync(vec![session(1), child.clone()]).await;
21826            let mut child_route = rig
21827                .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
21828                .await;
21829            assert_eq!(
21830                child_route.stamp().unwrap().parent_state,
21831                Some(ParentState::Linked)
21832            );
21833            rig.sync(vec![child]).await;
21834            assert!(!rig.live(&child_route));
21835            assert_eq!(
21836                child_route.closed_reason(),
21837                RouteCloseReason::ScopeParentEnded
21838            );
21839        }
21840
21841        #[tokio::test]
21842        async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
21843        ) {
21844            let mut rig = rig().await;
21845            rig.sync(vec![session(1)]).await;
21846            let mut route = rig
21847                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21848                .await;
21849            let before = rig.forwarding.published_scope_tag(OWNER, "s");
21850            rig.sync(vec![session(1)]).await;
21851            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
21852            assert!(rig.live(&route) && route.untouched());
21853
21854            // A call in flight on the route when another carrier is added. A
21855            // forwarded REQUEST holds one credit on the route's flow until the
21856            // module answers; the router takes it exactly like this.
21857            let DataRoute::Client(DataRouteState::Bound(binding)) = rig
21858                .forwarding
21859                .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
21860                .unwrap()
21861            else {
21862                panic!("the route is bound");
21863            };
21864            binding.flow.acquire_tagged(9, false).await.unwrap();
21865            let mut widened = session(1);
21866            widened.carriers.push(carrier(MAGIC, None));
21867            rig.sync(vec![widened]).await;
21868            assert!(rig.live(&route) && route.untouched());
21869            let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
21870            assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
21871            // The call's credit is still held on an open flow, so its answer
21872            // will be delivered: closing the route would have closed the flow.
21873            assert_eq!(binding.flow.in_flight(), 1);
21874            binding
21875                .flow
21876                .acquire_tagged(10, false)
21877                .await
21878                .expect("the flow is still open");
21879        }
21880
21881        /// A swap's superseded endpoint keeps its routes until drained; ending
21882        /// the scope closes them there too.
21883        #[tokio::test]
21884        async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
21885            let mut rig = rig().await;
21886            rig.sync(vec![session(1)]).await;
21887            let mut on_incumbent = rig
21888                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21889                .await;
21890
21891            // Swap plexus: register a candidate and cut over, leaving the
21892            // incumbent superseded with the route still on it.
21893            let (candidate, _candidate_rx) = wide_ctx(9);
21894            let registration = rig
21895                .handler
21896                .registry
21897                .register_candidate_with_control_ops(
21898                    manifest(PLEXUS, PROTOCOL_VERSION),
21899                    PROTOCOL_VERSION,
21900                    candidate.connection_id,
21901                    module_baseline_control_ops(),
21902                )
21903                .unwrap();
21904            rig.forwarding
21905                .register_candidate_module_connection(
21906                    candidate.connection_id,
21907                    PLEXUS.to_string(),
21908                    PROTOCOL_VERSION,
21909                    manifest_concurrency(&registration.manifest),
21910                    candidate.egress.clone(),
21911                )
21912                .unwrap();
21913            rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
21914            rig.handler
21915                .registry
21916                .promote_candidate(PLEXUS)
21917                .unwrap()
21918                .unwrap();
21919            assert!(rig.live(&on_incumbent), "cutover alone does not drain");
21920
21921            rig.sync(Vec::new()).await;
21922            assert!(!rig.live(&on_incumbent));
21923            assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
21924            let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
21925            let goodbye = incumbent_rx
21926                .try_recv()
21927                .expect("the superseded endpoint is told")
21928                .frame;
21929            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
21930        }
21931    }
21932}
21933
21934#[cfg(test)]
21935mod concurrency_default_exposure_tests {
21936    use super::*;
21937
21938    fn hello_body(role_json: &str) -> Vec<u8> {
21939        format!(
21940            r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
21941        )
21942        .into_bytes()
21943    }
21944
21945    fn manifest_from(body: &[u8]) -> ModuleManifest {
21946        let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
21947        serde_json::from_value(value.get("manifest").expect("manifest key").clone())
21948            .expect("manifest parses")
21949    }
21950
21951    const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
21952
21953    #[test]
21954    fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
21955        let body = hello_body(&format!(
21956            r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
21957        ));
21958        let manifest = manifest_from(&body);
21959        // Precondition: serde really resolved it to the default, so the typed
21960        // manifest alone cannot answer the question this probe exists for.
21961        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
21962        assert!(manifest_concurrency_was_defaulted(&body, &manifest));
21963    }
21964
21965    #[test]
21966    fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
21967        let body = hello_body(&format!(
21968            r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
21969        ));
21970        let manifest = manifest_from(&body);
21971        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
21972        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
21973    }
21974
21975    #[test]
21976    fn non_management_roles_are_never_reported() {
21977        let body = hello_body(
21978            r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
21979        );
21980        let manifest = manifest_from(&body);
21981        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
21982    }
21983}