1#[path = "control/list_diagnostics.rs"]
2pub(crate) mod list_diagnostics;
3
4use std::{
5 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
6 fmt,
7 net::SocketAddr,
8 path::{Path, PathBuf},
9 sync::{Arc, Mutex, RwLock},
10 time::{Duration, Instant as StdInstant},
11};
12
13use serde::{Deserialize, Serialize};
14use subc_control::{
15 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
16 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
17 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
18 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
19 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
20 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
21 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
22};
23use subc_protocol::{
24 call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
25 error_codes,
26 manifest::{
27 validate_hello_capability_grammar, validate_hello_confirm_operations,
28 validate_hello_display_name, validate_hello_event_declarations,
29 validate_hello_flow_operations, validate_hello_rich_ui_descriptor,
30 validate_hello_role_declarations, validate_hello_self_signal_declarations,
31 CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
32 ProviderRole,
33 },
34 peer::{ObservedPeer, RouteBindPeer, OBSERVED_PEER_CAPABILITY, OPERATOR_APP_CAPABILITY},
35 scope::{
36 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
37 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
38 SCOPE_SYNC_OP,
39 },
40 session::{
41 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
42 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
43 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
44 ROLE_VERSIONS_FIELD,
45 },
46 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
47 Priority, RouteTarget, PROTOCOL_VERSION,
48};
49use tokio::time::{timeout_at, Instant};
50use tracing::{debug, info, instrument::WithSubscriber, warn};
51
52use crate::{
53 caller_identity::CallerIdentity,
54 capability_requirements::{
55 log_duplicate_claim_events, log_manifest_change, log_requirement_events,
56 CapabilityRequirementEvaluator, CapabilityVerdict, DuplicateClaimSource, RegisteredModule,
57 RequirementStatus, RuntimeModule,
58 },
59 child_roster::DaemonShutdownFlag,
60 daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
61 forwarding::{
62 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
63 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
64 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
65 },
66 observability::{
67 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
68 },
69 privileged_prompt::{prompt_text, PrivilegedOperation},
70 provenance::{
71 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
72 },
73 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
74 router::{daemon_error_flags, RouteCtx, RouterError},
75 scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
76 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
77 stderr_tail::{CaptureState, TailEntry},
78 supervise::{
79 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
80 SupervisorHandle, SwapHelloAdmission,
81 },
82 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
83};
84
85pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
91
92const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
93const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
94const CAP_PING_PONG: &str = "ping_pong_v1";
95const CAP_SESSION_ATTACH: &str = "session_attach_v1";
96const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
97
98const SUBC_CONTROL_OPS: &[&str] = &[
99 ops::DAEMON_SHUTDOWN_INTENT,
100 ops::ACTIVITY_RECORD,
101 ops::ACTIVITY_QUERY,
102 ops::ACTIVITY_SUBSCRIBE,
103 ops::SERVER_DESCRIBE,
104 ops::CATALOG_LIST,
105 ops::ROUTE_OPEN,
106 ops::ROUTE_POLL,
107 ops::ROUTE_CLOSING,
108 ops::ROUTE_CLOSED,
109 ops::SUPERVISOR_LIST,
110 ops::SUPERVISOR_RESTART,
111 ops::SUPERVISOR_SWAP,
112 ops::SUPERVISOR_RELOAD,
113 ops::SUPERVISOR_RESCAN,
114 ops::SUPERVISOR_RELEASE_RESERVED,
115 ops::SUPERVISOR_SET_ENABLED,
116 ops::SUPERVISOR_HEALTH_PROBE,
117 ops::SUPERVISOR_HEALTH,
118 ops::SUPERVISOR_STDERR_TAIL,
119 ops::SUPERVISOR_TERMINALS,
120 ops::SUPERVISOR_ROUTES,
121 ops::SUPERVISOR_PROVENANCE,
122 ops::SUPERVISOR_SPAWN_SNAPSHOT,
123 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
124];
125
126const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[ops::SUPERVISOR_RESCAN_CHECKED];
133
134mod agent_commands;
135mod rescan_checked;
136use rescan_checked::{RescanMode, RescanPlan};
137
138const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
139 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
140 "supervisor.live_roots",
141 "supervisor.agent_commands",
142 SCOPE_SYNC_OP,
143 SCOPE_APPLY_OP,
144 SCOPE_DESCRIBE_OP,
145 "operator.confirm",
146];
147
148const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
152
153const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
154
155pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
163
164pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
179
180pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
198
199const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
200const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
201
202fn reload_verdict(
203 configured: &Path,
204 spawned_from: Option<&Path>,
205 image: subc_control::RunningImageAgreement,
206) -> PendingReloadVerdict {
207 let path = match spawned_from {
208 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
209 ReloadPathAgreement::Match
210 }
211 Some(spawned_from) => ReloadPathAgreement::Mismatch {
212 configured: configured.to_path_buf(),
213 spawned_from: spawned_from.to_path_buf(),
214 },
215 None => ReloadPathAgreement::Unavailable {
216 reason: if matches!(
217 image,
218 subc_control::RunningImageAgreement::Unavailable {
219 reason: subc_control::RunningImageUnavailableReason::NotRunning
220 }
221 ) {
222 ReloadPathUnavailableReason::NotRunning
223 } else {
224 ReloadPathUnavailableReason::SpawnedPathUnavailable
225 },
226 },
227 };
228 PendingReloadVerdict {
229 path,
230 image,
231 spec: None,
232 }
233}
234
235fn launch_spec_agreement(
236 configured: &crate::supervise::ModuleSpec,
237 running: bool,
238 spawned: Option<&crate::supervise::LaunchSpecDigest>,
239) -> subc_control::LaunchSpecAgreement {
240 use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
241 if !running {
242 return Agreement::Unavailable {
243 reason: Reason::NotRunning,
244 };
245 }
246 match spawned {
247 None => Agreement::Unavailable {
248 reason: Reason::SpawnedSpecUnavailable,
249 },
250 Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
251 Agreement::Match
252 }
253 Some(_) => Agreement::Mismatch,
254 }
255}
256
257fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
258 if configured == spawned_from {
259 return true;
260 }
261 #[cfg(windows)]
262 {
263 match (
269 std::fs::canonicalize(configured),
270 std::fs::canonicalize(spawned_from),
271 ) {
272 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
273 _ => false,
274 }
275 }
276 #[cfg(not(windows))]
277 {
278 false
279 }
280}
281
282#[derive(Clone)]
283struct DaemonProvenanceFacts {
284 build: DaemonBuildProvenance,
285 pid: Option<u32>,
286 started_at_ms: Option<u64>,
287 start_clock: Option<crate::clock::StartClock>,
288 executable_path: Option<PathBuf>,
289 executable_identity: Option<SpawnedFileIdentity>,
290 process_start_time: Option<u64>,
291 probe: ExecutableIdentityProbe,
292}
293
294impl Default for DaemonProvenanceFacts {
295 fn default() -> Self {
296 Self {
297 build: DaemonBuildProvenance {
298 build_git_sha: None,
299 build_lock_digest: None,
300 },
301 pid: None,
302 started_at_ms: None,
303 start_clock: None,
304 executable_path: None,
305 executable_identity: None,
306 process_start_time: None,
307 probe: ExecutableIdentityProbe::default(),
308 }
309 }
310}
311
312#[derive(Debug, Clone)]
313struct SupervisorRescanContext {
314 supervisor: Supervisor,
315 config_path: PathBuf,
316 configured_port: Option<u16>,
317 storage_config: Option<crate::daemon_config::StorageConfig>,
318 admission_facts_carrier_module_id: Option<String>,
319 admission_facts_targets: Option<Vec<String>>,
320 scope_authority_owners: Vec<String>,
321 privileged_operator_prompt: PrivilegedOperatorPrompt,
322 generation: Arc<std::sync::atomic::AtomicU64>,
328}
329
330const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
351 "reloading",
352 "supervisor_not_live",
353 "registration_not_active",
354 "no_forwarding_connection",
355 "relay_send_failed",
356];
357
358#[derive(Clone)]
360pub struct ControlHandler {
361 registry: Arc<Registry>,
362 activity: Option<crate::activity::ActivityLedger>,
363 activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
364 forwarding: Arc<ForwardingTable>,
365 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
366 supervisor: SupervisorHandle,
367 pub(crate) resource_tracker: Arc<crate::resource_totals::Tracker>,
368 agent_declarations: agent_commands::Queues,
369 list_admission: Arc<tokio::sync::Semaphore>,
370 subc_capabilities: Arc<[String]>,
371 route_bind_relay_timeout: Duration,
375 route_bind_relay_timeouts: BTreeMap<String, Duration>,
379 route_bind_breakers: RouteBindBreakers,
382 route_bind_concurrency: RouteBindConcurrency,
385 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
390 route_bind_breaker_threshold: u32,
392 route_bind_breaker_cooldown: Duration,
394 health_probe_timeout: Duration,
395 storage_config: Option<crate::daemon_config::StorageConfig>,
398 machine_id: Option<crate::machine_id::MachineId>,
402 admission_facts_carrier_module_id: Option<String>,
403 admission_facts_targets: Option<Vec<String>>,
404 scopes: Arc<RwLock<ScopeTable>>,
408 scope_deadline_added: Arc<tokio::sync::Notify>,
412 scope_authority_owners: Vec<String>,
415 privileged_operator_prompt: PrivilegedOperatorPrompt,
417 caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
420 build_class_warned: Arc<Mutex<HashSet<ConnectionId>>>,
421 observed_peers: Arc<Mutex<HashMap<ConnectionId, ObservedPeer>>>,
422 local_connections: Arc<Mutex<HashSet<ConnectionId>>>,
423 pub(crate) operator_app_policy: crate::operator_apps::OperatorAppPolicy,
424 pub(crate) operator_apps:
425 Arc<Mutex<HashMap<ConnectionId, crate::operator_apps::OperatorAppStamp>>>,
426 pub(crate) launch_table: std::sync::Arc<crate::app_open::LaunchTable>,
431 pub(crate) app_open_launch_lock: std::sync::Arc<tokio::sync::Mutex<()>>,
437 #[cfg(test)]
439 pub(crate) app_open_test_platform: Option<crate::operator_apps::AppOpenPlatform>,
440 #[cfg(test)]
441 pub(crate) app_open_test_launch: Option<fn() -> Result<u32, crate::app_open::AppOpenRefusal>>,
442 #[allow(dead_code)]
446 #[cfg(windows)]
447 pub(crate) windows_verify: Option<crate::app_open::WindowsVerifyFn>,
448 #[allow(dead_code)]
449 #[cfg(windows)]
450 pub(crate) windows_flash: Option<crate::app_open::WindowsFlashFn>,
451 #[allow(dead_code)]
455 #[cfg(windows)]
456 pub(crate) windows_track: Option<crate::app_open::WindowsTrackFn>,
457 #[cfg(all(test, windows))]
460 pub(crate) windows_spawn_attempts: Arc<std::sync::atomic::AtomicU64>,
461 #[cfg(test)]
462 caller_lookup: Option<TestCallerLookup>,
463 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
466 rescan: Option<SupervisorRescanContext>,
467 connected_clients: ConnectedClients,
468 counters: DaemonCounters,
469 local_endpoint_unavailable: Option<String>,
472 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
473 daemon_shutdown: DaemonShutdownFlag,
478 daemon_provenance: DaemonProvenanceFacts,
479 #[cfg(test)]
480 control_dispatch_delay: Option<Duration>,
481 #[cfg(test)]
482 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
483 #[cfg(test)]
487 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
488 #[cfg(test)]
494 hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
495}
496
497impl fmt::Debug for ControlHandler {
498 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
499 f.debug_struct("ControlHandler")
500 .field("registry", &self.registry)
501 .field("forwarding", &self.forwarding)
502 .field("process_liveness", &self.process_liveness.is_some())
503 .field("supervisor", &self.supervisor)
504 .field("subc_capabilities", &self.subc_capabilities)
505 .finish()
506 }
507}
508
509struct RouteOpenRequest {
510 target: RouteTarget,
511 identity: BindIdentity,
512 consumer_identity: Option<ConsumerIdentity>,
513 consumer_capabilities: Option<Vec<String>>,
514 role_versions: Option<BTreeMap<String, String>>,
515 busy_refusals: Option<bool>,
516 admission_facts: Option<serde_json::Value>,
517 scope: Option<ScopeSelector>,
518}
519
520#[derive(Clone, Copy)]
521enum RouteOpenPhase {
522 Admission,
523 ProjectRoot,
524 EgressReservation,
525 BindSend,
526 BindAckWait,
527}
528
529struct RouteOpenTimings {
533 module_id: String,
534 principal: String,
535 admission: Duration,
536 project_root: Duration,
537 egress_reservation: Duration,
538 bind_send: Duration,
539 bind_ack_wait: Duration,
540 phase: Option<RouteOpenPhase>,
541 phase_started_at: Instant,
542}
543
544impl RouteOpenTimings {
545 fn new() -> Self {
546 Self {
547 module_id: String::new(),
548 principal: "unresolved".to_string(),
550 admission: Duration::ZERO,
551 project_root: Duration::ZERO,
552 egress_reservation: Duration::ZERO,
553 bind_send: Duration::ZERO,
554 bind_ack_wait: Duration::ZERO,
555 phase: Some(RouteOpenPhase::Admission),
556 phase_started_at: Instant::now(),
557 }
558 }
559
560 fn finish(&mut self) {
561 let elapsed = self.phase_started_at.elapsed();
562 match self.phase.take() {
563 Some(RouteOpenPhase::Admission) => self.admission += elapsed,
564 Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
565 Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
566 Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
567 Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
568 None => {}
569 }
570 }
571
572 fn enter(&mut self, phase: RouteOpenPhase) {
573 self.finish();
574 self.phase = Some(phase);
575 self.phase_started_at = Instant::now();
576 }
577}
578
579struct RouteBindReservationGuard {
580 forwarding: Arc<ForwardingTable>,
581 endpoint: ModuleEndpointId,
582 relay_corr: u64,
583 armed: bool,
584}
585
586struct ModuleControlRpcGuard {
587 forwarding: Arc<ForwardingTable>,
588 endpoint: ModuleEndpointId,
589 corr: u64,
590 armed: bool,
591}
592
593impl ModuleControlRpcGuard {
594 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
595 Self {
596 forwarding,
597 endpoint,
598 corr,
599 armed: true,
600 }
601 }
602
603 fn disarm(&mut self) {
604 self.armed = false;
605 }
606}
607
608impl Drop for ModuleControlRpcGuard {
609 fn drop(&mut self) {
610 if self.armed {
611 let _ = self
612 .forwarding
613 .cancel_module_control_rpc(self.endpoint, self.corr);
614 }
615 }
616}
617
618impl RouteBindReservationGuard {
619 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
620 Self {
621 forwarding,
622 endpoint,
623 relay_corr,
624 armed: true,
625 }
626 }
627
628 fn release_and_disarm(&mut self) {
629 if !self.armed {
630 return;
631 }
632 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
633 self.endpoint,
634 self.relay_corr,
635 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
636 ) {
637 send_goodbye_target_best_effort(
638 &self.forwarding.counters(),
639 &target,
640 "canceled route.bind",
641 );
642 }
643 self.armed = false;
644 }
645
646 fn disarm(&mut self) {
647 self.armed = false;
648 }
649}
650
651impl Drop for RouteBindReservationGuard {
652 fn drop(&mut self) {
653 self.release_and_disarm();
654 }
655}
656
657#[derive(Debug, Clone, Default)]
687pub(crate) struct RouteBindBreakers {
688 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
689}
690
691#[derive(Debug, Clone, Default)]
692pub(crate) struct RouteBindConcurrency {
693 modules: Arc<Mutex<HashMap<String, usize>>>,
694}
695
696struct RouteBindConcurrencyGuard {
697 concurrency: RouteBindConcurrency,
698 module_id: String,
699}
700
701impl RouteBindConcurrency {
702 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
705 let mut modules = self
706 .modules
707 .lock()
708 .expect("route.bind concurrency mutex poisoned");
709 let in_flight = modules.entry(module_id.to_string()).or_default();
710 if *in_flight >= limit {
711 return Err(*in_flight);
712 }
713 *in_flight += 1;
714 Ok(RouteBindConcurrencyGuard {
715 concurrency: self.clone(),
716 module_id: module_id.to_string(),
717 })
718 }
719}
720
721impl Drop for RouteBindConcurrencyGuard {
722 fn drop(&mut self) {
723 let mut modules = self
724 .concurrency
725 .modules
726 .lock()
727 .expect("route.bind concurrency mutex poisoned");
728 let remove = {
729 let in_flight = modules
730 .get_mut(&self.module_id)
731 .expect("admitted route.bind has a concurrency entry");
732 *in_flight -= 1;
733 *in_flight == 0
734 };
735 if remove {
736 modules.remove(&self.module_id);
737 }
738 }
739}
740
741#[derive(Debug, Default)]
742struct ModuleBreakerState {
743 consecutive_timeouts: u32,
745 cooldown_until: Option<Instant>,
748 probe_in_flight: Option<Arc<()>>,
753}
754
755enum RouteBindAdmission<'a> {
757 Admitted {
758 guard: RouteBindBreakerGuard<'a>,
759 probe: bool,
762 },
763 Refused {
764 consecutive_timeouts: u32,
765 retry_in: Duration,
769 probe_in_flight: bool,
770 },
771}
772
773struct RouteBindBreakerGuard<'a> {
780 breakers: RouteBindBreakers,
781 module_id: &'a str,
782 probe_token: Option<Arc<()>>,
783 settled: bool,
784}
785
786impl RouteBindBreakerGuard<'_> {
787 fn record_accepted(&mut self) -> bool {
791 self.settled = true;
792 self.breakers.record_accepted(self.module_id)
793 }
794
795 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
798 self.settled = true;
799 self.breakers.record_timeout(
800 self.module_id,
801 self.probe_token.as_ref(),
802 threshold,
803 cooldown,
804 )
805 }
806
807 fn record_inconclusive(&mut self) {
816 self.settled = true;
817 self.breakers
818 .record_inconclusive(self.module_id, self.probe_token.as_ref());
819 }
820}
821
822impl Drop for RouteBindBreakerGuard<'_> {
823 fn drop(&mut self) {
824 if !self.settled {
825 self.breakers
826 .record_inconclusive(self.module_id, self.probe_token.as_ref());
827 }
828 }
829}
830
831struct BreakerOpened {
835 consecutive_timeouts: u32,
836 reopened_after_probe: bool,
839}
840
841impl RouteBindBreakers {
842 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
843 self.modules
844 .lock()
845 .expect("route.bind breaker mutex poisoned")
846 }
847
848 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
851 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
852 probe: probe_token.is_some(),
853 guard: RouteBindBreakerGuard {
854 breakers: self.clone(),
855 module_id,
856 probe_token,
857 settled: false,
858 },
859 };
860
861 let mut modules = self.lock();
862 let Some(state) = modules.get_mut(module_id) else {
863 return admitted(None);
864 };
865 let Some(cooldown_until) = state.cooldown_until else {
866 return admitted(None);
867 };
868 if state.probe_in_flight.is_some() {
869 return RouteBindAdmission::Refused {
870 consecutive_timeouts: state.consecutive_timeouts,
871 retry_in: Duration::ZERO,
872 probe_in_flight: true,
873 };
874 }
875 let now = Instant::now();
876 if now < cooldown_until {
877 return RouteBindAdmission::Refused {
878 consecutive_timeouts: state.consecutive_timeouts,
879 retry_in: cooldown_until - now,
880 probe_in_flight: false,
881 };
882 }
883 let token = Arc::new(());
884 state.probe_in_flight = Some(Arc::clone(&token));
885 admitted(Some(token))
886 }
887
888 fn record_accepted(&self, module_id: &str) -> bool {
889 self.lock()
890 .remove(module_id)
891 .is_some_and(|state| state.cooldown_until.is_some())
892 }
893
894 fn record_timeout(
895 &self,
896 module_id: &str,
897 probe_token: Option<&Arc<()>>,
898 threshold: u32,
899 cooldown: Duration,
900 ) -> Option<BreakerOpened> {
901 let mut modules = self.lock();
902 let state = modules.entry(module_id.to_string()).or_default();
903 let was_open = state.cooldown_until.is_some();
904 let was_probe = Self::owns_probe(state, probe_token);
905 if was_probe {
906 state.probe_in_flight = None;
907 }
908 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
909 if state.consecutive_timeouts < threshold {
910 return None;
911 }
912 state.cooldown_until = Some(Instant::now() + cooldown);
913 Some(BreakerOpened {
914 consecutive_timeouts: state.consecutive_timeouts,
915 reopened_after_probe: was_open && was_probe,
916 })
917 }
918
919 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
920 state
924 .probe_in_flight
925 .as_ref()
926 .zip(token)
927 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
928 }
929
930 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
931 if let Some(state) = self.lock().get_mut(module_id) {
932 if Self::owns_probe(state, probe_token) {
933 state.probe_in_flight = None;
934 }
935 }
936 }
937
938 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
953 self.lock()
954 .remove(module_id)
955 .map(|state| state.consecutive_timeouts)
956 .filter(|discarded| *discarded > 0)
957 }
958
959 fn open_snapshot(&self) -> Option<serde_json::Value> {
966 let now = Instant::now();
967 let modules = self.lock();
968 let open = modules
969 .iter()
970 .filter_map(|(module_id, state)| {
971 let cooldown_until = state.cooldown_until?;
972 Some((
973 module_id.clone(),
974 serde_json::json!({
975 "consecutive_timeouts": state.consecutive_timeouts,
976 "cooldown_remaining_ms":
977 cooldown_until.saturating_duration_since(now).as_millis() as u64,
978 "probe_in_flight": state.probe_in_flight.is_some(),
979 }),
980 ))
981 })
982 .collect::<serde_json::Map<String, serde_json::Value>>();
983 (!open.is_empty()).then_some(serde_json::Value::Object(open))
984 }
985}
986
987impl ControlHandler {
988 pub fn new(registry: Arc<Registry>) -> Self {
989 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
990 }
991
992 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
993 let counters = forwarding.counters();
994 let route_bind_breakers = forwarding.route_bind_breakers();
998 let route_bind_concurrency = forwarding.route_bind_concurrency();
999 let route_outages = forwarding.route_outages();
1000 Self {
1001 registry,
1002 activity: None,
1003 activity_streams: Arc::new(Mutex::new(HashMap::new())),
1004 forwarding,
1005 process_liveness: None,
1006 supervisor: SupervisorHandle::new(),
1007 resource_tracker: Arc::new(crate::resource_totals::Tracker::new(
1008 env!("CARGO_PKG_VERSION").to_owned(),
1009 )),
1010 agent_declarations: agent_commands::Queues::default(),
1011 list_admission: Arc::new(tokio::sync::Semaphore::new(4)),
1012 subc_capabilities: Arc::from([
1013 CAP_MANIFEST_REGISTRATION.to_string(),
1014 CAP_CHANNEL_LIFECYCLE.to_string(),
1015 CAP_PING_PONG.to_string(),
1016 CAP_SESSION_ATTACH.to_string(),
1017 CAP_ADMISSION_FACTS_RELAY.to_string(),
1018 CAP_SCOPES_V1.to_string(),
1019 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
1020 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
1021 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
1022 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
1023 subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
1024 subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
1025 subc_protocol::scope::WORKER_RUNS_CAPABILITY.to_string(),
1026 subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
1027 subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY.to_string(),
1028 subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
1029 subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
1030 ]),
1031 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
1032 route_bind_relay_timeouts: BTreeMap::new(),
1033 route_bind_breakers,
1034 route_bind_concurrency,
1035 route_outages,
1036 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
1037 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
1038 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
1039 storage_config: None,
1040 machine_id: None,
1041 admission_facts_carrier_module_id: None,
1042 admission_facts_targets: None,
1043 scopes: Arc::new(RwLock::new(ScopeTable::new(
1044 crate::daemon_config::default_scope_authority_owners(),
1045 ))),
1046 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
1047 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
1048 privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
1049 caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
1050 build_class_warned: Arc::new(Mutex::new(HashSet::new())),
1051 observed_peers: Arc::new(Mutex::new(HashMap::new())),
1052 local_connections: Arc::new(Mutex::new(HashSet::new())),
1053 operator_app_policy: Default::default(),
1054 operator_apps: Arc::new(Mutex::new(HashMap::new())),
1055 launch_table: std::sync::Arc::new(crate::app_open::LaunchTable::new()),
1056 app_open_launch_lock: std::sync::Arc::new(tokio::sync::Mutex::new(())),
1057 #[cfg(test)]
1058 app_open_test_platform: None,
1059 #[cfg(test)]
1060 app_open_test_launch: None,
1061 #[cfg(windows)]
1062 windows_verify: None,
1063 #[cfg(windows)]
1064 windows_flash: None,
1065 #[cfg(windows)]
1066 windows_track: None,
1067 #[cfg(all(test, windows))]
1068 windows_spawn_attempts: Arc::new(std::sync::atomic::AtomicU64::new(0)),
1069 #[cfg(test)]
1070 caller_lookup: None,
1071 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
1072 rescan: None,
1073 connected_clients: ConnectedClients::new(),
1074 local_endpoint_unavailable: None,
1075 counters,
1076 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
1077 daemon_shutdown: DaemonShutdownFlag::default(),
1078 daemon_provenance: DaemonProvenanceFacts::default(),
1079 #[cfg(test)]
1080 control_dispatch_delay: None,
1081 #[cfg(test)]
1082 provenance_probe_override: None,
1083 #[cfg(test)]
1084 route_bind_relay_sent_hold: None,
1085 #[cfg(test)]
1086 hello_registry_to_forwarding_hold: None,
1087 }
1088 }
1089
1090 pub fn with_storage_config(
1093 mut self,
1094 storage_config: Option<crate::daemon_config::StorageConfig>,
1095 ) -> Self {
1096 self.storage_config = storage_config;
1097 self
1098 }
1099
1100 pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1103 self.activity = Some(crate::activity::ActivityLedger::open(
1104 run_dir.join("activity.jsonl"),
1105 incarnation,
1106 ));
1107 self.supervisor.set_activity(self.activity.clone());
1108 self
1109 }
1110
1111 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1114 self.machine_id = machine_id;
1115 self
1116 }
1117
1118 pub fn with_admission_facts_config(
1122 mut self,
1123 carrier_module_id: Option<String>,
1124 targets: Option<Vec<String>>,
1125 ) -> Self {
1126 self.admission_facts_carrier_module_id = carrier_module_id;
1127 self.admission_facts_targets = targets;
1128 self
1129 }
1130
1131 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1135 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
1136 self.scope_authority_owners = owners;
1137 self
1138 }
1139
1140 pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1142 self.privileged_operator_prompt = mode;
1143 self
1144 }
1145
1146 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1149 self.route_bind_relay_timeout = timeout;
1150 self
1151 }
1152
1153 pub fn with_route_bind_relay_timeouts(
1159 mut self,
1160 timeouts: impl IntoIterator<Item = (String, Duration)>,
1161 ) -> Self {
1162 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1163 self
1164 }
1165
1166 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1172 self.route_bind_relay_timeouts
1173 .get(module_id)
1174 .copied()
1175 .unwrap_or(self.route_bind_relay_timeout)
1176 }
1177
1178 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1186 self.route_bind_breaker_threshold = threshold.max(1);
1187 self.route_bind_breaker_cooldown = cooldown;
1188 self
1189 }
1190
1191 #[cfg(test)]
1192 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1193 self.health_probe_timeout = timeout;
1194 self
1195 }
1196
1197 #[cfg(test)]
1198 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1199 self.control_dispatch_delay = Some(delay);
1200 self
1201 }
1202
1203 #[cfg(test)]
1207 pub(crate) fn with_route_bind_relay_sent_hold(
1208 mut self,
1209 hold: Arc<tokio::sync::Barrier>,
1210 ) -> Self {
1211 self.route_bind_relay_sent_hold = Some(hold);
1212 self
1213 }
1214
1215 #[cfg(test)]
1220 pub(crate) fn with_hello_registry_to_forwarding_hold(
1221 mut self,
1222 hold: Arc<std::sync::Barrier>,
1223 ) -> Self {
1224 self.hello_registry_to_forwarding_hold = Some(hold);
1225 self
1226 }
1227
1228 #[cfg(test)]
1229 fn hold_between_registry_and_forwarding(&self) {
1230 if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1231 hold.wait();
1232 hold.wait();
1233 }
1234 }
1235
1236 pub fn with_process_liveness(
1237 mut self,
1238 process_liveness: Arc<dyn ModuleProcessLiveness>,
1239 ) -> Self {
1240 self.process_liveness = Some(process_liveness);
1241 self
1242 }
1243
1244 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1245 self.supervisor = supervisor;
1246 self.supervisor.set_activity(self.activity.clone());
1247 self
1248 }
1249
1250 pub fn with_daemon_provenance(
1251 mut self,
1252 pid: u32,
1253 started_at_ms: u64,
1254 executable_path: Option<PathBuf>,
1255 build_git_sha: Option<String>,
1256 build_lock_digest: Option<String>,
1257 ) -> Self {
1258 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1259 let process_start_time = process_start_time(pid);
1260 self.daemon_provenance = DaemonProvenanceFacts {
1261 build: DaemonBuildProvenance {
1262 build_git_sha,
1263 build_lock_digest,
1264 },
1265 pid: Some(pid),
1266 started_at_ms: Some(started_at_ms),
1267 start_clock: None,
1268 executable_path,
1269 executable_identity,
1270 process_start_time,
1271 probe: ExecutableIdentityProbe::default(),
1272 };
1273 self
1274 }
1275
1276 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1277 self.daemon_provenance.start_clock = Some(clock);
1278 self
1279 }
1280
1281 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1283 self.scopes
1284 .write()
1285 .unwrap_or_else(|poisoned| poisoned.into_inner())
1286 .set_wall_clock(Arc::new(clock));
1287 self
1288 }
1289
1290 #[cfg(test)]
1291 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1292 self.provenance_probe_override = Some(result);
1293 self
1294 }
1295
1296 pub fn with_capability_config(
1300 self,
1301 modules: impl IntoIterator<Item = (String, bool)>,
1302 reserved_capabilities: BTreeMap<String, String>,
1303 ) -> Self {
1304 self.capability_evaluator
1305 .configure(modules, reserved_capabilities);
1306 self
1307 }
1308
1309 pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1313 self.daemon_shutdown = flag;
1314 self
1315 }
1316
1317 pub fn with_supervisor_rescan(
1318 mut self,
1319 supervisor: Supervisor,
1320 config_path: impl Into<PathBuf>,
1321 configured_port: Option<u16>,
1322 ) -> Self {
1323 self.rescan = Some(SupervisorRescanContext {
1324 supervisor,
1325 config_path: config_path.into(),
1326 configured_port,
1327 storage_config: self.storage_config.clone(),
1328 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1329 admission_facts_targets: self.admission_facts_targets.clone(),
1330 scope_authority_owners: self.scope_authority_owners.clone(),
1331 privileged_operator_prompt: self.privileged_operator_prompt,
1332 generation: Arc::default(),
1333 });
1334 self
1335 }
1336
1337 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1338 self.connected_clients = connected_clients;
1339 self
1340 }
1341
1342 pub fn with_local_endpoint_unavailable(mut self, reason: Option<String>) -> Self {
1345 self.local_endpoint_unavailable = reason;
1346 self
1347 }
1348
1349 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1350 Arc::clone(&self.forwarding)
1351 }
1352
1353 pub(crate) fn counters(&self) -> DaemonCounters {
1354 self.counters.clone()
1355 }
1356
1357 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1360 tokio::spawn(async move {
1361 loop {
1362 self.capability_evaluator
1363 .wait_for_change_or_deadline()
1364 .await;
1365 self.refresh_capability_requirements();
1366 }
1367 });
1368 }
1369
1370 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1376 tokio::spawn(async move {
1377 loop {
1378 let has_deadlines = self
1379 .scopes
1380 .read()
1381 .unwrap_or_else(|poisoned| poisoned.into_inner())
1382 .has_deadlines();
1383 if !has_deadlines {
1384 self.scope_deadline_added.notified().await;
1385 continue;
1386 }
1387 tokio::time::sleep(Duration::from_secs(1)).await;
1388 if let Err(error) = self.sweep_expired_scopes() {
1389 warn!(%error, "scope expiry sweep failed");
1390 }
1391 }
1392 });
1393 }
1394
1395 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1396 let mut table = self
1397 .scopes
1398 .write()
1399 .unwrap_or_else(|poisoned| poisoned.into_inner());
1400 let swept = table.sweep_expired();
1401 let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1402 drop(table);
1403 self.close_scope_drained_routes(drained);
1404 Ok(())
1405 }
1406
1407 fn publish_scope_changes(
1408 &self,
1409 changes: &[crate::scopes::ScopeTagChange],
1410 expired: &[crate::scopes::ScopeExpired],
1411 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1412 if changes.is_empty() {
1413 return Ok(Vec::new());
1414 }
1415 let drained = self
1416 .forwarding
1417 .publish_scope_changes(changes)
1418 .map_err(RouterError::Forwarding)?;
1419 let mut counts = HashMap::new();
1420 for route in &drained {
1421 *counts
1422 .entry((
1423 route.scope.owner.as_str(),
1424 route.scope.scope_ref.as_str(),
1425 route.scope.tag.scope_epoch,
1426 ))
1427 .or_insert(0usize) += 1;
1428 }
1429 for scope in expired {
1430 let routes_closed = counts
1431 .get(&(
1432 scope.owner.as_str(),
1433 scope.scope_ref.as_str(),
1434 scope.scope_epoch,
1435 ))
1436 .copied()
1437 .unwrap_or(0);
1438 warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1439 scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1440 routes_closed, "scope expired");
1441 }
1442 Ok(drained)
1443 }
1444
1445 fn runtime_capability_snapshot(
1446 &self,
1447 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1448 let runtime = self
1449 .supervisor
1450 .list()
1451 .into_iter()
1452 .map(|module| {
1453 let status = module.status().map_err(|err| {
1454 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1455 })?;
1456 Ok(RuntimeModule {
1457 module_id: status.module_id,
1458 state: status.state,
1459 enabled: status.enabled,
1460 })
1461 })
1462 .collect::<Result<Vec<_>, RouterError>>()?;
1463 let registrations = self.registry.list_registered_modules().map_err(|err| {
1469 RouterError::backend(
1470 0,
1471 0,
1472 format!("failed to list capability registrations: {err}"),
1473 )
1474 })?;
1475 let registrations = registrations
1476 .into_iter()
1477 .map(|registration| RegisteredModule {
1478 module_id: registration.manifest.module_id,
1479 module_version: registration.manifest.module_version,
1480 capabilities: registration.manifest.capabilities,
1481 })
1482 .collect();
1483 Ok((runtime, registrations))
1484 }
1485
1486 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1493 let cached_registration = RegisteredModule {
1494 module_id: registration.manifest.module_id.clone(),
1495 module_version: registration.manifest.module_version.clone(),
1496 capabilities: registration.manifest.capabilities.clone(),
1497 };
1498 let change = self.capability_evaluator.record_hello(&cached_registration);
1499 log_manifest_change(&cached_registration.module_id, &change);
1500 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1501 self.enforce_capability_denies();
1502 }
1503 self.refresh_capability_requirements();
1504 }
1505
1506 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1510 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1511 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1512 self.supervisor.set_swap_promotion_observer(observer);
1513 }
1514
1515 pub fn refresh_capability_requirements(&self) {
1516 match self.runtime_capability_snapshot() {
1517 Ok((runtime, registrations)) => {
1518 log_requirement_events(
1519 self.capability_evaluator
1520 .evaluate_now(&runtime, ®istrations),
1521 &self.daemon_shutdown,
1522 );
1523 }
1524 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1525 }
1526 }
1527
1528 fn enforce_capability_denies(&self) {
1532 let registrations = match self.registry.list_registered_modules() {
1533 Ok(snapshot) => snapshot,
1534 Err(err) => {
1535 warn!(error = %err, "failed to read registrations for capability deny census");
1536 return;
1537 }
1538 };
1539 let manifests = registrations
1540 .into_iter()
1541 .map(|registration| {
1542 (
1543 registration.manifest.module_id.clone(),
1544 registration.manifest,
1545 )
1546 })
1547 .collect::<BTreeMap<_, _>>();
1548 let census = match self.forwarding.route_census(None) {
1549 Ok(census) => census,
1550 Err(err) => {
1551 warn!(error = %err, "failed to read route census for capability deny enforcement");
1552 return;
1553 }
1554 };
1555
1556 for (target_module_id, routes) in census {
1557 let Some(target_manifest) = manifests.get(&target_module_id) else {
1558 continue;
1559 };
1560 let mut closed_routes = Vec::new();
1561 let mut goodbyes = Vec::new();
1562 for route in routes {
1563 let Principal::Reserved {
1564 module_id: opening_module_id,
1565 } = &route.principal
1566 else {
1567 continue;
1568 };
1569 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1570 continue;
1571 };
1572 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1573 continue;
1574 };
1575
1576 match self.forwarding.release_client_route(
1577 route.goodbye_target.connection_id,
1578 route.goodbye_target.channel,
1579 route.goodbye_target.epoch,
1580 ) {
1581 Ok(RouteRelease::Removed(module_goodbye)) => {
1582 warn!(
1583 opening_module_id,
1584 target_module_id,
1585 capability,
1586 "force-closing route because an attested capability deny edge now matches"
1587 );
1588 goodbyes.push(module_goodbye);
1589 goodbyes.push(route.goodbye_target.clone());
1590 closed_routes.push(route);
1591 }
1592 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1593 Err(err) => warn!(
1594 opening_module_id,
1595 target_module_id,
1596 capability,
1597 error = %err,
1598 "failed to force-close capability-denied route"
1599 ),
1600 }
1601 }
1602
1603 if closed_routes.is_empty() {
1604 continue;
1605 }
1606 send_route_control_pushes(
1607 &self.forwarding,
1608 closed_routes,
1609 ClientControlPush::RouteClosed {
1610 module_id: target_module_id,
1611 channels: Vec::new(),
1612 reason: RouteCloseReason::CapabilityDenied,
1613 drained: false,
1614 abandoned: 0,
1615 excluded_subscriptions: 0,
1616 terminal: Some(false),
1617 },
1618 );
1619 self.emit_route_goodbyes(goodbyes);
1622 }
1623 }
1624
1625 fn not_ready_reason(
1631 &self,
1632 registration: &crate::registry::ModuleRegistration,
1633 ) -> Option<NotReadyReason> {
1634 if !registration.ready {
1635 return Some(NotReadyReason {
1636 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1637 capability: None,
1638 });
1639 }
1640 self.first_unprovided_required_capability(registration)
1641 .map(|capability| NotReadyReason {
1642 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1643 capability: Some(capability),
1644 })
1645 }
1646
1647 fn first_unprovided_required_capability(
1665 &self,
1666 registration: &crate::registry::ModuleRegistration,
1667 ) -> Option<String> {
1668 let required = registration
1669 .manifest
1670 .capabilities
1671 .iter()
1672 .flat_map(|declarations| declarations.requires.iter())
1673 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1674 .map(|requirement| requirement.capability.as_str())
1675 .collect::<BTreeSet<_>>();
1676 if required.is_empty() {
1677 return None;
1678 }
1679 let module_id = registration.manifest.module_id.as_str();
1680 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1681 if required
1682 .iter()
1683 .any(|capability| verdict(capability).is_none())
1684 {
1685 self.refresh_capability_requirements();
1686 }
1687 required
1688 .into_iter()
1689 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1690 .map(str::to_string)
1691 }
1692
1693 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1694 self.capability_evaluator
1695 .statuses()
1696 .into_iter()
1697 .map(capability_requirement_status)
1698 .collect()
1699 }
1700
1701 fn deregister_connection(
1708 &self,
1709 connection_id: ConnectionId,
1710 reason: RegistrationEndReason,
1711 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1712 self.registry
1713 .deregister_connection_with_reason(connection_id, reason)
1714 }
1715
1716 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1717 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1718 return None;
1719 }
1720 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1721 parse_client_control_request(&frame.body)
1722 else {
1723 return None;
1724 };
1725 Some(target_module_id(&target).to_string())
1726 }
1727
1728 pub(crate) fn route_open_capacity_refusal(
1729 &self,
1730 ctx: &RouteCtx,
1731 frame: &Frame,
1732 target_module_id: &str,
1733 in_flight: usize,
1734 limit: usize,
1735 ) -> Result<Frame, RouterError> {
1736 let caller = self.route_open_caller_from_frame(frame);
1739 self.route_open_admission_refusal_frame(
1740 ctx,
1741 &caller,
1742 frame,
1743 target_module_id,
1744 "open_admission_full",
1745 (in_flight, limit),
1746 format!(
1747 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1748 ),
1749 )
1750 }
1751
1752 fn route_open_target_capacity_refusal(
1753 &self,
1754 ctx: &RouteCtx,
1755 caller: &RouteOpenCaller,
1756 frame: &Frame,
1757 target_module_id: &str,
1758 in_flight: usize,
1759 ) -> Result<Frame, RouterError> {
1760 self.route_open_admission_refusal_frame(
1761 ctx,
1762 caller,
1763 frame,
1764 target_module_id,
1765 "target_binds_full",
1766 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1767 format!(
1768 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1769 ),
1770 )
1771 }
1772
1773 #[allow(clippy::too_many_arguments)]
1781 fn route_open_admission_refusal_frame(
1782 &self,
1783 ctx: &RouteCtx,
1784 caller: &RouteOpenCaller,
1785 frame: &Frame,
1786 target_module_id: &str,
1787 reason: &'static str,
1788 (in_flight, limit): (usize, usize),
1789 message: impl Into<String>,
1790 ) -> Result<Frame, RouterError> {
1791 let code = error_codes::TARGET_UNAVAILABLE;
1792 self.counters.increment_route_open_refused(code);
1793 info!(
1794 target: "control",
1795 code,
1796 reason,
1797 module_id = ?target_module_id,
1798 connection_id = ctx.connection_id.get(),
1799 principal = %caller.principal,
1800 scope = %caller.scope,
1801 harness = %caller.harness,
1802 in_flight,
1803 limit,
1804 "route.open refused"
1805 );
1806 control_error_frame(frame, code, message.into())
1807 }
1808
1809 #[cfg(test)]
1815 pub fn handle_control(
1816 &self,
1817 connection_id: ConnectionId,
1818 frame: Frame,
1819 ) -> Result<Vec<Frame>, RouterError> {
1820 match frame.header.ty {
1821 FrameType::Ping => Ok(vec![pong(&frame)?]),
1822 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1823 FrameType::Goodbye => self.handle_goodbye(connection_id),
1824 ty => Ok(vec![control_error_frame(
1825 &frame,
1826 "unsupported_control_frame",
1827 format!("unsupported channel-0 frame {ty:?}"),
1828 )?]),
1829 }
1830 }
1831
1832 pub async fn handle_control_frame(
1833 &self,
1834 ctx: &RouteCtx,
1835 frame: Frame,
1836 ) -> Result<Vec<Frame>, RouterError> {
1837 self.handle_control_frame_timed(ctx, frame, None).await
1838 }
1839
1840 #[cfg(any(test, feature = "test-support"))]
1841 pub fn with_test_operator_app_policy(
1842 mut self,
1843 policy: crate::operator_apps::TestOperatorAppPolicy,
1844 ) -> Self {
1845 self.operator_app_policy = policy.into();
1846 self
1847 }
1848
1849 #[cfg(all(windows, any(test, feature = "test-support")))]
1853 pub fn with_test_windows_verifier(mut self, verify: crate::app_open::WindowsVerifyFn) -> Self {
1854 self.windows_verify = Some(verify);
1855 self
1856 }
1857
1858 #[cfg(all(windows, any(test, feature = "test-support")))]
1860 pub fn with_test_windows_flash(mut self, flash: crate::app_open::WindowsFlashFn) -> Self {
1861 self.windows_flash = Some(flash);
1862 self
1863 }
1864
1865 #[cfg(all(windows, any(test, feature = "test-support")))]
1869 pub fn with_test_windows_track(mut self, track: crate::app_open::WindowsTrackFn) -> Self {
1870 self.windows_track = Some(track);
1871 self
1872 }
1873
1874 pub(crate) fn record_captured_peer(
1875 &self,
1876 connection: ConnectionId,
1877 peer: subc_os::local_peer::CapturedPeer,
1878 ) {
1879 self.record_observed_peer(connection, peer.observation().clone());
1880 #[cfg(windows)]
1881 if let Some(stamp) = self.stamp_windows_launch(&peer) {
1882 self.operator_apps
1883 .lock()
1884 .unwrap_or_else(|p| p.into_inner())
1885 .insert(connection, stamp);
1886 }
1887 if let Some(stamp) = self.operator_app_policy.stamp(peer) {
1888 self.operator_apps
1889 .lock()
1890 .unwrap_or_else(|p| p.into_inner())
1891 .insert(connection, stamp);
1892 }
1893 }
1894
1895 pub(crate) fn record_local_connection(&self, connection: ConnectionId) {
1896 self.local_connections
1897 .lock()
1898 .unwrap_or_else(|p| p.into_inner())
1899 .insert(connection);
1900 }
1901
1902 fn connection_operator_app(
1905 &self,
1906 stamp: Option<&crate::operator_apps::OperatorAppStamp>,
1907 ) -> Option<subc_protocol::peer::OperatorApp> {
1908 stamp
1909 .filter(|stamp| stamp.delivery_enabled(&self.operator_app_policy))
1910 .map(|stamp| stamp.app.clone())
1911 }
1912
1913 fn handle_connection_describe(
1914 &self,
1915 ctx: &RouteCtx,
1916 frame: Frame,
1917 ) -> Result<Vec<Frame>, RouterError> {
1918 if serde_json::from_slice::<subc_control::ConnectionDescribeRequest>(&frame.body).is_err() {
1919 return Ok(vec![control_error_frame(
1920 &frame,
1921 "invalid_control_body",
1922 "malformed connection.describe body",
1923 )?]);
1924 }
1925 let local = self
1926 .local_connections
1927 .lock()
1928 .unwrap_or_else(|p| p.into_inner())
1929 .contains(&ctx.connection_id);
1930 let stamps = self.operator_apps.lock().unwrap_or_else(|p| p.into_inner());
1931 let operator_app = self.connection_operator_app(stamps.get(&ctx.connection_id));
1932 Ok(vec![control_response_body_frame(
1933 &frame,
1934 &subc_control::ConnectionDescribeReply::new(local, operator_app),
1935 ops::CONNECTION_DESCRIBE,
1936 )?])
1937 }
1938
1939 pub(crate) fn record_observed_peer(
1940 &self,
1941 connection: ConnectionId,
1942 peer: subc_os::local_peer::ObservedPeer,
1943 ) {
1944 use subc_os::local_peer::{ObservedPeer as OsPeer, PeerUser};
1945 let peer = match peer {
1946 OsPeer::Unobserved => ObservedPeer::unobserved(),
1947 OsPeer::Observed(peer) => {
1948 let observed = match peer.user {
1949 PeerUser::Uid(uid) => ObservedPeer::observed_uid(uid),
1950 PeerUser::Sid(sid) => ObservedPeer::observed_sid(sid),
1951 };
1952 observed
1953 .with_exe(
1954 peer.exe
1955 .and_then(|path| path.into_os_string().into_string().ok()),
1956 )
1957 .with_signing_id(peer.signing_id)
1958 }
1959 };
1960 self.observed_peers
1961 .lock()
1962 .unwrap_or_else(|p| p.into_inner())
1963 .insert(connection, peer);
1964 }
1965
1966 pub(crate) fn record_caller_endpoints(
1967 &self,
1968 connection: ConnectionId,
1969 endpoints: Option<(SocketAddr, SocketAddr)>,
1970 ) {
1971 let mut callers = self
1972 .caller_endpoints
1973 .lock()
1974 .unwrap_or_else(|p| p.into_inner());
1975 callers.remove(&connection);
1976 if let Some(endpoints) = endpoints {
1977 callers.insert(connection, endpoints);
1978 }
1979 }
1980
1981 #[cfg(test)]
1982 pub(crate) fn with_caller_lookup_for_test(
1983 mut self,
1984 lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1985 ) -> Self {
1986 self.caller_lookup = Some(Arc::new(lookup));
1987 self
1988 }
1989
1990 async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
1991 let endpoints = self
1992 .caller_endpoints
1993 .lock()
1994 .unwrap_or_else(|p| p.into_inner())
1995 .get(&connection)
1996 .copied();
1997 #[cfg(test)]
1998 let lookup = self.caller_lookup.clone();
1999 tokio::task::spawn_blocking(move || {
2000 #[cfg(test)]
2001 if let Some(lookup) = lookup {
2002 return lookup(endpoints);
2003 }
2004 endpoints
2005 .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
2006 .unwrap_or_default()
2007 })
2008 .await
2009 .unwrap_or_default()
2010 }
2011
2012 pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
2013 frame.header.channel == 0
2014 && frame.header.ty == FrameType::Request
2015 && self
2016 .forwarding
2017 .module_endpoint_for_connection(ctx.connection_id)
2018 .is_ok_and(|endpoint| endpoint.is_none())
2019 && parse_client_control_request(&frame.body)
2020 .ok()
2021 .is_some_and(|request| privileged_operation(&request).is_some())
2022 }
2023
2024 pub(crate) async fn handle_control_frame_timed(
2025 &self,
2026 ctx: &RouteCtx,
2027 frame: Frame,
2028 dispatch_started_at: Option<StdInstant>,
2029 ) -> Result<Vec<Frame>, RouterError> {
2030 match frame.header.ty {
2031 FrameType::Ping => Ok(vec![pong(&frame)?]),
2032 FrameType::Hello => {
2033 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
2034 }
2035 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
2036 FrameType::Cancel => {
2037 if self
2040 .agent_declarations
2041 .cancel(ctx.connection_id, frame.header.corr)
2042 || self
2043 .forwarding
2044 .operator_confirms()
2045 .cancel(ctx.connection_id, frame.header.corr)
2046 || self
2047 .supervisor
2048 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
2049 || self.cancel_activity_subscription(ctx, &frame)?
2050 {
2051 Ok(Vec::new())
2052 } else {
2053 Ok(vec![control_error_frame(
2054 &frame,
2055 "unknown_subscription",
2056 "no subscription has this correlation id",
2057 )?])
2058 }
2059 }
2060 FrameType::Request => {
2061 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
2064 if op
2065 .as_ref()
2066 .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
2067 {
2068 return self.handle_activity_record(ctx, frame).await;
2069 }
2070 if op.as_ref().is_some_and(|probe| {
2071 probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
2072 }) {
2073 return self.handle_activity_read(ctx, frame).await;
2074 }
2075 if op
2076 .as_ref()
2077 .is_some_and(|probe| probe.op == "operator.confirm")
2078 {
2079 return self.handle_operator_confirm(ctx, frame);
2080 }
2081 if op.as_ref().is_some_and(|probe| probe.op == ops::APP_OPEN) {
2082 return self
2083 .observe_control_wait(
2084 ctx,
2085 None,
2086 ops::APP_OPEN,
2087 self.handle_app_open(ctx, frame),
2088 )
2089 .await;
2090 }
2091 if op
2092 .as_ref()
2093 .is_some_and(|probe| probe.op == ops::CONNECTION_DESCRIBE)
2094 {
2095 return self.handle_connection_describe(ctx, frame);
2096 }
2097 if op
2098 .as_ref()
2099 .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
2100 {
2101 return self.handle_scope_served_by(ctx.connection_id, frame);
2102 }
2103 if self
2104 .forwarding
2105 .module_endpoint_for_connection(ctx.connection_id)
2106 .map_err(RouterError::Forwarding)?
2107 .is_some()
2108 {
2109 if !is_known_module_request_op(&frame.body) {
2110 return Ok(vec![control_error_frame(
2111 &frame,
2112 "unsupported_control_frame",
2113 "module-originated channel-0 REQUEST is not supported",
2114 )?]);
2115 }
2116 let request = match parse_module_control_request_from_module(&frame.body) {
2117 Ok(request) => request,
2118 Err((err, ControlRequestBodyError::UnknownOp)) => {
2119 return Ok(vec![control_error_frame(
2120 &frame,
2121 "unsupported_control_frame",
2122 format!("unsupported module-originated channel-0 REQUEST: {err}"),
2123 )?])
2124 }
2125 Err((err, ControlRequestBodyError::InvalidBody)) => {
2126 return Ok(vec![control_error_frame(
2127 &frame,
2128 "invalid_control_body",
2129 format!("malformed module control body: {err}"),
2130 )?])
2131 }
2132 };
2133 let op = module_control_request_op(&request);
2134 let corr = frame.header.corr;
2135 log_control_dispatch_arrival(op, ctx.connection_id, corr);
2136 if let ModuleControlRequestFromModule::AgentCommands {
2137 started,
2138 finished,
2139 known,
2140 } = request
2141 {
2142 return self
2143 .agent_declarations
2144 .enqueue(
2145 self,
2146 ctx,
2147 frame,
2148 crate::resource_totals::Declaration {
2149 started,
2150 finished,
2151 known,
2152 },
2153 )
2154 .await;
2155 }
2156 let result =
2157 self.handle_module_control_request(ctx.connection_id, frame, request);
2158 log_slow_control_dispatch(
2159 dispatch_started_at,
2160 op,
2161 ctx.connection_id,
2162 corr,
2163 None,
2164 );
2165 return result;
2166 }
2167
2168 if is_known_module_request_op(&frame.body) {
2169 let is_agents = serde_json::from_slice::<ControlOpProbe>(&frame.body)
2170 .is_ok_and(|probe| probe.op == "supervisor.agent_commands");
2171 return Ok(vec![control_error_frame(
2172 &frame,
2173 if is_agents { "not_supervised" } else { "not_registered" },
2174 "module-originated control requests require the module's own registered connection",
2175 )?]);
2176 }
2177
2178 let request = match parse_client_control_request(&frame.body) {
2179 Ok(request) => request,
2180 Err((err, ControlRequestBodyError::UnknownOp)) => {
2181 return Ok(vec![control_error_frame(
2182 &frame,
2183 "unknown_control_op",
2184 format!("unknown client control op: {err}"),
2185 )?])
2186 }
2187 Err((err, ControlRequestBodyError::InvalidBody)) => {
2188 return Ok(vec![control_error_frame(
2189 &frame,
2190 "invalid_control_body",
2191 format!("malformed client control body: {err}"),
2192 )?])
2193 }
2194 };
2195 let op = client_control_request_op(&request);
2196 let corr = frame.header.corr;
2197 log_control_dispatch_arrival(op, ctx.connection_id, corr);
2198 let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
2199 #[cfg(test)]
2200 if let Some(delay) = self.control_dispatch_delay {
2201 tokio::time::sleep(delay).await;
2202 }
2203 let result = self
2204 .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
2205 .await;
2206 if let Some(timings) = route_open_timings.as_mut() {
2207 timings.finish();
2208 }
2209 log_slow_control_dispatch(
2210 dispatch_started_at,
2211 op,
2212 ctx.connection_id,
2213 corr,
2214 route_open_timings.as_ref(),
2215 );
2216 result
2217 }
2218 FrameType::Push => {
2219 let Some(endpoint) = self
2220 .forwarding
2221 .module_endpoint_for_connection(ctx.connection_id)
2222 .map_err(RouterError::Forwarding)?
2223 else {
2224 return Ok(vec![control_error_frame(
2225 &frame,
2226 "unsupported_control_frame",
2227 "client-originated channel-0 PUSH is not supported",
2228 )?]);
2229 };
2230 self.handle_status_update(endpoint, frame)
2231 }
2232 FrameType::Response | FrameType::Error
2233 if self
2234 .forwarding
2235 .module_endpoint_for_connection(ctx.connection_id)
2236 .map_err(RouterError::Forwarding)?
2237 .is_some() =>
2238 {
2239 self.handle_module_relay_response(ctx.connection_id, frame)
2240 }
2241 ty => Ok(vec![control_error_frame(
2242 &frame,
2243 "unsupported_control_frame",
2244 format!("unsupported channel-0 frame {ty:?}"),
2245 )?]),
2246 }
2247 }
2248
2249 pub fn cleanup_connection(
2250 &self,
2251 connection_id: ConnectionId,
2252 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2253 self.local_connections
2254 .lock()
2255 .unwrap_or_else(|p| p.into_inner())
2256 .remove(&connection_id);
2257 self.caller_endpoints
2258 .lock()
2259 .unwrap_or_else(|p| p.into_inner())
2260 .remove(&connection_id);
2261 self.build_class_warned
2262 .lock()
2263 .unwrap_or_else(|p| p.into_inner())
2264 .remove(&connection_id);
2265 self.observed_peers
2266 .lock()
2267 .unwrap_or_else(|p| p.into_inner())
2268 .remove(&connection_id);
2269 self.operator_apps
2270 .lock()
2271 .unwrap_or_else(|p| p.into_inner())
2272 .remove(&connection_id);
2273 self.forwarding
2274 .operator_confirms()
2275 .connection_closed(connection_id);
2276 self.cleanup_connection_with_end_reason(
2277 connection_id,
2278 RegistrationEndReason::ConnectionClosed,
2279 )
2280 }
2281
2282 fn cleanup_connection_with_end_reason(
2283 &self,
2284 connection_id: ConnectionId,
2285 requested_reason: RegistrationEndReason,
2286 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2287 self.agent_declarations.close(connection_id);
2288 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2289 self.registry
2290 .get_module_by_connection(connection_id)?
2291 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
2292 .and_then(|module| module.registration_end_reason().ok().flatten())
2293 .unwrap_or(requested_reason)
2294 } else {
2295 requested_reason
2296 };
2297 let crash_closed = self
2298 .registry
2299 .get_module_by_connection(connection_id)?
2300 .and_then(|registration| {
2301 self.forwarding
2302 .module_endpoint_for_connection(connection_id)
2303 .ok()
2304 .flatten()
2305 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2306 .map(|routes| (registration.manifest.module_id, routes))
2307 });
2308 let crash_closed = crash_closed.map(|(module_id, routes)| {
2309 let terminal = match self.supervisor.get(&module_id) {
2310 None => false,
2311 Some(module) => match module.will_recover_after_connection_loss() {
2312 Ok(will_recover) => !will_recover,
2313 Err(err) => {
2314 warn!(
2315 %module_id,
2316 error = %err,
2317 "failed to read crash recovery verdict; reporting non-terminal conservatively"
2318 );
2319 false
2320 }
2321 },
2322 };
2323 let (reason, drained) = match self
2327 .forwarding
2328 .daemon_drain_result_for_connection(connection_id)
2329 {
2330 Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2331 Ok(None) => (RouteCloseReason::Crash, false),
2332 Err(err) => {
2333 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2334 (RouteCloseReason::Crash, false)
2335 }
2336 };
2337 (module_id, routes, reason, drained, terminal)
2338 });
2339 let registrations = self.deregister_connection(connection_id, end_reason);
2340 let cleanup = if crash_closed.is_some() {
2341 self.forwarding.cleanup_connection_counted(connection_id)
2342 } else {
2343 self.forwarding
2349 .cleanup_connection(connection_id)
2350 .map(|released| crate::forwarding::ConnectionCleanup {
2351 released,
2352 abandoned_relays: 0,
2353 })
2354 };
2355 if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2360 let abandoned = cleanup
2361 .as_ref()
2362 .map_or(0, |cleanup| cleanup.abandoned_relays);
2363 send_route_control_pushes(
2364 &self.forwarding,
2365 routes,
2366 ClientControlPush::RouteClosed {
2367 module_id,
2368 channels: Vec::new(),
2369 reason,
2370 drained,
2371 abandoned,
2372 excluded_subscriptions: 0,
2373 terminal: Some(terminal),
2374 },
2375 );
2376 }
2377 if let Ok(cleanup) = cleanup {
2378 self.emit_route_goodbyes(cleanup.released);
2379 }
2380 if matches!(®istrations, Ok(r) if !r.is_empty()) {
2384 crate::supervise::notify_registration_release();
2385 self.capability_evaluator.wake_deadline_loop();
2386 self.refresh_capability_requirements();
2387 }
2388 self.supervisor.remove_spawn_subscribers(connection_id);
2389 self.activity_streams
2390 .lock()
2391 .unwrap_or_else(|p| p.into_inner())
2392 .retain(|(id, _), task| {
2393 if *id == connection_id {
2394 task.abort();
2395 false
2396 } else {
2397 true
2398 }
2399 });
2400 self.hello_launch_nonces
2403 .lock()
2404 .unwrap_or_else(|poisoned| poisoned.into_inner())
2405 .forget(connection_id);
2406 self.scopes
2407 .write()
2408 .unwrap_or_else(|poisoned| poisoned.into_inner())
2409 .release_connection(connection_id);
2410 registrations
2411 }
2412
2413 pub(crate) fn handle_route_goodbye(
2414 &self,
2415 connection_id: ConnectionId,
2416 route_channel: u16,
2417 route_epoch: u32,
2418 ) -> Result<bool, RouterError> {
2419 debug!(
2420 connection_id = connection_id.get(),
2421 route_channel, route_epoch, "handling route GOODBYE"
2422 );
2423 let RouteRelease::Removed(released_route) = self
2424 .forwarding
2425 .release_client_route(connection_id, route_channel, route_epoch)
2426 .map_err(RouterError::Forwarding)?
2427 else {
2428 return Ok(false);
2429 };
2430 self.emit_route_goodbyes(vec![released_route]);
2431 Ok(true)
2432 }
2433
2434 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2435 for released in released_routes {
2436 let frame = match Frame::build_with_version(
2437 released.negotiated_ver,
2438 FrameType::Goodbye,
2439 control_flags(),
2440 released.channel,
2441 released.epoch,
2442 0,
2443 Vec::new(),
2444 ) {
2445 Ok(frame) => frame,
2446 Err(err) => {
2447 warn!(
2448 route_channel = released.channel,
2449 error = %err,
2450 "failed to build route GOODBYE frame"
2451 );
2452 continue;
2453 }
2454 };
2455 if !released.close_on_delivery_failure() {
2456 crate::forwarding::send_module_route_goodbye(
2457 &self.counters,
2458 &released.sink,
2459 frame,
2460 released.module_id.as_deref(),
2461 "client route released",
2462 );
2463 continue;
2464 }
2465 if let Err(err) = released.sink.try_send(frame) {
2466 warn!(
2467 target_connection_id = released.connection_id.get(),
2468 route_channel = released.channel,
2469 error = %err,
2470 "route GOODBYE was not delivered to client; closing target connection"
2471 );
2472 if self
2473 .forwarding
2474 .escalate_client_delivery_failure(
2475 released.connection_id,
2476 released.channel,
2477 released.epoch,
2478 CloseReason::new(
2479 "route_goodbye_delivery_failed",
2480 format!(
2481 "failed to enqueue route GOODBYE for channel {}: {err}",
2482 released.channel
2483 ),
2484 ),
2485 crate::forwarding::UndeliveredFrame {
2486 module_id: released.module_id.as_deref(),
2487 sink: &released.sink,
2488 },
2489 )
2490 .unwrap_or(false)
2491 {
2492 self.counters.increment_goodbye_relay_client_failed();
2493 }
2494 }
2495 }
2496 }
2497
2498 fn send_abandoned_route_bind_goodbye(
2510 &self,
2511 module_sink: &crate::FrameSink,
2512 negotiated_ver: u8,
2513 module_channel: u16,
2514 module_epoch: u32,
2515 ) {
2516 let frame = match Frame::build_with_version(
2517 negotiated_ver,
2518 FrameType::Goodbye,
2519 control_flags(),
2520 module_channel,
2521 module_epoch,
2522 0,
2523 Vec::new(),
2524 ) {
2525 Ok(frame) => frame,
2526 Err(err) => {
2527 warn!(
2528 route_channel = module_channel,
2529 error = %err,
2530 "failed to build GOODBYE for abandoned route.bind"
2531 );
2532 return;
2533 }
2534 };
2535 crate::forwarding::send_module_route_goodbye(
2536 &self.counters,
2537 module_sink,
2538 frame,
2539 None,
2540 "abandoned route.bind",
2541 );
2542 }
2543
2544 fn handle_hello(
2545 &self,
2546 connection_id: ConnectionId,
2547 sink: Option<crate::FrameSink>,
2548 frame: Frame,
2549 ) -> Result<Vec<Frame>, RouterError> {
2550 debug!(
2551 connection_id = connection_id.get(),
2552 corr = frame.header.corr,
2553 "handling HELLO"
2554 );
2555 if self
2559 .registry
2560 .get_module_by_connection(connection_id)
2561 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2562 .is_some()
2563 {
2564 return Ok(vec![control_error_frame(
2565 &frame,
2566 "invalid_hello",
2567 "connection is already registered as a module",
2568 )?]);
2569 }
2570 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2571 Ok(value) => value,
2572 Err(err) => {
2573 return Ok(vec![control_error_frame(
2574 &frame,
2575 "invalid_hello",
2576 format!("malformed HELLO body: {err}"),
2577 )?])
2578 }
2579 };
2580 if let Err(err) = validate_hello_display_name(&hello_value) {
2581 return Ok(vec![control_error_frame(
2582 &frame,
2583 "invalid_manifest",
2584 err.to_string(),
2585 )?]);
2586 }
2587 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2588 return Ok(vec![control_error_frame(
2589 &frame,
2590 "invalid_capability_grammar",
2591 err.to_string(),
2592 )?]);
2593 }
2594 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2595 return Ok(vec![control_error_frame(
2596 &frame,
2597 "invalid_manifest",
2598 err.to_string(),
2599 )?]);
2600 }
2601 if let Err(err) = validate_hello_event_declarations(&hello_value) {
2602 return Ok(vec![control_error_frame(
2603 &frame,
2604 "invalid_event_declaration",
2605 err.to_string(),
2606 )?]);
2607 }
2608 if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
2609 return Ok(vec![control_error_frame(
2610 &frame,
2611 "invalid_manifest",
2612 err.to_string(),
2613 )?]);
2614 }
2615 if let Some(provenance) = hello_value
2616 .get("manifest")
2617 .and_then(|manifest| manifest.get("provenance"))
2618 {
2619 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2620 return Ok(vec![control_error_frame(
2621 &frame,
2622 "invalid_manifest",
2623 format!("malformed manifest provenance: {err}"),
2624 )?]);
2625 }
2626 }
2627 if let Err(reason) = validate_hello_role_declarations(&hello_value) {
2628 let module_id = hello_value["manifest"]["module_id"]
2629 .as_str()
2630 .unwrap_or("<unknown>");
2631 warn!(module_id, %reason, "dropping whole role_declarations map from HELLO");
2632 }
2633 if let Err(reason) = validate_hello_flow_operations(&hello_value) {
2634 let module_id = hello_value["manifest"]["module_id"]
2635 .as_str()
2636 .unwrap_or("<unknown>");
2637 warn!(module_id, %reason, "dropping flow_operations from HELLO");
2638 }
2639 if let Err(reason) = validate_hello_confirm_operations(&hello_value) {
2640 let module_id = hello_value["manifest"]["module_id"]
2641 .as_str()
2642 .unwrap_or("<unknown>");
2643 warn!(module_id, %reason, "dropping confirm_operations from HELLO");
2644 }
2645 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2646 Ok(hello) => hello,
2647 Err(err) => {
2648 return Ok(vec![control_error_frame(
2649 &frame,
2650 "invalid_hello",
2651 format!("malformed HELLO body: {err}"),
2652 )?])
2653 }
2654 };
2655
2656 if hello.protocol_ver != hello.manifest.protocol_ver {
2657 return Ok(vec![control_error_frame(
2658 &frame,
2659 "invalid_manifest",
2660 format!(
2661 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2662 hello.protocol_ver, hello.manifest.protocol_ver
2663 ),
2664 )?]);
2665 }
2666
2667 if hello.manifest.module_id.trim().is_empty() {
2668 return Ok(vec![control_error_frame(
2669 &frame,
2670 "invalid_manifest",
2671 "manifest module_id must not be empty",
2672 )?]);
2673 }
2674
2675 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2676 Ok(negotiated_ver) => negotiated_ver,
2677 Err(message) => {
2678 return Ok(vec![control_error_frame(
2679 &frame,
2680 "version_unsupported",
2681 message,
2682 )?])
2683 }
2684 };
2685
2686 let swap_admission = self
2694 .supervisor
2695 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2696 if swap_admission == SwapHelloAdmission::Refused {
2697 warn!(
2698 module_id = %hello.manifest.module_id,
2699 connection_id = connection_id.get(),
2700 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2701 );
2702 return Ok(vec![control_error_frame(
2703 &frame,
2704 "swap_token_invalid",
2705 format!(
2706 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2707 hello.manifest.module_id
2708 ),
2709 )?]);
2710 }
2711 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2712
2713 let nonce_verified = if swap_candidate {
2718 true
2719 } else {
2720 match self.supervisor.reserved_hello_verification(
2721 &hello.manifest.module_id,
2722 hello.launch_nonce.as_deref(),
2723 ) {
2724 Ok(verified) => verified,
2725 Err(rejection) => {
2726 let message = match rejection {
2727 ReservedHelloRejection::Exact { module_id } => format!(
2728 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2729 ),
2730 ReservedHelloRejection::Prefix {
2731 prefix,
2732 owner_module_id,
2733 } => format!(
2734 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2735 hello.manifest.module_id
2736 ),
2737 };
2738 return Ok(vec![control_error_frame(
2739 &frame,
2740 "reserved_module",
2741 message,
2742 )?]);
2743 }
2744 }
2745 };
2746
2747 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2748 &hello.manifest.module_id,
2749 hello.manifest.capabilities.as_ref(),
2750 );
2751 if let Some(refusal) = reserved_capability_refusals.first() {
2752 let capability = refusal.capability.clone();
2753 let bound_module = refusal.claimants[0].clone();
2754 log_duplicate_claim_events(reserved_capability_refusals);
2755 return Ok(vec![control_error_frame(
2756 &frame,
2757 "reserved_capability",
2758 format!(
2759 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2760 capability, bound_module, hello.manifest.module_id
2761 ),
2762 )?]);
2763 }
2764
2765 if self
2768 .forwarding
2769 .connection_has_client_routes(connection_id)
2770 .map_err(RouterError::Forwarding)?
2771 {
2772 return Ok(vec![control_error_frame(
2773 &frame,
2774 "invalid_hello",
2775 "connection has open client routes and cannot also register as a module",
2776 )?]);
2777 }
2778
2779 self.hello_launch_nonces
2784 .lock()
2785 .unwrap_or_else(|poisoned| poisoned.into_inner())
2786 .record(connection_id, hello.launch_nonce.as_deref());
2787 let control_ops = effective_module_control_ops(hello.control_ops);
2788 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2791 if swap_candidate {
2792 return self.register_swap_candidate(
2793 connection_id,
2794 sink,
2795 &frame,
2796 hello.manifest,
2797 negotiated_ver,
2798 control_ops,
2799 hello_ack,
2800 );
2801 }
2802 let registration = match self
2809 .registry
2810 .register_unpublished_verified_with_control_ops(
2811 hello.manifest,
2812 negotiated_ver,
2813 connection_id,
2814 control_ops,
2815 nonce_verified,
2816 ) {
2817 Ok(registration) => registration,
2818 Err(RegistryError::DuplicateModuleId { module_id }) => {
2819 return Ok(vec![control_error_frame(
2820 &frame,
2821 "duplicate_module_id",
2822 format!(
2823 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2824 ),
2825 )?])
2826 }
2827 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2828 return Ok(vec![control_error_frame(
2829 &frame,
2830 "invalid_module_id",
2831 err.to_string(),
2832 )?])
2833 }
2834 Err(err) => {
2835 return Ok(vec![control_error_frame(
2836 &frame,
2837 "registry_error",
2838 err.to_string(),
2839 )?])
2840 }
2841 };
2842 #[cfg(test)]
2843 self.hold_between_registry_and_forwarding();
2844
2845 let reply = if let Some(sink) = sink {
2846 let concurrency = manifest_concurrency(®istration.manifest);
2862 if let Err(err) = self.forwarding.register_module_connection_acked(
2863 connection_id,
2864 registration.manifest.module_id.clone(),
2865 negotiated_ver,
2866 concurrency,
2867 sink,
2868 hello_ack,
2869 ) {
2870 if matches!(
2875 self.deregister_connection(
2876 connection_id,
2877 RegistrationEndReason::RegistrationFailed,
2878 ),
2879 Ok(r) if !r.is_empty()
2880 ) {
2881 crate::supervise::notify_registration_release();
2882 }
2883 return Ok(vec![control_error_frame(
2884 &frame,
2885 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2886 "invalid_hello"
2887 } else {
2888 forwarding_error_code(&err)
2889 },
2890 err.to_string(),
2891 )?]);
2892 }
2893 Vec::new()
2894 } else {
2895 vec![hello_ack]
2898 };
2899
2900 if !self
2905 .registry
2906 .publish_registration(connection_id)
2907 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2908 {
2909 debug!(
2910 module_id = %registration.manifest.module_id,
2911 connection_id = connection_id.get(),
2912 "registration was removed before it could be published"
2913 );
2914 }
2915
2916 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2925 info!(
2926 module_id = %registration.manifest.module_id,
2927 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2928 );
2929 }
2930
2931 self.apply_registration_capabilities(®istration);
2932
2933 self.supervisor
2934 .record_activity_registration(®istration.manifest.module_id);
2935
2936 info!(
2937 module_id = %registration.manifest.module_id,
2938 module_version = %registration.manifest.module_version,
2939 negotiated_ver,
2940 routable_provider = manifest_provides_routable_role(®istration.manifest),
2941 connection_id = connection_id.get(),
2942 "module registered"
2943 );
2944
2945 Ok(reply)
2946 }
2947
2948 #[allow(clippy::too_many_arguments)]
2965 fn register_swap_candidate(
2966 &self,
2967 connection_id: ConnectionId,
2968 sink: Option<crate::FrameSink>,
2969 frame: &Frame,
2970 manifest: ModuleManifest,
2971 negotiated_ver: u8,
2972 control_ops: Vec<String>,
2973 hello_ack: Frame,
2974 ) -> Result<Vec<Frame>, RouterError> {
2975 let module_id = manifest.module_id.clone();
2976 let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
2977 manifest,
2978 negotiated_ver,
2979 connection_id,
2980 control_ops,
2981 ) {
2982 Ok(registration) => registration,
2983 Err(RegistryError::DuplicateModuleId { module_id }) => {
2984 return Ok(vec![control_error_frame(
2985 frame,
2986 "duplicate_module_id",
2987 format!(
2988 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2989 ),
2990 )?])
2991 }
2992 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2993 return Ok(vec![control_error_frame(
2994 frame,
2995 "invalid_module_id",
2996 err.to_string(),
2997 )?])
2998 }
2999 Err(err) => {
3000 return Ok(vec![control_error_frame(
3001 frame,
3002 "registry_error",
3003 err.to_string(),
3004 )?])
3005 }
3006 };
3007 #[cfg(test)]
3008 self.hold_between_registry_and_forwarding();
3009 let reply = if let Some(sink) = sink {
3010 let concurrency = manifest_concurrency(®istration.manifest);
3014 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
3015 connection_id,
3016 module_id.clone(),
3017 negotiated_ver,
3018 concurrency,
3019 sink,
3020 hello_ack,
3021 ) {
3022 if matches!(
3023 self.deregister_connection(
3024 connection_id,
3025 RegistrationEndReason::RegistrationFailed,
3026 ),
3027 Ok(r) if !r.is_empty()
3028 ) {
3029 crate::supervise::notify_registration_release();
3030 }
3031 return Ok(vec![control_error_frame(
3032 frame,
3033 forwarding_error_code(&err),
3034 err.to_string(),
3035 )?]);
3036 }
3037 Vec::new()
3038 } else {
3039 vec![hello_ack]
3040 };
3041 if !self
3042 .registry
3043 .publish_registration(connection_id)
3044 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3045 {
3046 debug!(
3047 module_id = %module_id,
3048 connection_id = connection_id.get(),
3049 "swap candidate registration was removed before it could be published"
3050 );
3051 }
3052 self.supervisor.mark_swap_candidate_admitted(&module_id);
3053 info!(
3054 module_id = %module_id,
3055 module_version = %registration.manifest.module_version,
3056 negotiated_ver,
3057 ready = registration.ready,
3058 connection_id = connection_id.get(),
3059 "swap candidate registered; not routable until cutover"
3060 );
3061 Ok(reply)
3062 }
3063
3064 fn build_hello_ack(
3065 &self,
3066 frame: &Frame,
3067 negotiated_ver: u8,
3068 module_id: &str,
3069 ) -> Result<Frame, RouterError> {
3070 let ack = ModuleHelloAckBody {
3071 negotiated_ver,
3072 subc_ops: module_subc_ops(),
3073 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
3074 storage: self
3075 .storage_config
3076 .as_ref()
3077 .map(|cfg| cfg.descriptor_for(module_id)),
3078 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3079 };
3080 let body = serde_json::to_vec(&ack).map_err(|err| {
3081 RouterError::backend(
3082 0,
3083 frame.header.corr,
3084 format!("failed to encode HELLO_ACK: {err}"),
3085 )
3086 })?;
3087
3088 Frame::build_with_version(
3089 negotiated_ver,
3090 FrameType::HelloAck,
3091 control_flags(),
3092 0,
3093 0,
3094 frame.header.corr,
3095 body,
3096 )
3097 .map_err(RouterError::FrameBuild)
3098 }
3099
3100 async fn handle_client_control_request(
3101 &self,
3102 ctx: &RouteCtx,
3103 frame: Frame,
3104 request: ClientControlRequest,
3105 route_open_timings: &mut Option<RouteOpenTimings>,
3106 ) -> Result<Vec<Frame>, RouterError> {
3107 if let Some(operation) = privileged_operation(&request) {
3108 let caller = self.resolve_caller(ctx.connection_id).await;
3109 let op = client_control_request_op(&request);
3110 let text = prompt_text(operation, &caller);
3111 match self.privileged_operator_prompt {
3112 PrivilegedOperatorPrompt::Log => {
3113 info!(target: "subc_daemon::privileged_prompt", operation = op,
3114 target = ?operation, caller = %caller.description(), summary = %text,
3115 "privileged request would prompt the operator");
3116 }
3117 PrivilegedOperatorPrompt::Enforce => {
3118 let key =
3119 crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
3120 let stamp = self
3121 .operator_apps
3122 .lock()
3123 .unwrap_or_else(|p| p.into_inner())
3124 .get(&ctx.connection_id)
3125 .cloned();
3126 let shortcut = crate::operator_confirm::ShortcutContext::eligible_stamp(
3127 &self.operator_app_policy,
3128 &self.launch_table,
3129 stamp.as_ref(),
3130 )
3131 .then(|| self.shortcut_context(stamp, None, None));
3132 let outcome = self
3133 .forwarding
3134 .operator_confirms()
3135 .approve_privileged_with_shortcut(
3136 ctx,
3137 &frame,
3138 op,
3139 text,
3140 caller.description(),
3141 key,
3142 shortcut,
3143 )
3144 .await;
3145 if outcome.code != "confirmed" {
3146 return Ok(vec![outcome.frame(&frame)]);
3147 }
3148 }
3149 }
3150 }
3151 match request {
3152 ClientControlRequest::DaemonShutdownIntent { intent } => {
3153 if intent.as_str() != Some("restart") {
3154 return Ok(vec![control_error_frame(
3155 &frame,
3156 "invalid_params",
3157 "intent must be 'restart'",
3158 )?]);
3159 }
3160 let response = ClientControlResponse::DaemonShutdownIntent {
3161 intent: "restart".to_owned(),
3162 expires_at_ms: self.supervisor.announce_daemon_restart(),
3163 };
3164 Ok(vec![control_response_body_frame(
3165 &frame,
3166 &response,
3167 "ClientControlResponse::DaemonShutdownIntent",
3168 )?])
3169 }
3170 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
3171 ClientControlRequest::CatalogList { module_id } => {
3172 self.handle_catalog_list(frame, module_id)
3173 }
3174 ClientControlRequest::RouteOpen {
3175 target,
3176 identity,
3177 consumer_identity,
3178 consumer_capabilities,
3179 role_versions,
3180 busy_refusals,
3181 admission_facts,
3182 scope,
3183 ..
3184 } => {
3185 self.handle_route_open(
3186 ctx,
3187 frame,
3188 RouteOpenRequest {
3189 target,
3190 identity,
3191 consumer_identity,
3192 consumer_capabilities,
3193 role_versions,
3194 busy_refusals,
3195 admission_facts,
3196 scope,
3197 },
3198 route_open_timings.get_or_insert_with(RouteOpenTimings::new),
3199 )
3200 .await
3201 }
3202 ClientControlRequest::RoutePoll {
3203 route_channel,
3204 route_epoch,
3205 kind,
3206 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
3207 ClientControlRequest::SupervisorList {} => {
3208 let Ok(permit) = Arc::clone(&self.list_admission).try_acquire_owned() else {
3209 self.agent_declarations.refuse(
3210 self,
3211 ctx,
3212 control_error_frame(
3213 &frame,
3214 "busy",
3215 "supervisor list work is at capacity; retry",
3216 )?,
3217 );
3218 return Ok(Vec::new());
3219 };
3220 let handler = self.clone();
3222 let sink = ctx.egress.clone();
3223 tokio::spawn(
3224 async move {
3225 let _permit = permit;
3226 let result = handler.handle_supervisor_list(frame.clone()).await;
3227 let replies = match result {
3228 Ok(replies) => replies,
3229 Err(error) => {
3230 control_error_frame(&frame, "backend_error", error.to_string())
3231 .into_iter()
3232 .collect()
3233 }
3234 };
3235 for reply in replies {
3236 if !matches!(
3237 tokio::time::timeout(Duration::from_millis(200), sink.send(reply))
3238 .await,
3239 Ok(Ok(()))
3240 ) {
3241 break;
3242 }
3243 }
3244 }
3245 .with_current_subscriber(),
3246 );
3247 Ok(Vec::new())
3248 }
3249 ClientControlRequest::SupervisorSpawnSnapshot {} => {
3250 self.handle_supervisor_spawn_snapshot(frame)
3251 }
3252 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
3253 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
3254 }
3255 ClientControlRequest::SupervisorRestart {
3256 module_id,
3257 drain_timeout_ms,
3258 } => {
3259 self.log_supervisor_request_received(
3260 ctx,
3261 frame.header.corr,
3262 ops::SUPERVISOR_RESTART,
3263 Some(&module_id),
3264 None,
3265 )?;
3266 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3267 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
3268 .await
3269 }
3270 ClientControlRequest::SupervisorSwap {
3271 module_id,
3272 ready_timeout_ms,
3273 } => {
3274 self.log_supervisor_request_received(
3275 ctx,
3276 frame.header.corr,
3277 ops::SUPERVISOR_SWAP,
3278 Some(&module_id),
3279 None,
3280 )?;
3281 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3282 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
3283 .await
3284 }
3285 ClientControlRequest::SupervisorReload { module_id } => {
3286 self.log_supervisor_request_received(
3287 ctx,
3288 frame.header.corr,
3289 ops::SUPERVISOR_RELOAD,
3290 Some(&module_id),
3291 None,
3292 )?;
3293 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3294 self.handle_supervisor_reload(frame, module_id, requester)
3295 .await
3296 }
3297 ClientControlRequest::SupervisorRescan { preview } => {
3298 if !preview {
3299 self.log_supervisor_request_received(
3300 ctx,
3301 frame.header.corr,
3302 ops::SUPERVISOR_RESCAN,
3303 None,
3304 None,
3305 )?;
3306 }
3307 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3308 let operation = self.handle_supervisor_rescan(
3309 frame,
3310 if preview {
3311 RescanMode::Preview
3312 } else {
3313 RescanMode::Execute
3314 },
3315 requester,
3316 );
3317 self.observe_control_wait(ctx, None, ops::SUPERVISOR_RESCAN, operation)
3318 .await
3319 }
3320 ClientControlRequest::SupervisorRescanChecked {
3321 preview,
3322 candidate,
3323 expect,
3324 } => {
3325 self.observe_control_wait(
3326 ctx,
3327 None,
3328 ops::SUPERVISOR_RESCAN,
3329 self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect),
3330 )
3331 .await
3332 }
3333 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
3334 self.log_supervisor_request_received(
3335 ctx,
3336 frame.header.corr,
3337 ops::SUPERVISOR_RELEASE_RESERVED,
3338 Some(&module_id),
3339 None,
3340 )?;
3341 self.handle_supervisor_release_reserved(frame, module_id)
3342 .await
3343 }
3344 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
3345 self.log_supervisor_request_received(
3346 ctx,
3347 frame.header.corr,
3348 ops::SUPERVISOR_SET_ENABLED,
3349 Some(&module_id),
3350 Some(enabled),
3351 )?;
3352 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3353 self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
3354 .await
3355 }
3356 ClientControlRequest::SupervisorHealthProbe { module_id } => {
3357 self.handle_supervisor_health_probe(ctx, frame, module_id)
3358 .await
3359 }
3360 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
3361 ClientControlRequest::SupervisorRoutes { module_id } => {
3362 self.handle_supervisor_routes(frame, module_id)
3363 }
3364 ClientControlRequest::SupervisorProvenance { module_id } => {
3365 self.handle_supervisor_provenance(ctx, frame, module_id)
3366 .await
3367 }
3368 ClientControlRequest::SupervisorStderrTail {
3369 module_id,
3370 max_lines,
3371 max_bytes,
3372 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
3373 ClientControlRequest::SupervisorTerminals { module_id } => {
3374 self.handle_supervisor_terminals(ctx, frame, module_id)
3375 .await
3376 }
3377 }
3378 }
3379
3380 fn log_supervisor_request_received(
3381 &self,
3382 ctx: &RouteCtx,
3383 corr: u64,
3384 op: &'static str,
3385 module_id: Option<&str>,
3386 enabled: Option<bool>,
3387 ) -> Result<(), RouterError> {
3388 let caller = self.supervisor_requester(ctx, corr)?;
3389 let caller = principal_label(&caller);
3390 info!(
3393 target: "control",
3394 op,
3395 module_id,
3396 enabled,
3397 connection_id = ctx.connection_id.get(),
3398 caller = %caller,
3399 "supervisor request received"
3400 );
3401 Ok(())
3402 }
3403
3404 fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
3405 Ok(self
3406 .registry
3407 .get_module_by_connection(ctx.connection_id)
3408 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
3409 .map(|registration| Principal::Reserved {
3410 module_id: registration.manifest.module_id,
3411 })
3412 .unwrap_or(Principal::Direct))
3413 }
3414
3415 fn handle_module_control_request(
3416 &self,
3417 connection_id: ConnectionId,
3418 frame: Frame,
3419 request: ModuleControlRequestFromModule,
3420 ) -> Result<Vec<Frame>, RouterError> {
3421 match request {
3422 ModuleControlRequestFromModule::CatalogUpdate {
3423 provides,
3424 capabilities,
3425 ready,
3426 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
3427 ModuleControlRequestFromModule::LiveRoots {} => {
3428 let registered = self
3429 .registry
3430 .get_module_by_connection(connection_id)
3431 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3432 let Some(registration) = registered else {
3433 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
3434 };
3435 let response = self
3436 .forwarding
3437 .live_roots(®istration.manifest.module_id)
3438 .map_err(RouterError::Forwarding)?;
3439 Ok(vec![control_response_body_frame(
3440 &frame,
3441 &response,
3442 "ModuleControlResponseToModule::LiveRoots",
3443 )?])
3444 }
3445 ModuleControlRequestFromModule::AgentCommands {
3446 started,
3447 finished,
3448 known,
3449 } => self.handle_agent_commands(connection_id, frame, started, finished, known),
3450 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
3451 self.handle_scope_sync(connection_id, frame, generation, scopes)
3452 }
3453 ModuleControlRequestFromModule::ScopeApply {
3454 generation,
3455 upsert,
3456 end,
3457 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
3458 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
3459 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
3460 }
3461 _ => Ok(vec![control_error_frame(
3462 &frame,
3463 "unsupported_control_frame",
3464 "unsupported module control operation",
3465 )?]),
3466 }
3467 }
3468
3469 fn handle_scope_served_by(
3470 &self,
3471 connection_id: ConnectionId,
3472 frame: Frame,
3473 ) -> Result<Vec<Frame>, RouterError> {
3474 let registered = self
3475 .registry
3476 .get_module_by_connection(connection_id)
3477 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3478 let Some(registered) = registered else {
3479 return Ok(vec![control_error_frame(
3480 &frame,
3481 "not_registered",
3482 "scope.served_by requires an active module registration owned by this connection",
3483 )?]);
3484 };
3485 let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
3486 {
3487 Ok(request) => request,
3488 Err(err) => {
3489 return Ok(vec![control_error_frame(
3490 &frame,
3491 "invalid_control_body",
3492 format!("malformed scope.served_by body: {err}"),
3493 )?])
3494 }
3495 };
3496 let answer = self
3497 .scopes
3498 .read()
3499 .unwrap_or_else(|poisoned| poisoned.into_inner())
3500 .served_by(
3501 ®istered.manifest.module_id,
3502 &request.scope_ref,
3503 request.scope_epoch,
3504 &request.module_id,
3505 );
3506 let reply = subc_control::ScopeServedByReply::new(
3507 answer,
3508 self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3509 );
3510 Ok(vec![control_response_body_frame(
3511 &frame,
3512 &reply,
3513 "ScopeServedByReply",
3514 )?])
3515 }
3516
3517 fn handle_operator_confirm(
3518 &self,
3519 ctx: &RouteCtx,
3520 frame: Frame,
3521 ) -> Result<Vec<Frame>, RouterError> {
3522 use crate::operator_confirm::{audit, Outcome};
3523 let registration = self
3524 .registry
3525 .get_module_by_connection(ctx.connection_id)
3526 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3527 let Some(registration) = registration else {
3528 let outcome = Outcome::refusal("not_registered");
3529 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3530 return Ok(vec![outcome.frame(&frame)]);
3531 };
3532 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3533 Ok(request) => request,
3534 Err(_) => {
3535 let outcome = Outcome::refusal("invalid_control_body");
3536 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3537 return Ok(vec![outcome.frame(&frame)]);
3538 }
3539 };
3540 let declared = matches!(
3541 request.kind,
3542 Some(subc_protocol::session::OperatorConfirmKind::Operation)
3543 ) && request.operation.as_ref().is_some_and(|operation| {
3544 registration
3545 .manifest
3546 .confirm_operations
3547 .as_ref()
3548 .is_some_and(|operations| operations.contains(operation))
3549 });
3550 let module_id = registration.manifest.module_id;
3551 let nonce = self
3555 .hello_launch_nonces
3556 .lock()
3557 .unwrap_or_else(|p| p.into_inner())
3558 .nonce(ctx.connection_id)
3559 .map(str::to_owned);
3560 let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3561 self.supervisor
3562 .spawned_consumer_authorized(&module_id, nonce)
3563 });
3564 let confirms = self.forwarding.operator_confirms();
3565 self.forwarding
3566 .with_operator_route(
3567 ctx.connection_id,
3568 request.route_channel,
3569 request.route_epoch,
3570 |binding| {
3571 let shortcut = binding
3572 .filter(|binding| {
3573 nonce_proven
3574 && crate::operator_confirm::valid_summary(&request.summary)
3575 && declared
3576 && crate::operator_confirm::ShortcutContext::route_eligible(
3577 &self.operator_app_policy,
3578 &self.launch_table,
3579 binding,
3580 )
3581 })
3582 .map(|binding| {
3583 let phone = crate::operator_confirm::ShortcutContext::phone_fingerprint(
3584 &self.operator_app_policy,
3585 binding,
3586 );
3587 let stamp = if phone.is_some() {
3588 None
3589 } else {
3590 binding.operator_app.clone()
3591 };
3592 self.shortcut_context(
3593 stamp,
3594 phone,
3595 Some((request.route_channel, request.route_epoch)),
3596 )
3597 });
3598 confirms.admit_with_shortcut(
3599 ctx,
3600 frame,
3601 module_id,
3602 nonce_proven,
3603 request,
3604 binding,
3605 shortcut,
3606 )
3607 },
3608 )
3609 .map_err(RouterError::Forwarding)
3610 }
3611
3612 fn shortcut_context(
3613 &self,
3614 stamp: Option<crate::operator_apps::OperatorAppStamp>,
3615 phone_fingerprint: Option<String>,
3616 route: Option<(u16, u32)>,
3617 ) -> crate::operator_confirm::ShortcutContext {
3618 crate::operator_confirm::ShortcutContext {
3619 ledger: self.activity.clone(),
3620 forwarding: self.forwarding.clone(),
3621 launches: self.launch_table.clone(),
3622 stamp,
3623 phone_fingerprint,
3624 route,
3625 connections: self.operator_apps.clone(),
3626 #[cfg(test)]
3627 hooks: Default::default(),
3628 }
3629 }
3630
3631 #[cfg(any(test, feature = "test-support"))]
3633 pub fn with_test_phone_shortcut(mut self, enabled: bool) -> Self {
3634 self.operator_app_policy.phone_enabled = enabled;
3635 self
3636 }
3637
3638 pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
3641 let mut scopes = ScopeRecords::from(scopes);
3642 for (index, record) in scopes.records.iter_mut().enumerate() {
3643 if let Some(refusal) = Self::canonicalize_workdir(record) {
3644 scopes.refusals.insert(index, refusal);
3645 }
3646 }
3647 scopes
3648 }
3649
3650 fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
3651 let workdir = record.workdir.as_mut()?;
3652 let path = std::path::Path::new(workdir);
3653 if !path.is_absolute() {
3654 return Some((
3655 error_codes::INVALID_WORKDIR,
3656 "workdir must be absolute".to_string(),
3657 ));
3658 }
3659 match ProjectRootId::from_path_allowing_missing(path) {
3660 Ok(root) => match root.as_path().to_str() {
3661 Some(canonical) => {
3662 *workdir = canonical.to_string();
3663 None
3664 }
3665 None => Some((
3666 error_codes::INVALID_WORKDIR,
3667 "canonical workdir is not UTF-8".to_string(),
3668 )),
3669 },
3670 Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
3671 }
3672 }
3673
3674 fn handle_scope_sync(
3678 &self,
3679 connection_id: ConnectionId,
3680 frame: Frame,
3681 generation: u64,
3682 scopes: Vec<ScopeRecord>,
3683 ) -> Result<Vec<Frame>, RouterError> {
3684 self.handle_scope_change(connection_id, frame, generation, scopes, None)
3685 }
3686
3687 fn handle_scope_change(
3688 &self,
3689 connection_id: ConnectionId,
3690 frame: Frame,
3691 generation: u64,
3692 scopes: Vec<ScopeRecord>,
3693 end: Option<Vec<ScopeEnd>>,
3694 ) -> Result<Vec<Frame>, RouterError> {
3695 let is_apply = end.is_some();
3696 let Some(registration) = self
3697 .registry
3698 .get_module_by_connection(connection_id)
3699 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3700 else {
3701 return Ok(vec![control_error_frame(
3702 &frame,
3703 "not_registered",
3704 "scope.sync and scope.apply require an active module registration owned by this connection",
3705 )?]);
3706 };
3707 let owner = registration.manifest.module_id;
3708 let scopes = Self::prepare_scope_records(scopes);
3709 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3710 let is_current_launch = |connection: ConnectionId| {
3711 self.hello_launch_nonces
3712 .lock()
3713 .unwrap_or_else(|poisoned| poisoned.into_inner())
3714 .presented(connection, current_nonce.as_deref())
3715 };
3716 let mut table = self
3721 .scopes
3722 .write()
3723 .unwrap_or_else(|poisoned| poisoned.into_inner());
3724 let outcome = match end {
3725 Some(end) => table.apply(
3726 &owner,
3727 connection_id,
3728 is_current_launch,
3729 generation,
3730 scopes,
3731 end,
3732 ),
3733 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3734 };
3735 let drained = match &outcome {
3736 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3737 Err(_) => Vec::new(),
3738 };
3739 let has_deadlines = outcome.is_ok() && table.has_deadlines();
3740 drop(table);
3741 if has_deadlines {
3742 self.scope_deadline_added.notify_one();
3743 }
3744 match outcome {
3745 Ok(applied) => {
3746 let counts = ScopeOutcomeCounts::of(&applied.results);
3747 info!(
3748 owner = %owner,
3749 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3750 generation,
3751 records = applied.results.len(),
3752 created = counts.created,
3753 replaced = counts.replaced,
3754 updated = counts.updated,
3755 unchanged = counts.unchanged,
3756 refused = counts.refused,
3757 ended = applied.ended.len(),
3758 tag_changes = applied.tag_changes.len(),
3759 routes_closed = drained.len(),
3760 "scope change accepted"
3761 );
3762 for refused in applied
3768 .results
3769 .iter()
3770 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3771 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3772 {
3773 warn!(
3774 owner = %owner,
3775 generation,
3776 scope_ref = %refused.scope_ref,
3777 scope_epoch = refused.scope_epoch,
3778 code = refused.code.as_deref().unwrap_or(""),
3779 "scope record refused"
3780 );
3781 }
3782 self.close_scope_drained_routes(drained);
3783 let response = if is_apply {
3784 ModuleControlResponseToModule::ScopeApply {
3785 generation,
3786 results: applied.results,
3787 end_results: applied.end_results,
3788 ended: applied.ended,
3789 }
3790 } else {
3791 ModuleControlResponseToModule::ScopeSync {
3792 generation,
3793 results: applied.results,
3794 ended: applied.ended,
3795 }
3796 };
3797 Ok(vec![control_response_body_frame(
3798 &frame,
3799 &response,
3800 "scope change response",
3801 )?])
3802 }
3803 Err(refusal) => {
3804 info!(
3805 owner = %owner,
3806 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3807 generation,
3808 code = refusal.code,
3809 "scope change refused"
3810 );
3811 Ok(vec![control_error_frame(
3812 &frame,
3813 refusal.code,
3814 refusal.message,
3815 )?])
3816 }
3817 }
3818 }
3819
3820 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3826 if drained.is_empty() {
3827 return;
3828 }
3829 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3830 BTreeMap::new();
3831 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3832 for route in drained {
3833 warn!(
3834 module_id = %route.module_id,
3835 reason = ?route.reason,
3836 client_connection_id = route.client.connection_id.get(),
3837 route_channel = route.client.channel,
3838 "closing route because its scope changed"
3839 );
3840 pushes
3841 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3842 .or_insert_with(|| (route.reason, Vec::new()))
3843 .1
3844 .push(EndpointRoute {
3845 goodbye_target: route.client.clone(),
3846 principal: Principal::Unverified,
3847 bound_at: Instant::now(),
3848 draining: false,
3849 drain_reason: None,
3850 });
3851 goodbyes.push(route.module);
3852 goodbyes.push(route.client);
3853 }
3854 for ((module_id, _), (reason, routes)) in pushes {
3855 send_route_control_pushes(
3856 &self.forwarding,
3857 routes,
3858 ClientControlPush::RouteClosed {
3859 module_id,
3860 channels: Vec::new(),
3861 reason,
3862 drained: false,
3863 abandoned: 0,
3864 excluded_subscriptions: 0,
3865 terminal: Some(false),
3866 },
3867 );
3868 }
3869 self.emit_route_goodbyes(goodbyes);
3870 }
3871
3872 fn handle_scope_describe(
3875 &self,
3876 connection_id: ConnectionId,
3877 frame: Frame,
3878 owner: Principal,
3879 scope_ref: String,
3880 ) -> Result<Vec<Frame>, RouterError> {
3881 let registered = self
3882 .registry
3883 .get_module_by_connection(connection_id)
3884 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3885 let Some(registered) = registered else {
3886 return Ok(vec![control_error_frame(
3887 &frame,
3888 "not_registered",
3889 "scope.describe requires an active module registration owned by this connection",
3890 )?]);
3891 };
3892 let mut description = self
3893 .scopes
3894 .read()
3895 .unwrap_or_else(|poisoned| poisoned.into_inner())
3896 .describe(&owner, &scope_ref);
3897 if !registered
3900 .manifest
3901 .capabilities
3902 .as_ref()
3903 .is_some_and(|caps| {
3904 caps.provides
3905 .iter()
3906 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
3907 })
3908 {
3909 if let Some(stamp) = description.stamp.as_mut() {
3910 stamp.attributes.on_behalf_of_agent = None;
3911 }
3912 }
3913 if !registered
3914 .manifest
3915 .capabilities
3916 .as_ref()
3917 .is_some_and(|caps| {
3918 caps.provides
3919 .iter()
3920 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
3921 })
3922 {
3923 if let Some(stamp) = description.stamp.as_mut() {
3924 stamp.attributes.flow_version = None;
3925 stamp.attributes.flow_code_hash = None;
3926 stamp.attributes.flow_install_id = None;
3927 }
3928 }
3929 let owner_configured = match &owner {
3930 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3935 _ => false,
3936 };
3937 let response = ModuleControlResponseToModule::ScopeDescribe {
3938 status: description.status,
3939 scope_epoch: description.scope_epoch,
3940 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3941 owner_synced: description.owner_synced,
3942 owner_configured,
3943 scope: description.stamp,
3944 };
3945 Ok(vec![control_response_body_frame(
3946 &frame,
3947 &response,
3948 "ModuleControlResponseToModule::ScopeDescribe",
3949 )?])
3950 }
3951
3952 fn handle_catalog_update(
3953 &self,
3954 connection_id: ConnectionId,
3955 frame: Frame,
3956 provides: Vec<ProviderRole>,
3957 capabilities: Option<CapabilityDeclarations>,
3958 ready: Option<bool>,
3959 ) -> Result<Vec<Frame>, RouterError> {
3960 self.refresh_capability_requirements();
3961 let Some(registration) = self
3962 .registry
3963 .get_module_by_connection(connection_id)
3964 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3965 else {
3966 return Ok(vec![control_error_frame(
3967 &frame,
3968 "not_registered",
3969 "catalog.update requires an active module registration owned by this connection",
3970 )?]);
3971 };
3972
3973 if let Some(message) =
3974 catalog_update_frozen_field_message(®istration.manifest, &provides)
3975 {
3976 return Ok(vec![control_error_frame(
3977 &frame,
3978 "catalog_update_frozen_field",
3979 message,
3980 )?]);
3981 }
3982
3983 let mut candidate = registration.manifest.clone();
3984 candidate.provides = provides.clone();
3985 candidate.capabilities = capabilities
3986 .clone()
3987 .or_else(|| registration.manifest.capabilities.clone());
3988 if let Some(rich_ui) = &candidate.rich_ui {
3989 if let Err(err) = rich_ui.validate() {
3990 return Ok(vec![control_error_frame(
3991 &frame,
3992 "invalid_manifest",
3993 err.to_string(),
3994 )?]);
3995 }
3996 }
3997 if let Err(err) = candidate.validate_capability_grammar() {
3998 return Ok(vec![control_error_frame(
3999 &frame,
4000 "invalid_capability_grammar",
4001 err.to_string(),
4002 )?]);
4003 }
4004
4005 let mut conflicts = self
4008 .capability_evaluator
4009 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
4010 if let Some(conflict) = conflicts.first() {
4011 let message = format!(
4012 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
4013 conflict.capability, conflict.claimants[0], candidate.module_id
4014 );
4015 for conflict in &mut conflicts {
4016 conflict.source = DuplicateClaimSource::CatalogUpdate;
4017 }
4018 log_duplicate_claim_events(conflicts);
4019 return Ok(vec![control_error_frame(
4020 &frame,
4021 "reserved_capability",
4022 message,
4023 )?]);
4024 }
4025
4026 let updated = self
4027 .registry
4028 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
4029 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4030 if updated.is_none() {
4031 return Ok(vec![control_error_frame(
4032 &frame,
4033 "not_registered",
4034 "catalog.update requires an active module registration owned by this connection",
4035 )?]);
4036 }
4037 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
4038 log_duplicate_claim_events(
4039 self.capability_evaluator
4040 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
4041 );
4042 }
4043 if capability_census_trigger(
4044 registration.manifest.capabilities.as_ref(),
4045 updated
4046 .as_ref()
4047 .and_then(|entry| entry.manifest.capabilities.as_ref()),
4048 ) {
4049 self.enforce_capability_denies();
4050 }
4051 self.refresh_capability_requirements();
4052
4053 let response = ModuleControlResponseToModule::CatalogUpdate {};
4054 control_response_body_frame(
4055 &frame,
4056 &response,
4057 "ModuleControlResponseToModule::CatalogUpdate",
4058 )
4059 .map(|frame| vec![frame])
4060 }
4061
4062 pub(crate) fn with_activity_ledger_config(
4063 mut self,
4064 run_dir: &Path,
4065 incarnation: String,
4066 config: crate::activity::ActivityConfig,
4067 ) -> Self {
4068 self.activity = Some(crate::activity::ActivityLedger::open_config(
4069 run_dir.join("activity.jsonl"),
4070 incarnation,
4071 config,
4072 ));
4073 self.supervisor.set_activity(self.activity.clone());
4074 self
4075 }
4076
4077 pub(crate) async fn configure_activity(
4078 &self,
4079 config: crate::activity::ActivityConfig,
4080 ) -> Result<(), &'static str> {
4081 match &self.activity {
4082 Some(ledger) => ledger.configure(config).await,
4083 None => Ok(()),
4084 }
4085 }
4086
4087 fn cancel_activity_subscription(
4088 &self,
4089 ctx: &RouteCtx,
4090 frame: &Frame,
4091 ) -> Result<bool, RouterError> {
4092 let task = self
4093 .activity_streams
4094 .lock()
4095 .unwrap_or_else(|p| p.into_inner())
4096 .remove(&(ctx.connection_id, frame.header.corr));
4097 let Some(task) = task else {
4098 return Ok(false);
4099 };
4100 task.abort();
4101 let end = Frame::build_with_version(
4102 response_version(frame),
4103 FrameType::StreamEnd,
4104 control_flags(),
4105 0,
4106 0,
4107 frame.header.corr,
4108 Vec::new(),
4109 )
4110 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
4111 let sink = ctx.egress.clone();
4112 tokio::spawn(async move {
4113 let _ = sink.send(end).await;
4114 });
4115 Ok(true)
4116 }
4117
4118 async fn handle_activity_read(
4119 &self,
4120 ctx: &RouteCtx,
4121 frame: Frame,
4122 ) -> Result<Vec<Frame>, RouterError> {
4123 use subc_control::{
4124 ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
4125 ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
4126 };
4127 #[derive(Deserialize)]
4128 struct Identity {
4129 #[serde(default)]
4130 consumer_identity: Option<ConsumerIdentity>,
4131 }
4132 let principal = match self
4136 .forwarding
4137 .module_id_for_connection(ctx.connection_id)
4138 .map_err(RouterError::Forwarding)?
4139 {
4140 Some(module_id) => Ok(Principal::Reserved { module_id }),
4141 None => {
4142 let identity: Identity = match serde_json::from_slice(&frame.body) {
4143 Ok(identity) => identity,
4144 Err(_) => {
4145 return Ok(vec![control_error_frame(
4146 &frame,
4147 ACTIVITY_MALFORMED,
4148 "malformed activity read",
4149 )?])
4150 }
4151 };
4152 self.route_open_principal(&frame, identity.consumer_identity)?
4153 }
4154 };
4155 let admitted = matches!(principal, Ok(Principal::Direct))
4156 || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
4157 if !admitted {
4158 return Ok(vec![control_error_frame(
4159 &frame,
4160 ACTIVITY_READ_FORBIDDEN,
4161 "activity reads require an operator or trusted mirror principal",
4162 )?]);
4163 }
4164 let Some(ledger) = &self.activity else {
4165 return Ok(vec![control_error_frame(
4166 &frame,
4167 ACTIVITY_UNAVAILABLE,
4168 "activity ledger unavailable",
4169 )?]);
4170 };
4171 if serde_json::from_slice::<ControlOpProbe>(&frame.body)
4172 .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
4173 {
4174 let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
4175 Ok(request) => ledger.query(request).await,
4176 Err(_) => Err(ACTIVITY_MALFORMED),
4177 };
4178 return match result {
4179 Ok(reply) => Ok(vec![control_response_body_frame(
4180 &frame,
4181 &reply,
4182 ops::ACTIVITY_QUERY,
4183 )?]),
4184 Err(code) => Ok(vec![control_error_frame(
4185 &frame,
4186 code,
4187 "activity query refused",
4188 )?]),
4189 };
4190 }
4191 let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
4192 Ok(request) => request,
4193 Err(_) => {
4194 return Ok(vec![control_error_frame(
4195 &frame,
4196 ACTIVITY_MALFORMED,
4197 "malformed activity subscription",
4198 )?])
4199 }
4200 };
4201 let mut subscription = match ledger.subscribe(request.since).await {
4202 Ok(subscription) => subscription,
4203 Err(code) => {
4204 return Ok(vec![control_error_frame(
4205 &frame,
4206 code,
4207 "activity subscription refused",
4208 )?])
4209 }
4210 };
4211 let key = (ctx.connection_id, frame.header.corr);
4212 let sink = ctx.egress.clone();
4213 let mut streams = self
4214 .activity_streams
4215 .lock()
4216 .unwrap_or_else(|p| p.into_inner());
4217 streams.retain(|_, task| !task.is_finished());
4218 if streams.get(&key).is_some_and(|task| !task.is_finished()) {
4219 return Ok(vec![control_error_frame(
4220 &frame,
4221 "duplicate_subscription",
4222 "activity subscription correlation id already in use",
4223 )?]);
4224 }
4225 let task = tokio::spawn(async move {
4226 loop {
4227 let entry = if let Some(entry) = subscription.replay.pop_front() {
4228 Some(entry)
4229 } else {
4230 subscription.receiver.recv().await
4231 };
4232 let Some(entry) = entry else {
4233 break;
4234 };
4235 let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
4236 return;
4237 };
4238 let Ok(next) = Frame::build_with_version(
4239 response_version(&frame),
4240 FrameType::StreamData,
4241 control_flags(),
4242 0,
4243 0,
4244 frame.header.corr,
4245 body,
4246 ) else {
4247 return;
4248 };
4249 if sink.send(next).await.is_err() {
4250 return;
4251 }
4252 }
4253 if let Some(code) = subscription.terminal_code() {
4256 if let Ok(end) =
4257 control_error_frame(&frame, code, "activity subscriber fell behind")
4258 {
4259 let _ = sink.send(end).await;
4260 }
4261 }
4262 });
4263 streams.insert(key, task.abort_handle());
4264 Ok(Vec::new())
4265 }
4266
4267 async fn handle_activity_record(
4268 &self,
4269 ctx: &RouteCtx,
4270 frame: Frame,
4271 ) -> Result<Vec<Frame>, RouterError> {
4272 use subc_control::{
4273 ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
4274 ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
4275 };
4276 let result = async {
4277 let module = self
4278 .forwarding
4279 .module_id_for_connection(ctx.connection_id)
4280 .ok()
4281 .flatten()
4282 .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
4283 let request: ActivityRecordRequest =
4284 serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
4285 crate::activity::validate(&module, &request)?;
4286 let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
4287 let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
4288 let at_ms = std::time::SystemTime::now()
4289 .duration_since(std::time::UNIX_EPOCH)
4290 .unwrap_or_default()
4291 .as_millis() as u64;
4292 ledger.record(module, request, actor, at_ms).await
4293 }
4294 .await;
4295 match result {
4296 Ok(entry) => Ok(vec![control_response_body_frame(
4297 &frame,
4298 &ActivityRecordResponse { seq: entry.seq },
4299 "activity.record",
4300 )?]),
4301 Err(code) => Ok(vec![control_error_frame(
4302 &frame,
4303 code,
4304 "activity recording refused",
4305 )?]),
4306 }
4307 }
4308
4309 fn activity_actor(
4310 &self,
4311 connection: ConnectionId,
4312 module: &str,
4313 on: Option<&subc_control::ActivityOn>,
4314 ) -> Result<subc_control::ActivityActor, &'static str> {
4315 use crate::forwarding::{DataRoute, DataRouteState};
4316 use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
4317 let principal = |id: &str| Principal::Reserved {
4318 module_id: id.to_owned(),
4319 };
4320 let mut actor = ActivityActor {
4321 principal: principal(module),
4322 scope_owner: None,
4323 agent_id: None,
4324 run_id: None,
4325 flow_id: None,
4326 project_root: None,
4327 workdir: None,
4328 project_id: None,
4329 };
4330 let scope = match on {
4331 None => return Ok(actor),
4332 Some(ActivityOn::Scope(scope)) => {
4333 let selector = subc_protocol::scope::ScopeSelector::new(
4334 principal(module),
4335 scope.scope_ref.clone(),
4336 Some(scope.epoch),
4337 );
4338 self.scopes
4339 .read()
4340 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4341 .attribute(&principal(module), module, &selector, true)
4342 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4343 }
4344 Some(ActivityOn::Route(route)) => {
4345 let DataRoute::Module(DataRouteState::Bound(binding)) = self
4346 .forwarding
4347 .lookup_data_route(connection, route.channel, route.epoch)
4348 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4349 else {
4350 return Err(ACTIVITY_ACTOR_UNKNOWN);
4351 };
4352 if binding.module_id != module {
4353 return Err(ACTIVITY_ACTOR_UNKNOWN);
4354 }
4355 actor.principal = binding.principal.clone();
4356 actor.project_root = binding
4357 .project_root
4358 .as_ref()
4359 .map(|root| root.as_path().to_path_buf());
4360 let Some(bound) = &binding.scope else {
4361 return Ok(actor);
4362 };
4363 let owner = principal(&bound.owner);
4364 let selector = subc_protocol::scope::ScopeSelector::new(
4365 owner.clone(),
4366 bound.scope_ref.clone(),
4367 Some(bound.tag.scope_epoch),
4368 );
4369 let admitted = self
4370 .scopes
4371 .read()
4372 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4373 .attribute(&owner, module, &selector, true)
4374 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
4375 if admitted.tag != bound.tag {
4376 return Err(ACTIVITY_ACTOR_UNKNOWN);
4377 }
4378 admitted
4379 }
4380 };
4381 actor.scope_owner = Some(scope.stamp.owner);
4382 actor.agent_id = scope.stamp.attributes.agent_id;
4383 actor.run_id = scope.stamp.attributes.run_id;
4384 actor.flow_id = scope.stamp.attributes.flow_id;
4385 actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
4386 actor.project_id = scope.stamp.project_id;
4387 Ok(actor)
4388 }
4389
4390 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4391 self.refresh_capability_requirements();
4392 let mut counters = self.counters.snapshot();
4398 let losses = self.resource_tracker.loss_diagnostics();
4399 if losses.as_object().is_some_and(|losses| !losses.is_empty()) {
4400 if let Some(obj) = counters.as_object_mut() {
4401 obj.insert("agent_command_declaration_loss_by_module".into(), losses);
4402 }
4403 }
4404 if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
4405 obj.insert(
4406 "activity_unparseable_lines".into(),
4407 ledger.read_errors().into(),
4408 );
4409 obj.insert(
4410 "activity_lifecycle_append_failures".into(),
4411 ledger.lifecycle_failures().into(),
4412 );
4413 obj.insert(
4414 "activity_started_digest_skips".into(),
4415 ledger.lifecycle_digest_skips().into(),
4416 );
4417 }
4418 if let (Ok((connections_with_routes, max)), Some(obj)) = (
4419 self.forwarding.client_route_concentration(),
4420 counters.as_object_mut(),
4421 ) {
4422 obj.insert(
4423 "client_connections_with_routes".into(),
4424 connections_with_routes.into(),
4425 );
4426 obj.insert("max_routes_on_one_connection".into(), max.into());
4427 }
4428 if let (Some(open_breakers), Some(obj)) = (
4434 self.route_bind_breakers.open_snapshot(),
4435 counters.as_object_mut(),
4436 ) {
4437 obj.insert("route_bind_breakers_open".into(), open_breakers);
4438 }
4439 if let (Some(reason), Some(obj)) = (
4444 self.local_endpoint_unavailable.clone(),
4445 counters.as_object_mut(),
4446 ) {
4447 obj.insert("local_endpoint_unavailable".into(), reason.into());
4448 }
4449 let response = ClientControlResponse::ServerDescribe {
4450 protocol_ver: PROTOCOL_VERSION,
4451 subc_ops: subc_ops(),
4452 capabilities: self.describe_capabilities(),
4453 connected_clients: self.connected_clients.count(),
4454 counters: Some(counters),
4455 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
4456 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
4457 capability_requirements: self.capability_requirement_statuses(),
4458 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
4459 config_path: self
4460 .rescan
4461 .as_ref()
4462 .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
4463 };
4464 Ok(vec![control_response_body_frame(
4465 &frame,
4466 &response,
4467 "ClientControlResponse::ServerDescribe",
4468 )?])
4469 }
4470
4471 fn handle_catalog_list(
4472 &self,
4473 frame: Frame,
4474 module_id: Option<String>,
4475 ) -> Result<Vec<Frame>, RouterError> {
4476 let (generation, modules) = self
4477 .registry
4478 .list_modules_with_nonce_verification()
4479 .map_err(|err| {
4480 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
4481 })?;
4482 let entries = modules
4483 .into_iter()
4484 .filter(|(registration, _)| {
4485 module_id
4486 .as_deref()
4487 .map(|wanted| registration.manifest.module_id == wanted)
4488 .unwrap_or(true)
4489 })
4490 .map(|(registration, nonce_verified)| {
4491 let not_ready = self.not_ready_reason(®istration);
4492 let roles = registration.manifest.provides;
4493 CatalogEntry::new(
4494 registration.manifest.module_id,
4495 roles,
4496 registration.control_ops,
4497 )
4498 .with_ready(not_ready.is_none())
4499 .with_display_name(registration.manifest.display_name)
4500 .with_not_ready(not_ready)
4501 .with_module_version(Some(registration.manifest.module_version))
4502 .with_provenance(registration.manifest.provenance)
4503 .with_capabilities(registration.manifest.capabilities)
4504 .with_self_signals(registration.manifest.self_signals)
4505 .with_events(registration.manifest.events)
4506 .with_operation_delivery(registration.manifest.operation_delivery)
4507 .with_operation_payload(registration.manifest.operation_payload)
4508 .with_flow_operations(registration.manifest.flow_operations)
4509 .with_confirm_operations(registration.manifest.confirm_operations)
4510 .with_rich_ui(registration.manifest.rich_ui)
4511 .with_role_declarations(registration.manifest.role_declarations)
4512 .with_nonce_verified(nonce_verified)
4513 })
4514 .collect();
4515 let response = ClientControlResponse::CatalogList {
4516 generation,
4517 modules: entries,
4518 subc_ops: subc_ops(),
4519 };
4520 Ok(vec![control_response_body_frame(
4521 &frame,
4522 &response,
4523 "ClientControlResponse::CatalogList",
4524 )?])
4525 }
4526
4527 fn attest_route_open_consumer(
4532 &self,
4533 consumer_identity: Option<&ConsumerIdentity>,
4534 ) -> Option<Principal> {
4535 let Some(consumer_identity) = consumer_identity else {
4536 return Some(Principal::Direct);
4537 };
4538 self.supervisor
4539 .spawned_consumer_authorized(
4540 &consumer_identity.module_id,
4541 &consumer_identity.launch_nonce,
4542 )
4543 .then(|| Principal::Reserved {
4544 module_id: consumer_identity.module_id.clone(),
4545 })
4546 }
4547
4548 fn route_open_caller_from_frame(&self, frame: &Frame) -> RouteOpenCaller {
4552 let Ok(ClientControlRequest::RouteOpen {
4553 identity,
4554 consumer_identity,
4555 scope,
4556 ..
4557 }) = parse_client_control_request(&frame.body)
4558 else {
4559 return RouteOpenCaller::unknown();
4560 };
4561 let principal = self.attest_route_open_consumer(consumer_identity.as_ref());
4562 RouteOpenCaller::new(principal.as_ref(), scope.as_ref(), &identity)
4563 }
4564
4565 fn route_open_principal(
4566 &self,
4567 frame: &Frame,
4568 consumer_identity: Option<ConsumerIdentity>,
4569 ) -> Result<Result<Principal, Frame>, RouterError> {
4570 let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4571 self.route_open_principal_attested(frame, consumer_identity, attested)
4572 }
4573
4574 fn route_open_principal_attested(
4578 &self,
4579 frame: &Frame,
4580 consumer_identity: Option<ConsumerIdentity>,
4581 attested: Option<Principal>,
4582 ) -> Result<Result<Principal, Frame>, RouterError> {
4583 if let Some(principal) = attested {
4584 return Ok(Ok(principal));
4585 }
4586 let Some(consumer_identity) = consumer_identity else {
4587 return Ok(Ok(Principal::Direct));
4588 };
4589
4590 Ok(Err(control_error_frame(
4591 frame,
4592 "bad_consumer_identity",
4593 format!(
4594 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
4595 consumer_identity.module_id
4596 ),
4597 )?))
4598 }
4599
4600 #[allow(clippy::too_many_arguments)]
4607 fn route_open_refusal_frame(
4608 &self,
4609 ctx: &RouteCtx,
4610 caller: &RouteOpenCaller,
4611 frame: &Frame,
4612 module_id: &str,
4613 reason: &'static str,
4614 code: &'static str,
4615 message: impl Into<String>,
4616 ) -> Result<Frame, RouterError> {
4617 self.observe_route_open_refusal(ctx, caller, module_id, reason, code);
4618 control_error_frame(frame, code, message.into())
4619 }
4620
4621 #[allow(clippy::too_many_arguments)]
4630 fn route_bind_rejection(
4631 &self,
4632 ctx: &RouteCtx,
4633 caller: &RouteOpenCaller,
4634 frame: &Frame,
4635 target_module_id: &str,
4636 body: ErrorBody,
4637 daemon_authored: bool,
4638 ) -> Result<Vec<Frame>, RouterError> {
4639 if daemon_authored {
4645 let scope_code = match body.code.as_str() {
4646 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
4647 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
4648 _ => None,
4649 };
4650 if let Some(code) = scope_code {
4651 self.observe_route_open_refusal(
4652 ctx,
4653 caller,
4654 target_module_id,
4655 "scope_changed_before_commit",
4656 code,
4657 );
4658 return Ok(vec![control_error_body_frame(frame, body)?]);
4659 }
4660 }
4661 self.counters
4662 .increment_route_open_refused("module_rejected");
4663 info!(
4664 target: "control",
4665 code = "module_rejected",
4666 module_code = ?body.code,
4667 module_id = ?target_module_id,
4668 connection_id = ctx.connection_id.get(),
4669 principal = %caller.principal,
4670 scope = %caller.scope,
4671 harness = %caller.harness,
4672 "route.open refused"
4673 );
4674 let refusal = if daemon_authored {
4675 control_error_body_frame(frame, body)?
4676 } else {
4677 control_relayed_error_body_frame(frame, body)?
4678 };
4679 Ok(vec![refusal])
4680 }
4681
4682 #[allow(clippy::too_many_arguments)]
4700 fn route_open_breaker_refusal_frame(
4701 &self,
4702 ctx: &RouteCtx,
4703 caller: &RouteOpenCaller,
4704 frame: &Frame,
4705 module_id: &str,
4706 consecutive_timeouts: u32,
4707 retry_in: Duration,
4708 probe_in_flight: bool,
4709 ) -> Result<Frame, RouterError> {
4710 self.counters
4711 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
4712 debug!(
4713 target: "control",
4714 code = "module_timeout",
4715 module_id = ?module_id,
4716 connection_id = ctx.connection_id.get(),
4717 principal = %caller.principal,
4718 scope = %caller.scope,
4719 harness = %caller.harness,
4720 consecutive_timeouts,
4721 retry_in_ms = retry_in.as_millis() as u64,
4722 probe_in_flight,
4723 "route.open refused by open bind-relay breaker"
4724 );
4725 let detail = if probe_in_flight {
4732 "one test route is already being tried; retry once it settles".to_string()
4733 } else {
4734 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
4735 };
4736 control_error_frame(
4737 frame,
4738 "module_timeout",
4739 format!(
4740 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
4741 timed out in a row); {detail}; its established routes are unaffected"
4742 ),
4743 )
4744 }
4745
4746 fn observe_route_open_refusal(
4759 &self,
4760 ctx: &RouteCtx,
4761 caller: &RouteOpenCaller,
4762 module_id: &str,
4763 reason: &'static str,
4764 code: &'static str,
4765 ) {
4766 self.counters.increment_route_open_refused(code);
4767 info!(
4768 target: "control",
4769 code,
4770 reason,
4771 module_id = ?module_id,
4772 connection_id = ctx.connection_id.get(),
4773 principal = %caller.principal,
4774 scope = %caller.scope,
4775 harness = %caller.harness,
4776 "route.open refused"
4777 );
4778 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
4779 self.route_outages.record_not_serving(module_id, reason);
4780 }
4781 }
4782
4783 fn observe_route_open_accept(
4828 &self,
4829 ctx: &RouteCtx,
4830 module_id: &str,
4831 principal: &str,
4832 scope: &str,
4833 ) {
4834 self.route_outages.record_accepted(module_id);
4835 self.counters.increment_route_open_accepted(principal);
4836 info!(
4837 target: "control",
4838 principal,
4839 module_id,
4840 connection_id = ctx.connection_id.get(),
4841 scope = %scope,
4842 "route.open accepted"
4843 );
4844 }
4845
4846 fn supervised_absent_route_open_refusal_frame(
4847 &self,
4848 ctx: &RouteCtx,
4849 caller: &RouteOpenCaller,
4850 frame: &Frame,
4851 module_id: &str,
4852 code: &'static str,
4853 status: &crate::supervise::ModuleStatus,
4854 ) -> Result<Frame, RouterError> {
4855 self.counters.increment_route_open_refused(code);
4856 info!(
4857 target: "control",
4858 code,
4859 reason = "supervised_not_registered",
4860 module_id = ?module_id,
4861 connection_id = ctx.connection_id.get(),
4862 principal = %caller.principal,
4863 scope = %caller.scope,
4864 harness = %caller.harness,
4865 state = %status.state,
4866 enabled = status.enabled,
4867 live = status.live,
4868 "route.open refused"
4869 );
4870 self.route_outages
4874 .record_not_serving(module_id, "supervised_not_registered");
4875 let message = if let Some(pid) = status.waiting_for_previous_pid {
4876 status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
4877 format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
4878 })
4879 } else {
4880 format!(
4881 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
4882 status.state, status.enabled, status.live
4883 )
4884 };
4885 control_error_frame(frame, code, message)
4886 }
4887
4888 async fn handle_route_open(
4889 &self,
4890 ctx: &RouteCtx,
4891 frame: Frame,
4892 request: RouteOpenRequest,
4893 timings: &mut RouteOpenTimings,
4894 ) -> Result<Vec<Frame>, RouterError> {
4895 let RouteOpenRequest {
4896 target,
4897 mut identity,
4898 consumer_identity,
4899 consumer_capabilities,
4900 role_versions,
4901 busy_refusals,
4902 admission_facts,
4903 scope,
4904 } = request;
4905 let target_module_id = target_module_id(&target).to_string();
4906 timings.module_id.clone_from(&target_module_id);
4907 if self
4908 .registry
4909 .get_module_by_connection(ctx.connection_id)
4910 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4911 .is_some()
4912 {
4913 return Ok(vec![control_error_frame(
4914 &frame,
4915 "invalid_request",
4916 "module connections cannot open client routes",
4917 )?]);
4918 }
4919 debug!(
4920 connection_id = ctx.connection_id.get(),
4921 corr = frame.header.corr,
4922 module_id = %target_module_id,
4923 "handling route.open"
4924 );
4925
4926 let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4931 let mut caller = RouteOpenCaller::new(attested.as_ref(), scope.as_ref(), &identity);
4932
4933 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
4939 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
4940 self.observe_route_open_refusal(
4941 ctx,
4942 &caller,
4943 &target_module_id,
4944 "invalid_role_versions",
4945 error_codes::INVALID_REQUEST,
4946 );
4947 return Ok(vec![control_error_body_frame(
4948 &frame,
4949 ErrorBody {
4950 code: error_codes::INVALID_REQUEST.to_string(),
4951 message: error.to_string(),
4952 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
4953 },
4954 )?]);
4955 }
4956
4957 if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
4958 if status.waiting_for_previous_pid.is_some() {
4959 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4960 ctx,
4961 &caller,
4962 &frame,
4963 &target_module_id,
4964 "module_warming",
4965 &status,
4966 )?]);
4967 }
4968 }
4969
4970 let Some(registration) = self
4989 .registry
4990 .get_module(&target_module_id)
4991 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4992 else {
4993 if let Some((status, warming)) =
4994 self.supervisor_status(&target_module_id, frame.header.corr)?
4995 {
4996 if status.protocol == ModuleProtocol::None {
5003 return Ok(vec![self.route_open_refusal_frame(
5004 ctx,
5005 &caller,
5006 &frame,
5007 &target_module_id,
5008 "protocol_none",
5009 error_codes::MODULE_NO_PROTOCOL,
5010 format!(
5011 "module_id '{target_module_id}' is declared protocol: none; \
5012 it speaks no subc wire and serves no routes"
5013 ),
5014 )?]);
5015 }
5016 let code = if warming {
5017 "module_warming"
5018 } else {
5019 "target_unavailable"
5020 };
5021 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
5022 ctx,
5023 &caller,
5024 &frame,
5025 &target_module_id,
5026 code,
5027 &status,
5028 )?]);
5029 }
5030 if let Some(removed_ago_ms) =
5031 self.supervisor.removal_tombstone_age_ms(&target_module_id)
5032 {
5033 return Ok(vec![self.route_open_refusal_frame(
5034 ctx,
5035 &caller,
5036 &frame,
5037 &target_module_id,
5038 "removed",
5039 error_codes::MODULE_REMOVED,
5040 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
5041 )?]);
5042 }
5043 return Ok(vec![self.route_open_refusal_frame(
5044 ctx,
5045 &caller,
5046 &frame,
5047 &target_module_id,
5048 "not_registered",
5049 error_codes::UNKNOWN_MODULE,
5050 format!("module_id '{target_module_id}' is not registered"),
5051 )?]);
5052 };
5053
5054 if !registration.ready {
5058 self.counters
5059 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
5060 info!(
5061 target: "control",
5062 code = error_codes::MODULE_WARMING,
5063 module_id = ?target_module_id,
5064 connection_id = ctx.connection_id.get(),
5065 principal = %caller.principal,
5066 scope = %caller.scope,
5067 harness = %caller.harness,
5068 reason = "declared_not_ready",
5069 "route.open refused"
5070 );
5071 self.route_outages
5074 .record_not_serving(&target_module_id, "declared_not_ready");
5075 return Ok(vec![control_error_body_frame(
5076 &frame,
5077 ErrorBody {
5078 code: error_codes::MODULE_WARMING.to_string(),
5079 message: format!(
5080 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
5081 ),
5082 detail: Some(serde_json::json!({
5083 "reason": "declared_not_ready"
5084 })),
5085 },
5086 )?]);
5087 }
5088
5089 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
5108 self.counters
5109 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
5110 info!(
5111 target: "control",
5112 code = error_codes::MODULE_WARMING,
5113 module_id = ?target_module_id,
5114 connection_id = ctx.connection_id.get(),
5115 principal = %caller.principal,
5116 scope = %caller.scope,
5117 harness = %caller.harness,
5118 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
5119 capability = %capability,
5120 "route.open refused"
5121 );
5122 return Ok(vec![control_error_body_frame(
5123 &frame,
5124 ErrorBody {
5125 code: error_codes::MODULE_WARMING.to_string(),
5126 message: format!(
5127 "module_id '{target_module_id}' requires capability '{capability}', \
5128 which no registered module provides; retry"
5129 ),
5130 detail: Some(serde_json::json!({
5131 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
5132 "capability": capability,
5133 })),
5134 },
5135 )?]);
5136 }
5137
5138 if !target_has_required_role(&target, ®istration.manifest.provides) {
5139 return Ok(vec![self.route_open_refusal_frame(
5140 ctx,
5141 &caller,
5142 &frame,
5143 &target_module_id,
5144 "role_not_provided",
5145 "target_unavailable",
5146 format!("module_id '{target_module_id}' does not provide the requested target"),
5147 )?]);
5148 }
5149
5150 if registration.state != ChannelState::Active {
5151 return Ok(vec![self.route_open_refusal_frame(
5152 ctx,
5153 &caller,
5154 &frame,
5155 &target_module_id,
5156 "registration_not_active",
5157 "target_unavailable",
5158 format!("module_id '{target_module_id}' is not active"),
5159 )?]);
5160 }
5161
5162 if self
5163 .forwarding
5164 .module_is_draining(&target_module_id)
5165 .map_err(RouterError::Forwarding)?
5166 {
5167 return Ok(vec![self.route_open_refusal_frame(
5168 ctx,
5169 &caller,
5170 &frame,
5171 &target_module_id,
5172 "reloading",
5173 "module_reloading",
5174 format!("module_id '{target_module_id}' is reloading"),
5175 )?]);
5176 }
5177
5178 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
5179 process_liveness.process_live(&target_module_id) == Some(false)
5180 }) {
5181 if process_liveness.process_replacing(&target_module_id) {
5188 return Ok(vec![self.route_open_refusal_frame(
5189 ctx,
5190 &caller,
5191 &frame,
5192 &target_module_id,
5193 "reloading",
5194 "module_reloading",
5195 format!("module_id '{target_module_id}' is reloading"),
5196 )?]);
5197 }
5198 return Ok(vec![self.route_open_refusal_frame(
5199 ctx,
5200 &caller,
5201 &frame,
5202 &target_module_id,
5203 "supervisor_not_live",
5204 "target_unavailable",
5205 format!("module_id '{target_module_id}' is not live"),
5206 )?]);
5207 }
5208
5209 if !self
5210 .forwarding
5211 .has_live_module_connection(&target_module_id)
5212 .map_err(RouterError::Forwarding)?
5213 {
5214 return Ok(vec![self.route_open_refusal_frame(
5215 ctx,
5216 &caller,
5217 &frame,
5218 &target_module_id,
5219 "no_forwarding_connection",
5220 "target_unavailable",
5221 format!("module_id '{target_module_id}' has no live forwarding connection"),
5222 )?]);
5223 }
5224
5225 if let Some(error) =
5226 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
5227 {
5228 self.observe_route_open_refusal(
5229 ctx,
5230 &caller,
5231 &target_module_id,
5232 "op_not_allowed",
5233 "op_not_allowed",
5234 );
5235 return Ok(vec![error]);
5236 }
5237
5238 let principal =
5239 match self.route_open_principal_attested(&frame, consumer_identity, attested)? {
5240 Ok(principal) => principal,
5241 Err(error) => {
5242 self.observe_route_open_refusal(
5243 ctx,
5244 &caller,
5245 &target_module_id,
5246 "bad_consumer_identity",
5247 "bad_consumer_identity",
5248 );
5249 return Ok(vec![error]);
5250 }
5251 };
5252 timings.principal = principal_label(&principal);
5253
5254 let build_class = identity.build_class.take().filter(|build_class| {
5257 if matches!(&principal, Principal::Reserved { module_id } if module_id == "callosum") {
5258 build_class == subc_protocol::BUILD_CLASS_PRODUCTION
5259 } else {
5260 if self
5261 .build_class_warned
5262 .lock()
5263 .unwrap_or_else(|p| p.into_inner())
5264 .insert(ctx.connection_id)
5265 {
5266 warn!(connection_id = ctx.connection_id.get(), principal = %timings.principal,
5267 "ignoring build_class from a non-Callosum opener");
5268 }
5269 false
5270 }
5271 });
5272
5273 if let Principal::Reserved {
5277 module_id: opening_module_id,
5278 } = &principal
5279 {
5280 if let Some(opening_registration) = self
5281 .registry
5282 .get_module(opening_module_id)
5283 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5284 {
5285 if let Some(capability) =
5286 denied_capability(&opening_registration.manifest, ®istration.manifest)
5287 {
5288 warn!(
5289 opening_module_id,
5290 target_module_id,
5291 capability,
5292 "refusing route.open because an attested capability deny edge matches"
5293 );
5294 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
5295 &frame,
5296 &target_module_id,
5297 "capability_deny_edge",
5298 "capability_forbidden",
5299 format!(
5300 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
5301 ),
5302 )?]);
5303 }
5304 }
5305 }
5306
5307 if admission_facts.is_some() {
5308 let carrier_matches = matches!(
5309 &principal,
5310 Principal::Reserved { module_id }
5311 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
5312 );
5313 if !carrier_matches {
5314 return Ok(vec![self.route_open_refusal_frame(
5315 ctx,
5316 &caller,
5317 &frame,
5318 &target_module_id,
5319 "admission_facts_carrier_not_permitted",
5320 "admission_facts_not_permitted",
5321 "admission facts may only be carried by the configured reserved module",
5322 )?]);
5323 }
5324
5325 let target_allowed = self
5326 .admission_facts_targets
5327 .as_ref()
5328 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
5329 if !target_allowed {
5330 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
5331 &frame,
5332 &target_module_id,
5333 "admission_facts_target_not_listed",
5334 "admission_facts_target_not_allowed",
5335 format!(
5336 "admission facts are not permitted for target module_id '{target_module_id}'"
5337 ),
5338 )?]);
5339 }
5340
5341 }
5345
5346 let (bound_scope, mut scope_stamp) = match &scope {
5352 None => (None, None),
5353 Some(selector) => {
5354 let owner_configured = match &selector.owner {
5355 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
5361 _ => false,
5362 };
5363 let mut scopes = self
5364 .scopes
5365 .write()
5366 .unwrap_or_else(|poisoned| poisoned.into_inner());
5367 let admitted = scopes
5368 .admit(&principal, &target_module_id, selector, owner_configured)
5369 .and_then(|admission| {
5370 crate::scopes::check_target_flow_support(
5371 &admission.stamp,
5372 &target_module_id,
5373 registration.manifest.capabilities.as_ref(),
5374 )?;
5375 crate::scopes::check_target_agent_run_support(
5376 &admission.stamp,
5377 &target_module_id,
5378 registration.manifest.capabilities.as_ref(),
5379 )?;
5380 let bound = scopes.reserve_bind(
5381 &admission,
5382 &principal,
5383 ctx.connection_id,
5384 selector.hold_presence,
5385 );
5386 Ok((admission, bound))
5387 });
5388 match admitted {
5389 Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
5390 Err(refusal) => {
5391 return Ok(vec![self.route_open_refusal_frame(
5392 ctx,
5393 &caller,
5394 &frame,
5395 &target_module_id,
5396 refusal.code,
5397 refusal.code,
5398 refusal.message,
5399 )?]);
5400 }
5401 }
5402 }
5403 };
5404 if let Some(stamp) = &scope_stamp {
5407 caller.scope = route_open_scope_label(stamp);
5408 }
5409
5410 timings.enter(RouteOpenPhase::ProjectRoot);
5418 let legacy_project = match crate::bind_compat::LegacyProject::from_open(&frame.body) {
5419 Ok(project) => project,
5420 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
5421 };
5422 timings.enter(RouteOpenPhase::Admission);
5423 let project_root = match legacy_project.effective_root(scope_stamp.as_ref()) {
5424 Ok(root) => root,
5425 Err(err) => {
5426 return Ok(vec![control_error_frame(
5427 &frame,
5428 "invalid_project_root",
5429 err.to_string(),
5430 )?])
5431 }
5432 };
5433 let legacy_bind_project = legacy_project.for_target(scope_stamp.as_ref());
5439
5440 let _concurrency_guard = match self
5451 .route_bind_concurrency
5452 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
5453 {
5454 Ok(guard) => guard,
5455 Err(in_flight) => {
5456 return Ok(vec![self.route_open_target_capacity_refusal(
5457 ctx,
5458 &caller,
5459 &frame,
5460 &target_module_id,
5461 in_flight,
5462 )?]);
5463 }
5464 };
5465
5466 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
5469 RouteBindAdmission::Admitted { guard, probe } => {
5470 if probe {
5471 info!(
5472 module_id = %target_module_id,
5473 connection_id = ctx.connection_id.get(),
5474 "route.bind breaker half-open: admitting one probe"
5475 );
5476 }
5477 guard
5478 }
5479 RouteBindAdmission::Refused {
5480 consecutive_timeouts,
5481 retry_in,
5482 probe_in_flight,
5483 } => {
5484 return Ok(vec![self.route_open_breaker_refusal_frame(
5485 ctx,
5486 &caller,
5487 &frame,
5488 &target_module_id,
5489 consecutive_timeouts,
5490 retry_in,
5491 probe_in_flight,
5492 )?]);
5493 }
5494 };
5495
5496 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
5501 let relay_deadline = Instant::now() + route_bind_relay_timeout;
5502 let operator_app = self
5503 .operator_apps
5504 .lock()
5505 .unwrap_or_else(|p| p.into_inner())
5506 .get(&ctx.connection_id)
5507 .cloned();
5508 timings.enter(RouteOpenPhase::EgressReservation);
5509 let pending = match self
5510 .forwarding
5511 .begin_route_bind_relay_for(
5512 ctx.connection_id,
5513 ctx.egress.clone(),
5514 response_version(&frame),
5515 frame.header.corr,
5516 &target_module_id,
5517 principal.clone(),
5518 bound_scope,
5519 project_root,
5520 identity.harness.clone(),
5521 build_class,
5522 operator_app.clone(),
5523 relay_deadline,
5524 busy_refusals == Some(true),
5525 )
5526 .await
5527 {
5528 Ok(pending) => pending,
5529 Err(err) => {
5530 return Ok(vec![self.route_open_refusal_frame(
5531 ctx,
5532 &caller,
5533 &frame,
5534 &target_module_id,
5535 "relay_reservation_failed",
5536 forwarding_error_code(&err),
5537 err.to_string(),
5538 )?])
5539 }
5540 };
5541 timings.enter(RouteOpenPhase::Admission);
5542 let crate::forwarding::PendingRouteBindRelay {
5543 endpoint,
5544 module_sink,
5545 negotiated_ver,
5546 client_channel,
5547 client_epoch,
5548 module_channel,
5549 module_epoch,
5550 corr: relay_corr,
5551 receiver,
5552 } = pending;
5553 let mut reservation =
5554 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
5555
5556 let relay_registration = self
5561 .registry
5562 .get_module_by_connection(endpoint.connection_id)
5563 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5564 let relay_capabilities = relay_registration
5565 .as_ref()
5566 .and_then(|registration| registration.manifest.capabilities.as_ref());
5567 if let Some(selector) = &scope {
5568 if let Err(refusal) = self
5571 .scopes
5572 .read()
5573 .unwrap_or_else(|poisoned| poisoned.into_inner())
5574 .check_bind_proof(&principal, selector)
5575 {
5576 reservation.release_and_disarm();
5577 return Ok(vec![self.route_open_refusal_frame(
5578 ctx,
5579 &caller,
5580 &frame,
5581 &target_module_id,
5582 refusal.code,
5583 refusal.code,
5584 refusal.message,
5585 )?]);
5586 }
5587 }
5588 if let Some(stamp) = scope_stamp
5589 .as_ref()
5590 .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
5591 {
5592 if let Err(refusal) = crate::scopes::check_target_flow_support(
5593 stamp,
5594 &target_module_id,
5595 relay_capabilities,
5596 )
5597 .and_then(|()| {
5598 crate::scopes::check_target_agent_run_support(
5599 stamp,
5600 &target_module_id,
5601 relay_capabilities,
5602 )
5603 }) {
5604 reservation.release_and_disarm();
5605 return Ok(vec![self.route_open_refusal_frame(
5606 ctx,
5607 &caller,
5608 &frame,
5609 &target_module_id,
5610 refusal.code,
5611 refusal.code,
5612 refusal.message,
5613 )?]);
5614 }
5615 }
5616
5617 if !relay_capabilities.is_some_and(|caps| {
5620 caps.provides
5621 .iter()
5622 .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
5623 }) {
5624 if let Some(stamp) = scope_stamp.as_mut() {
5625 stamp.workdir = None;
5626 stamp.project_id = None;
5627 }
5628 }
5629 let worker_run = scope_stamp
5634 .as_ref()
5635 .is_some_and(crate::scopes::is_worker_run)
5636 && relay_capabilities.is_some_and(|caps| {
5637 caps.provides
5638 .iter()
5639 .any(|cap| cap == subc_protocol::scope::WORKER_RUNS_CAPABILITY)
5640 });
5641 if !worker_run
5642 && !relay_capabilities.is_some_and(|caps| {
5643 caps.provides
5644 .iter()
5645 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
5646 })
5647 {
5648 if let Some(stamp) = scope_stamp.as_mut() {
5649 stamp.attributes.on_behalf_of_agent = None;
5650 }
5651 }
5652 if !relay_capabilities.is_some_and(|caps| {
5653 caps.provides
5654 .iter()
5655 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
5656 }) {
5657 if let Some(stamp) = scope_stamp.as_mut() {
5658 stamp.attributes.flow_version = None;
5659 stamp.attributes.flow_code_hash = None;
5660 stamp.attributes.flow_install_id = None;
5661 }
5662 }
5663 let call_mode = if worker_run
5664 || relay_capabilities
5665 .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
5666 {
5667 Some(derive_call_mode(scope_stamp.as_ref()))
5668 } else {
5669 if let Some(stamp) = scope_stamp.as_mut() {
5670 stamp.expires_at_ms = None;
5671 }
5672 None
5673 };
5674
5675 debug!(
5676 connection_id = ctx.connection_id.get(),
5677 client_channel,
5678 client_epoch,
5679 module_channel,
5680 module_epoch,
5681 "reserved route handle pair"
5682 );
5683 let route_principal_label = principal_label(&principal);
5686 let relay = ModuleControlRequest::RouteBind {
5687 route_channel: module_channel,
5688 epoch: module_epoch,
5689 target,
5690 identity,
5691 principal: Some(principal),
5692 consumer_capabilities,
5693 role_versions,
5694 admission_facts,
5695 scope: scope_stamp,
5696 call_mode,
5697 };
5698 let peer = RouteBindPeer::new()
5699 .with_observed_peer(
5700 relay_capabilities
5701 .is_some_and(|caps| {
5702 caps.provides
5703 .iter()
5704 .any(|cap| cap == OBSERVED_PEER_CAPABILITY)
5705 })
5706 .then(|| {
5707 self.observed_peers
5708 .lock()
5709 .unwrap_or_else(|p| p.into_inner())
5710 .get(&ctx.connection_id)
5711 .cloned()
5712 .unwrap_or_else(ObservedPeer::unobserved)
5713 }),
5714 )
5715 .with_operator_app(
5716 self.connection_operator_app(operator_app.as_ref())
5717 .filter(|_| {
5718 relay_capabilities.is_some_and(|caps| {
5719 caps.provides
5720 .iter()
5721 .any(|cap| cap == OPERATOR_APP_CAPABILITY)
5722 })
5723 }),
5724 );
5725 let relay_body = crate::bind_compat::encode_bind(
5726 &relay,
5727 &peer,
5728 &legacy_bind_project,
5729 relay_capabilities,
5730 )
5731 .map_err(|err| {
5732 RouterError::backend(
5733 0,
5734 frame.header.corr,
5735 format!("failed to encode route.bind request: {err}"),
5736 )
5737 })?;
5738 let relay_frame = Frame::build_with_version(
5739 negotiated_ver,
5740 FrameType::Request,
5741 control_flags(),
5742 0,
5743 0,
5744 relay_corr,
5745 relay_body,
5746 )
5747 .map_err(RouterError::FrameBuild)?;
5748
5749 timings.enter(RouteOpenPhase::BindSend);
5750 let sent = module_sink.send(relay_frame).await;
5751 timings.enter(RouteOpenPhase::Admission);
5752 if let Err(err) = sent {
5753 reservation.release_and_disarm();
5754 return Ok(vec![self.route_open_refusal_frame(
5755 ctx,
5756 &caller,
5757 &frame,
5758 &target_module_id,
5759 "relay_send_failed",
5760 "target_unavailable",
5761 err.to_string(),
5762 )?]);
5763 }
5764
5765 #[cfg(test)]
5766 if let Some(hold) = &self.route_bind_relay_sent_hold {
5767 hold.wait().await;
5771 hold.wait().await;
5772 }
5773
5774 if self
5779 .forwarding
5780 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
5781 .map_err(RouterError::Forwarding)?
5782 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
5783 {
5784 self.send_abandoned_route_bind_goodbye(
5785 &module_sink,
5786 negotiated_ver,
5787 module_channel,
5788 module_epoch,
5789 );
5790 }
5791
5792 timings.enter(RouteOpenPhase::BindAckWait);
5793 let outcome = timeout_at(relay_deadline, receiver).await;
5794 timings.enter(RouteOpenPhase::Admission);
5795 match outcome {
5796 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
5797 reservation.disarm();
5798 if breaker.record_accepted() {
5799 info!(
5800 module_id = %target_module_id,
5801 "route.bind breaker closed: the probe was accepted"
5802 );
5803 }
5804 self.observe_route_open_accept(
5805 ctx,
5806 &target_module_id,
5807 &route_principal_label,
5808 &caller.scope,
5809 );
5810 Ok(Vec::new())
5811 }
5812 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
5813 reservation.release_and_disarm();
5814 breaker.record_inconclusive();
5818 self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, false)
5819 }
5820 Ok(Ok(RouteBindRelayOutcome::DaemonRejected(body))) => {
5821 reservation.release_and_disarm();
5822 breaker.record_inconclusive();
5823 self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, true)
5824 }
5825 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
5826 reservation.release_and_disarm();
5827 breaker.record_inconclusive();
5828 tracing::warn!(
5835 module_id = %target_module_id,
5836 "route.bind relay abandoned: {message}"
5837 );
5838 Ok(vec![self.route_open_refusal_frame(
5839 ctx,
5840 &caller,
5841 &frame,
5842 &target_module_id,
5843 "relay_abandoned",
5844 "target_unavailable",
5845 message,
5846 )?])
5847 }
5848 Ok(Err(_)) => {
5849 reservation.release_and_disarm();
5850 breaker.record_inconclusive();
5851 Ok(vec![self.route_open_refusal_frame(
5852 ctx,
5853 &caller,
5854 &frame,
5855 &target_module_id,
5856 "relay_waiter_canceled",
5857 "target_unavailable",
5858 "route.bind relay waiter was canceled before the module responded",
5859 )?])
5860 }
5861 Err(_) => {
5862 reservation.release_and_disarm();
5863 if let Some(opened) = breaker.record_timeout(
5867 self.route_bind_breaker_threshold,
5868 self.route_bind_breaker_cooldown,
5869 ) {
5870 warn!(
5871 module_id = %target_module_id,
5872 consecutive_timeouts = opened.consecutive_timeouts,
5873 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
5874 reopened_after_probe = opened.reopened_after_probe,
5875 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
5876 );
5877 }
5878 tracing::warn!(
5886 module_id = %target_module_id,
5887 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
5888 "route.bind relay timed out: module did not ack within budget"
5889 );
5890 Ok(vec![self.route_open_refusal_frame(
5891 ctx,
5892 &caller,
5893 &frame,
5894 &target_module_id,
5895 "relay_timed_out",
5896 "module_timeout",
5897 format!(
5898 "module_id '{target_module_id}' did not answer route.bind within {:?}",
5899 route_bind_relay_timeout
5900 ),
5901 )?])
5902 }
5903 }
5904 }
5905
5906 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5907 let response = ClientControlResponse::SupervisorSpawnSnapshot {
5908 snapshot: self.supervisor.spawn_snapshot(),
5909 };
5910 Ok(vec![control_response_body_frame(
5911 &frame,
5912 &response,
5913 "ClientControlResponse::SupervisorSpawnSnapshot",
5914 )?])
5915 }
5916
5917 fn handle_supervisor_spawn_subscribe(
5918 &self,
5919 ctx: &RouteCtx,
5920 frame: Frame,
5921 since: Option<SpawnCursor>,
5922 ) -> Result<Vec<Frame>, RouterError> {
5923 match self.supervisor.subscribe_spawns(
5924 ctx.connection_id,
5925 frame.header.corr,
5926 response_version(&frame),
5927 since,
5928 ctx.egress.clone(),
5929 ) {
5930 Ok(()) => Ok(Vec::new()),
5931 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
5932 Ok(vec![control_error_body_frame(
5933 &frame,
5934 ErrorBody {
5935 code: "spawn_cursor_incarnation_mismatch".to_string(),
5936 message: "spawn cursor belongs to a different daemon incarnation"
5937 .to_string(),
5938 detail: Some(serde_json::json!({
5939 "current_daemon_incarnation": current
5940 })),
5941 },
5942 )?])
5943 }
5944 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
5945 &frame,
5946 ErrorBody {
5947 code: "spawn_cursor_too_old".to_string(),
5948 message: "spawn cursor predates the retained event ring".to_string(),
5949 detail: Some(serde_json::json!({
5950 "oldest_retained_cursor": oldest
5951 })),
5952 },
5953 )?]),
5954 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
5955 0,
5956 frame.header.corr,
5957 format!("failed to open supervisor spawn subscription: {error}"),
5958 )),
5959 }
5960 }
5961
5962 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5963 let handler = self.clone();
5966 let runtime = tokio::runtime::Handle::current();
5967 let subscriber = tracing::dispatcher::get_default(Clone::clone);
5970 let corr = frame.header.corr;
5971 tokio::task::spawn_blocking(move || {
5972 runtime.block_on(
5973 list_diagnostics::scope(handler.handle_supervisor_list_inner(frame))
5974 .with_subscriber(subscriber),
5975 )
5976 })
5977 .await
5978 .map_err(|error| RouterError::backend(0, corr, error.to_string()))?
5979 }
5980
5981 async fn handle_supervisor_list_inner(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5982 let generation = self
5983 .registry
5984 .generation()
5985 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5986 let supervised = self.supervisor.list();
5987 let mut statuses = BTreeMap::new();
5988 let mut not_running = BTreeSet::new();
5989 for module in &supervised {
5990 let status = module.status_for_control("list").map_err(|err| {
5991 RouterError::backend(
5992 0,
5993 frame.header.corr,
5994 format!("failed to read supervisor status: {err}"),
5995 )
5996 })?;
5997 if !status.enabled || !status.process_alive {
5998 not_running.insert(module.module_id().to_owned());
5999 }
6000 statuses.insert(module.module_id().to_owned(), status);
6001 }
6002 let mut process_resources = BTreeMap::new();
6005 for module in &supervised {
6006 process_resources.insert(
6007 module.module_id().to_owned(),
6008 crate::subtree_resources::read_process(module.clone()).await,
6009 );
6010 }
6011 let mut image_readings = BTreeMap::new();
6014 for module in &supervised {
6015 image_readings.insert(
6016 module.module_id().to_owned(),
6017 module.running_image_agreement().await,
6018 );
6019 }
6020 let readings = self
6021 .resource_tracker
6022 .read(supervised.clone(), false, "list")
6023 .await;
6024 let mut modules = Vec::new();
6025 for module in supervised {
6026 let status = statuses
6027 .remove(module.module_id())
6028 .expect("each supervised module has a status snapshot");
6029 let (configured, _) = module.configuration().map_err(|err| {
6030 RouterError::backend(
6031 0,
6032 frame.header.corr,
6033 format!("failed to read module configuration: {err}"),
6034 )
6035 })?;
6036 let next_retry_at_ms = module.next_retry_at_ms().map_err(|err| {
6037 RouterError::backend(
6038 0,
6039 frame.header.corr,
6040 format!("failed to read retry time: {err}"),
6041 )
6042 })?;
6043 let image = image_readings
6044 .remove(module.module_id())
6045 .expect("each supervised module has an image snapshot");
6046 let resources = process_resources.remove(module.module_id());
6047 let mut verdict =
6048 reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
6049 verdict.spec = Some(launch_spec_agreement(
6050 &configured,
6051 status.process_alive,
6052 status.launch_spec_digest.as_ref(),
6053 ));
6054 let pending_reload = Some(verdict);
6055 let configured_enabled = self
6056 .capability_evaluator
6057 .configured_enabled(&status.module_id);
6058 modules.push(
6061 SupervisorEntry::new(
6062 status.module_id,
6063 status.state.to_string(),
6064 status.enabled,
6065 status.live,
6066 status.health.status,
6067 )
6068 .with_configured_enabled(configured_enabled)
6069 .with_next_retry_at_ms(next_retry_at_ms)
6070 .with_launch_nonce_env(Some(!cfg!(unix)))
6071 .with_launch_nonce_source(status.launch_nonce_source)
6072 .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
6073 .with_protocol(status.protocol)
6074 .with_pending_reload(pending_reload)
6075 .with_last_probe_ms(status.health.last_probe_ms)
6076 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
6077 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
6078 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
6079 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
6080 .with_restart_count(Some(status.restart_count))
6081 .with_max_restarts(Some(status.max_restarts))
6082 .with_lifetime_restarts(Some(status.lifetime_restarts))
6083 .with_spawn_generation(Some(status.spawn_generation))
6084 .with_restart_window_secs(Some(status.restart_window.as_secs()))
6085 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
6086 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
6087 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
6088 .with_resources(resources),
6089 );
6090 }
6091 for module in &mut modules {
6092 if let Some((tree, hour)) = readings.modules.get(&module.module_id) {
6093 module.tree = tree.clone();
6094 module.cpu_last_hour = hour.clone();
6095 }
6096 if module.tree.is_none() {
6097 module.tree_unavailable = Some(
6098 if not_running.contains(&module.module_id) {
6099 "not_running"
6100 } else if readings.modules.contains_key(&module.module_id) {
6101 "tree_unavailable"
6102 } else {
6103 "not_yet_measured"
6104 }
6105 .to_owned(),
6106 );
6107 }
6108 }
6109 let response = ClientControlResponse::SupervisorList {
6110 generation,
6111 modules,
6112 daemon: readings.daemon,
6113 agent_commands: readings.agent_commands,
6114 };
6115 Ok(vec![control_response_body_frame(
6116 &frame,
6117 &response,
6118 "ClientControlResponse::SupervisorList",
6119 )?])
6120 }
6121
6122 fn handle_supervisor_stderr_tail(
6123 &self,
6124 frame: Frame,
6125 module_id: String,
6126 max_lines: Option<u32>,
6127 max_bytes: Option<u32>,
6128 ) -> Result<Vec<Frame>, RouterError> {
6129 let Some(module) = self.supervisor.get(&module_id) else {
6130 return Ok(vec![control_error_frame(
6131 &frame,
6132 "unknown_module",
6133 format!("module_id '{module_id}' is not supervised"),
6134 )?]);
6135 };
6136
6137 let snapshot = module.stderr_tail(
6138 max_lines.map(|value| value as usize),
6139 max_bytes.map(|value| value as usize),
6140 );
6141
6142 let response = ClientControlResponse::SupervisorStderrTail {
6143 module_id,
6144 tail: StderrTail {
6145 capture: match snapshot.capture {
6146 CaptureState::Captured => StderrCaptureState::Captured,
6147 CaptureState::Incomplete { reason } => {
6148 StderrCaptureState::Incomplete { reason }
6149 }
6150 CaptureState::NotCaptured { reason } => {
6151 StderrCaptureState::NotCaptured { reason }
6152 }
6153 },
6154 entries: snapshot
6155 .entries
6156 .into_iter()
6157 .map(|entry| match entry {
6158 TailEntry::Line {
6159 text,
6160 truncated,
6161 at_ms,
6162 } => StderrTailEntry::Line {
6163 text,
6164 truncated,
6165 at_ms,
6166 },
6167 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
6168 })
6169 .collect(),
6170 dropped_lines: snapshot.dropped_lines,
6171 },
6172 };
6173 Ok(vec![control_response_body_frame(
6174 &frame,
6175 &response,
6176 "ClientControlResponse::SupervisorStderrTail",
6177 )?])
6178 }
6179
6180 async fn observe_control_wait<F: std::future::Future>(
6181 &self,
6182 ctx: &RouteCtx,
6183 target: Option<&str>,
6184 kind: &'static str,
6185 future: F,
6186 ) -> F::Output {
6187 self.counters
6188 .observe_reader_wait(
6189 || {
6190 let module = self
6191 .forwarding
6192 .module_id_for_connection(ctx.connection_id)
6193 .ok()
6194 .flatten();
6195 let principal = if module.is_some() { "module" } else { "direct" };
6196 (
6197 self.resolve_reader_wait_target(kind, target)
6198 .or(module)
6199 .unwrap_or_else(|| crate::observability::CONTROL_WAIT_BUCKET.into()),
6200 principal.into(),
6201 )
6202 },
6203 kind,
6204 future,
6205 )
6206 .await
6207 }
6208
6209 pub(crate) fn resolve_reader_wait_target(
6212 &self,
6213 op: &str,
6214 selector: Option<&str>,
6215 ) -> Option<String> {
6216 let selector = selector?;
6217 match op {
6218 ops::SUPERVISOR_TERMINALS | ops::SUPERVISOR_PROVENANCE => self
6219 .supervisor
6220 .get(selector)
6221 .map(|module| module.module_id().to_owned()),
6222 ops::SUPERVISOR_HEALTH_PROBE => self
6223 .registry
6224 .get_module(selector)
6225 .ok()
6226 .flatten()
6227 .map(|registration| registration.manifest.module_id),
6228 _ => None,
6229 }
6230 }
6231
6232 async fn handle_supervisor_terminals(
6233 &self,
6234 ctx: &RouteCtx,
6235 frame: Frame,
6236 module_id: String,
6237 ) -> Result<Vec<Frame>, RouterError> {
6238 let Some(module) = self.supervisor.get(&module_id) else {
6239 return Ok(vec![control_error_frame(
6240 &frame,
6241 "unknown_module",
6242 format!("module_id '{module_id}' is not supervised"),
6243 )?]);
6244 };
6245
6246 let terminals = self
6249 .observe_control_wait(
6250 ctx,
6251 Some(module.module_id()),
6252 ops::SUPERVISOR_TERMINALS,
6253 module.read_durable_terminal_history(),
6254 )
6255 .await
6256 .map_err(|error| {
6257 RouterError::backend(
6258 0,
6259 frame.header.corr,
6260 format!("failed to read terminal history: {error}"),
6261 )
6262 })?;
6263 let response = ClientControlResponse::SupervisorTerminals {
6264 module_id,
6265 terminals,
6266 };
6267 Ok(vec![control_response_body_frame(
6268 &frame,
6269 &response,
6270 "ClientControlResponse::SupervisorTerminals",
6271 )?])
6272 }
6273
6274 fn handle_supervisor_routes(
6275 &self,
6276 frame: Frame,
6277 module_id: Option<String>,
6278 ) -> Result<Vec<Frame>, RouterError> {
6279 let modules = self
6280 .forwarding
6281 .route_census(module_id.as_deref())
6282 .map_err(RouterError::Forwarding)?
6283 .into_iter()
6284 .map(|(module_id, routes)| SupervisorRouteModule {
6285 module_id,
6286 routes: routes
6287 .into_iter()
6288 .map(|route| SupervisorRoute {
6289 consumer: match route.principal {
6290 Principal::Reserved { module_id } => {
6291 SupervisorRouteConsumer::Reserved { module_id }
6292 }
6293 Principal::Direct | Principal::Unverified => {
6294 SupervisorRouteConsumer::Direct {
6295 connection_id: route.goodbye_target.connection_id.get(),
6296 }
6297 }
6298 },
6299 age_ms: Instant::now()
6300 .saturating_duration_since(route.bound_at)
6301 .as_millis()
6302 .try_into()
6303 .unwrap_or(u64::MAX),
6304 draining: route.draining,
6305 drain_reason: route.drain_reason,
6306 })
6307 .collect(),
6308 })
6309 .collect();
6310 let response = ClientControlResponse::SupervisorRoutes { modules };
6311 Ok(vec![control_response_body_frame(
6312 &frame,
6313 &response,
6314 "ClientControlResponse::SupervisorRoutes",
6315 )?])
6316 }
6317
6318 async fn handle_supervisor_provenance(
6319 &self,
6320 ctx: &RouteCtx,
6321 frame: Frame,
6322 module_id: Option<String>,
6323 ) -> Result<Vec<Frame>, RouterError> {
6324 let targeted = module_id.is_some();
6325 let selected = if let Some(module_id) = module_id {
6326 let Some(module) = self.supervisor.get(&module_id) else {
6327 return Ok(vec![control_error_frame(
6328 &frame,
6329 "unknown_module",
6330 format!("module_id '{module_id}' is not supervised"),
6331 )?]);
6332 };
6333 vec![module]
6334 } else {
6335 self.supervisor.list()
6336 };
6337
6338 let target = targeted.then(|| selected[0].module_id());
6339 self.observe_control_wait(ctx, target, ops::SUPERVISOR_PROVENANCE, async {
6340 let mut modules = Vec::with_capacity(selected.len());
6341 for module in &selected {
6342 let (status, observed_image) = module
6343 .status_and_running_image_agreement()
6344 .await
6345 .map_err(|err| {
6346 RouterError::backend(
6347 0,
6348 frame.header.corr,
6349 format!("failed to read supervisor status: {err}"),
6350 )
6351 })?;
6352 let module_declared = self
6353 .registry
6354 .get_module(&status.module_id)
6355 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6356 .and_then(|registration| registration.manifest.provenance)
6357 .map(|build| ModuleDeclaredProvenance::Reported { build })
6358 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
6359 #[cfg(test)]
6360 let running_image = match &self.provenance_probe_override {
6361 Some(result) => result.clone(),
6362 None => observed_image,
6363 };
6364 #[cfg(not(test))]
6365 let running_image = observed_image;
6366 modules.push(SupervisorModuleProvenance {
6367 module_id: status.module_id,
6368 module_declared,
6369 daemon_observed: SupervisorObservedProcess {
6370 launch_nonce_source: status.launch_nonce_source,
6371 pid: status.pid,
6372 spawned_at_ms: status.spawned_at_ms,
6373 spawned_from: status.spawned_from,
6374 running_image,
6375 },
6376 });
6377 }
6378 let daemon = SupervisorDaemonProvenance {
6379 daemon_build: self.daemon_provenance.build.clone(),
6380 daemon_observed: DaemonObservedProcess {
6381 pid: self.daemon_provenance.pid,
6382 started_at_ms: self
6383 .daemon_provenance
6384 .start_clock
6385 .map(|clock| clock.started_at_ms())
6386 .or(self.daemon_provenance.started_at_ms),
6387 running_image: self
6388 .daemon_provenance
6389 .probe
6390 .observe(
6391 self.daemon_provenance.pid,
6392 self.daemon_provenance.executable_path.as_deref(),
6393 self.daemon_provenance.executable_identity.clone(),
6394 self.daemon_provenance.process_start_time,
6395 )
6396 .await,
6397 },
6398 };
6399 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
6400 Ok(vec![control_response_body_frame(
6401 &frame,
6402 &response,
6403 "ClientControlResponse::SupervisorProvenance",
6404 )?])
6405 })
6406 .await
6407 }
6408
6409 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
6410 self.refresh_capability_requirements();
6411 let generation = self
6412 .registry
6413 .generation()
6414 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
6415 let modules = self
6416 .supervisor
6417 .list()
6418 .into_iter()
6419 .map(|module| {
6420 let status = module.status_for_control("health").map_err(|err| {
6421 RouterError::backend(
6422 0,
6423 frame.header.corr,
6424 format!("failed to read supervisor health: {err}"),
6425 )
6426 })?;
6427 let module_id = status.module_id;
6428 let capability_detail = self
6429 .capability_evaluator
6430 .required_problem_detail(&module_id);
6431 Ok(SupervisorHealthEntry {
6432 module_id,
6433 status: status.health.status,
6434 detail: append_capability_problem_detail(
6435 status.health.detail,
6436 capability_detail,
6437 ),
6438 metrics: status.health.metrics,
6439 consecutive_failures: status.health.consecutive_failures,
6440 late_answer_count: status.health.late_answer_count,
6441 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
6442 last_action: status.health.last_action,
6443 last_action_ms: status.health.last_action_ms,
6444 last_probe_ms: status.health.last_probe_ms,
6445 })
6446 })
6447 .collect::<Result<Vec<_>, RouterError>>()?;
6448 let response = ClientControlResponse::SupervisorHealth {
6449 generation,
6450 modules,
6451 };
6452 Ok(vec![control_response_body_frame(
6453 &frame,
6454 &response,
6455 "ClientControlResponse::SupervisorHealth",
6456 )?])
6457 }
6458
6459 async fn handle_supervisor_restart(
6460 &self,
6461 frame: Frame,
6462 module_id: String,
6463 drain_timeout_ms: Option<u64>,
6464 requester: Principal,
6465 ) -> Result<Vec<Frame>, RouterError> {
6466 let operation_lock = self.supervisor.operation_lock();
6467 let _operation_guard = operation_lock.lock().await;
6468 let Some(module) = self.supervisor.get(&module_id) else {
6469 return Ok(vec![control_error_frame(
6470 &frame,
6471 "unknown_module",
6472 format!("module_id '{module_id}' is not supervised"),
6473 )?]);
6474 };
6475
6476 self.route_outages.mark_operator_action(&module_id);
6477 if let Err(err) = module
6478 .restart_requested(drain_timeout_ms, Some(requester))
6479 .await
6480 {
6481 self.route_outages
6482 .operator_action_ended_unrefused(&module_id);
6483 let (code, message) = match err {
6484 crate::supervise::SuperviseError::Disabled { .. } => {
6485 ("module_disabled", err.to_string())
6486 }
6487 crate::supervise::SuperviseError::SwapInProgress { .. } => {
6488 ("swap_in_progress", err.to_string())
6489 }
6490 _ => (
6491 "target_unavailable",
6492 format!("failed to restart module_id '{module_id}': {err}"),
6493 ),
6494 };
6495 return Ok(vec![control_error_frame(&frame, code, message)?]);
6496 }
6497
6498 let response = ClientControlResponse::SupervisorAck {
6499 module_id,
6500 applied: true,
6501 };
6502 Ok(vec![control_response_body_frame(
6503 &frame,
6504 &response,
6505 "ClientControlResponse::SupervisorAck",
6506 )?])
6507 }
6508
6509 async fn handle_supervisor_swap(
6513 &self,
6514 frame: Frame,
6515 module_id: String,
6516 ready_timeout_ms: Option<u64>,
6517 requester: Principal,
6518 ) -> Result<Vec<Frame>, RouterError> {
6519 let module = {
6526 let operation_lock = self.supervisor.operation_lock();
6527 let _operation_guard = operation_lock.lock().await;
6528 self.supervisor.get(&module_id)
6529 };
6530 let Some(module) = module else {
6531 return Ok(vec![control_error_frame(
6532 &frame,
6533 "unknown_module",
6534 format!("module_id '{module_id}' is not supervised"),
6535 )?]);
6536 };
6537
6538 self.route_outages.mark_operator_action(&module_id);
6539 if let Err(err) = module
6540 .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
6541 .await
6542 {
6543 self.route_outages
6544 .operator_action_ended_unrefused(&module_id);
6545 use crate::supervise::SuperviseError;
6546 let message = err.to_string();
6547 let error = match err {
6548 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
6549 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
6550 code: "swap_refused".to_string(),
6551 message,
6552 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
6553 },
6554 SuperviseError::SwapFailed {
6555 arm,
6556 candidate_exit,
6557 ..
6558 } => ErrorBody {
6559 code: "swap_failed".to_string(),
6560 message,
6561 detail: Some(serde_json::json!({
6562 "arm": arm.as_str(),
6563 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
6564 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
6565 })),
6566 },
6567 _ => ErrorBody::new(
6568 "target_unavailable",
6569 format!("failed to swap module_id '{module_id}': {message}"),
6570 ),
6571 };
6572 return Ok(vec![control_error_body_frame(&frame, error)?]);
6573 }
6574 self.route_outages
6578 .operator_action_ended_unrefused(&module_id);
6579
6580 let response = ClientControlResponse::SupervisorAck {
6581 module_id,
6582 applied: true,
6583 };
6584 Ok(vec![control_response_body_frame(
6585 &frame,
6586 &response,
6587 "ClientControlResponse::SupervisorAck",
6588 )?])
6589 }
6590
6591 async fn handle_supervisor_reload(
6592 &self,
6593 frame: Frame,
6594 module_id: String,
6595 requester: Principal,
6596 ) -> Result<Vec<Frame>, RouterError> {
6597 let operation_lock = self.supervisor.operation_lock();
6598 let _operation_guard = operation_lock.lock().await;
6599 let Some(module) = self.supervisor.get(&module_id) else {
6600 return Ok(vec![control_error_frame(
6601 &frame,
6602 "unknown_module",
6603 format!("module_id '{module_id}' is not supervised"),
6604 )?]);
6605 };
6606
6607 self.route_outages.mark_operator_action(&module_id);
6608 if let Err(err) = module.reload_requested(Some(requester)).await {
6609 self.route_outages
6610 .operator_action_ended_unrefused(&module_id);
6611 let (code, message) = match err {
6612 crate::supervise::SuperviseError::Disabled { .. } => {
6613 ("module_disabled", err.to_string())
6614 }
6615 crate::supervise::SuperviseError::SwapInProgress { .. } => {
6616 ("swap_in_progress", err.to_string())
6617 }
6618 _ => (
6619 "reload_failed",
6620 format!("failed to reload module_id '{module_id}': {err}"),
6621 ),
6622 };
6623 return Ok(vec![control_error_frame(&frame, code, message)?]);
6624 }
6625
6626 let response = ClientControlResponse::SupervisorAck {
6627 module_id,
6628 applied: true,
6629 };
6630 Ok(vec![control_response_body_frame(
6631 &frame,
6632 &response,
6633 "ClientControlResponse::SupervisorAck",
6634 )?])
6635 }
6636
6637 async fn handle_supervisor_rescan(
6641 &self,
6642 frame: Frame,
6643 mode: RescanMode,
6644 requester: Principal,
6645 ) -> Result<Vec<Frame>, RouterError> {
6646 let preview = mode.previews();
6647 let Some(context) = self.rescan.clone() else {
6648 return Ok(vec![control_error_frame(
6649 &frame,
6650 "rescan_unavailable",
6651 "the daemon was not started with a reloadable config path".to_string(),
6652 )?]);
6653 };
6654
6655 let operation_lock = self.supervisor.operation_lock();
6656 let _operation_guard = operation_lock.lock().await;
6657 let loaded = match self.load_rescan_config(&mode, &context) {
6661 Ok(config) => config,
6662 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
6663 };
6664 let Some((config, activity_config)) = loaded else {
6672 return Ok(vec![control_error_frame(
6673 &frame,
6674 "invalid_daemon_config",
6675 format!(
6676 "daemon config not found at {}; refusing to rescan (an absent config would \
6677 retire every supervised module)",
6678 context.config_path.display()
6679 ),
6680 )?]);
6681 };
6682 let (
6683 configured_port,
6684 storage_config,
6685 admission_facts_carrier_module_id,
6686 admission_facts_targets,
6687 scope_authority_owners,
6688 privileged_operator_prompt,
6689 modules,
6690 reserved_capabilities,
6691 ) = (
6692 config.port,
6693 config.storage,
6694 config.admission_facts_carrier_module_id,
6695 config.admission_facts_targets,
6696 config.scope_authority_owners,
6697 config.privileged_operator_prompt,
6698 config.modules,
6699 config.reserved_capabilities,
6700 );
6701
6702 let mut restart_required = Vec::new();
6711 for section in RestartRequiredSection::ALL {
6712 let changed = match section {
6713 RestartRequiredSection::Port => configured_port != context.configured_port,
6714 RestartRequiredSection::Storage => storage_config != context.storage_config,
6715 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
6716 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
6717 }
6718 RestartRequiredSection::AdmissionFactsTargets => {
6719 admission_facts_targets != context.admission_facts_targets
6720 }
6721 RestartRequiredSection::ScopeAuthorityOwners => {
6722 scope_authority_owners != context.scope_authority_owners
6723 }
6724 RestartRequiredSection::PrivilegedOperatorPrompt => {
6725 privileged_operator_prompt != context.privileged_operator_prompt
6726 }
6727 };
6728 if changed {
6729 restart_required.push(section.label().to_string());
6730 }
6731 }
6732 if !restart_required.is_empty() {
6733 warn!(
6734 config_path = %context.config_path.display(),
6735 sections = %restart_required.join(", "),
6736 "daemon config changed outside the modules section; restart the daemon to apply those changes"
6737 );
6738 }
6739
6740 for configured in &modules {
6741 if let Err(err) = validate_spec(&configured.module_spec()) {
6742 return Ok(vec![control_error_frame(
6743 &frame,
6744 "invalid_daemon_config",
6745 format!("supervisor rescan rejected daemon config: {err}"),
6746 )?]);
6747 }
6748 }
6749
6750 let configured_capabilities = modules
6751 .iter()
6752 .map(|module| (module.module_id.clone(), module.enabled))
6753 .collect::<Vec<_>>();
6754 let preview_capability_warnings = if preview {
6755 let (_, registrations) = self.runtime_capability_snapshot()?;
6756 let current_modules = self
6757 .supervisor
6758 .list()
6759 .into_iter()
6760 .map(|module| module.module_id().to_string())
6761 .collect::<BTreeSet<_>>();
6762 let resulting_modules = configured_capabilities.clone();
6763 let removed = current_modules
6764 .into_iter()
6765 .filter(|module_id| {
6766 !resulting_modules
6767 .iter()
6768 .any(|(configured_id, _)| configured_id == module_id)
6769 })
6770 .collect::<Vec<_>>();
6771 let disabled = match &mode {
6774 RescanMode::Candidate(_) => resulting_modules
6775 .iter()
6776 .filter(|(_, enabled)| !enabled)
6777 .map(|(module_id, _)| module_id.clone())
6778 .collect::<Vec<_>>(),
6779 _ => Vec::new(),
6780 };
6781 self.capability_evaluator.preview_warnings(
6782 resulting_modules,
6783 &removed,
6784 &disabled,
6785 ®istrations,
6786 )
6787 } else {
6788 Vec::new()
6789 };
6790 let reconciled = match &mode {
6791 RescanMode::Candidate(candidate) => self
6792 .preview_candidate(&context, modules, candidate)
6793 .map(|result| (result, Vec::new())),
6794 _ => {
6795 self.reconcile_supervised_modules(
6796 &context.supervisor,
6797 modules,
6798 preview,
6799 requester.clone(),
6800 )
6801 .await
6802 }
6803 };
6804 let (result, changed) = match reconciled {
6805 Ok(result) => result,
6806 Err(message) => {
6807 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
6808 }
6809 };
6810 if !preview {
6811 if let Some(ledger) = &self.activity {
6812 ledger.module_rescanned(&result.added, &result.removed, requester.clone());
6813 ledger.config_changed(&result.added, &result.removed, &changed, requester);
6814 }
6815 if let Err(code) = self.configure_activity(activity_config).await {
6816 warn!(
6817 code,
6818 "activity retention could not be applied during rescan"
6819 );
6820 }
6821 self.capability_evaluator
6822 .configure(configured_capabilities, reserved_capabilities);
6823 self.capability_evaluator.wake_deadline_loop();
6824 self.refresh_capability_requirements();
6825 }
6826 let mut result = result;
6827 result.restart_required = restart_required;
6828 result.capability_warnings = preview_capability_warnings;
6829 let response = ClientControlResponse::SupervisorRescan { result };
6830 Ok(vec![control_response_body_frame(
6831 &frame,
6832 &response,
6833 "ClientControlResponse::SupervisorRescan",
6834 )?])
6835 }
6836
6837 async fn handle_supervisor_release_reserved(
6838 &self,
6839 frame: Frame,
6840 module_id: String,
6841 ) -> Result<Vec<Frame>, RouterError> {
6842 let Some(context) = self.rescan.clone() else {
6843 return Ok(vec![control_error_frame(
6844 &frame,
6845 "release_unavailable",
6846 "reserved-id release requires a daemon started with a reloadable config path",
6847 )?]);
6848 };
6849 let operation_lock = self.supervisor.operation_lock();
6850 let _operation_guard = operation_lock.lock().await;
6851 let loaded = match crate::daemon_config::load(&context.config_path) {
6852 Ok(Some(config)) => config,
6853 Ok(None) => {
6854 return Ok(vec![control_error_frame(
6855 &frame,
6856 "invalid_daemon_config",
6857 format!(
6858 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
6859 context.config_path.display()
6860 ),
6861 )?])
6862 }
6863 Err(err) => {
6864 return Ok(vec![control_error_frame(
6865 &frame,
6866 "invalid_daemon_config",
6867 format!("unable to verify reserved-id release against daemon config: {err}"),
6868 )?])
6869 }
6870 };
6871 if loaded
6872 .modules
6873 .iter()
6874 .any(|configured| configured.module_id == module_id)
6875 {
6876 return Ok(vec![control_error_frame(
6877 &frame,
6878 "reserved_module_configured",
6879 format!(
6880 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
6881 ),
6882 )?]);
6883 }
6884 if !self.supervisor.release_retained_reserved_gate(&module_id) {
6885 return Ok(vec![control_error_frame(
6886 &frame,
6887 "reserved_gate_not_retained",
6888 format!(
6889 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
6890 ),
6891 )?]);
6892 }
6893
6894 let response = ClientControlResponse::SupervisorAck {
6895 module_id,
6896 applied: true,
6897 };
6898 Ok(vec![control_response_body_frame(
6899 &frame,
6900 &response,
6901 "ClientControlResponse::SupervisorAck",
6902 )?])
6903 }
6904
6905 fn plan_rescan(
6909 &self,
6910 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6911 ) -> Result<RescanPlan, String> {
6912 let mut current = BTreeMap::new();
6913 for module in self.supervisor.list() {
6914 let (spec, health) = module.configuration().map_err(|err| {
6915 format!(
6916 "failed to read configuration for module_id '{}': {err}",
6917 module.module_id()
6918 )
6919 })?;
6920 let enabled = module
6921 .status()
6922 .map_err(|err| {
6923 format!(
6924 "failed to read status for module_id '{}': {err}",
6925 module.module_id()
6926 )
6927 })?
6928 .enabled;
6929 current.insert(
6930 module.module_id().to_string(),
6931 (module, spec, health, enabled),
6932 );
6933 }
6934 let configured = configured_modules
6935 .into_iter()
6936 .map(|module| (module.module_id.clone(), module))
6937 .collect::<BTreeMap<_, _>>();
6938
6939 let added = configured
6940 .keys()
6941 .filter(|module_id| !current.contains_key(*module_id))
6942 .cloned()
6943 .collect::<Vec<_>>();
6944 let removed = current
6945 .keys()
6946 .filter(|module_id| !configured.contains_key(*module_id))
6947 .cloned()
6948 .collect::<Vec<_>>();
6949 let mut changed_pending_reload = Vec::new();
6950 let mut configuration_changes = BTreeSet::new();
6951 let mut enabled_changes = BTreeSet::new();
6952 let mut unchanged = 0_u32;
6953
6954 for (module_id, configured_module) in &configured {
6955 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
6956 else {
6957 continue;
6958 };
6959 let launch_changed = *current_spec != configured_module.module_spec();
6963 let configuration_changed =
6964 launch_changed || *current_health != configured_module.health;
6965 let enabled_changed = *current_enabled != configured_module.enabled
6968 || self
6969 .capability_evaluator
6970 .configured_enabled(module_id)
6971 .is_some_and(|enabled| enabled != configured_module.enabled);
6972 if configuration_changed {
6973 configuration_changes.insert(module_id.clone());
6974 }
6975 if launch_changed {
6976 changed_pending_reload.push(module_id.clone());
6977 }
6978 if enabled_changed {
6979 enabled_changes.insert(module_id.clone());
6980 }
6981 if !configuration_changed && !enabled_changed {
6982 unchanged = unchanged.saturating_add(1);
6983 }
6984 }
6985 Ok(RescanPlan {
6986 current,
6987 configured,
6988 added,
6989 removed,
6990 changed_pending_reload,
6991 configuration_changes,
6992 enabled_changes,
6993 unchanged,
6994 })
6995 }
6996
6997 async fn reconcile_supervised_modules(
7006 &self,
7007 supervisor: &Supervisor,
7008 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
7009 preview: bool,
7010 requester: Principal,
7011 ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
7012 let plan = self.plan_rescan(configured_modules)?;
7013
7014 if preview {
7020 return Ok((plan.preview_result(), Vec::new()));
7021 }
7022 let changed = plan
7023 .configuration_changes
7024 .union(&plan.enabled_changes)
7025 .cloned()
7026 .collect::<Vec<_>>();
7027 let RescanPlan {
7028 current,
7029 configured,
7030 added,
7031 removed,
7032 changed_pending_reload,
7033 configuration_changes,
7034 enabled_changes,
7035 unchanged,
7036 } = plan;
7037 if !(added.is_empty()
7042 && removed.is_empty()
7043 && configuration_changes.is_empty()
7044 && enabled_changes.is_empty())
7045 {
7046 if let Some(context) = &self.rescan {
7047 context
7048 .generation
7049 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
7050 }
7051 }
7052
7053 for module_id in &removed {
7054 let module = ¤t
7055 .get(module_id)
7056 .expect("removed module came from current supervisor state")
7057 .0;
7058 module
7059 .retire_requested(Some(requester.clone()))
7060 .await
7061 .map_err(|err| {
7062 format!("failed to retire module_id '{module_id}' during rescan: {err}")
7063 })?;
7064 self.supervisor.record_rescan_removal(module_id);
7098 self.supervisor.retire(module_id);
7099 self.route_outages.forget(module_id);
7100 }
7101
7102 for module_id in configured.keys() {
7103 let Some((module, _, _, current_enabled)) = current.get(module_id) else {
7104 continue;
7105 };
7106 let configured_module = configured
7107 .get(module_id)
7108 .expect("configured module id came from configured map");
7109 if configuration_changes.contains(module_id) {
7110 module
7111 .update_configuration(
7112 configured_module.module_spec(),
7113 configured_module.health.clone(),
7114 configured_module.drain_timeout_ms,
7115 )
7116 .await
7117 .map_err(|err| {
7118 format!(
7119 "failed to update module_id '{module_id}' configuration during rescan: {err}"
7120 )
7121 })?;
7122 }
7123 if *current_enabled != configured_module.enabled {
7126 self.route_outages.mark_operator_action(module_id);
7129 module
7130 .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
7131 .await
7132 .map_err(|err| {
7133 self.route_outages.operator_action_ended_unrefused(module_id);
7134 format!(
7135 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
7136 configured_module.enabled
7137 )
7138 })?;
7139 }
7140 }
7141
7142 for module_id in &added {
7143 let configured_module = configured
7144 .get(module_id)
7145 .expect("added module id came from configured map");
7146 supervisor
7147 .supervise_configured_requested(
7148 configured_module.module_spec(),
7149 configured_module.enabled,
7150 configured_module.health.clone(),
7151 configured_module.drain_timeout_ms,
7152 configured_module.restart,
7153 configured_module.register_timeout,
7154 Some(requester.clone()),
7155 )
7156 .map_err(|err| {
7157 format!("failed to add module_id '{module_id}' during rescan: {err}")
7158 })?;
7159 }
7160
7161 Ok((
7162 SupervisorRescanResult {
7163 added,
7164 removed,
7165 changed_pending_reload,
7166 enabled_changes: enabled_changes.iter().cloned().collect(),
7167 unchanged,
7168 preview: false,
7169 restart_required: Vec::new(),
7172 capability_warnings: Vec::new(),
7173 module_actions: Vec::new(),
7174 earlier_pending: Vec::new(),
7175 expect: None,
7176 },
7177 changed,
7178 ))
7179 }
7180
7181 async fn handle_supervisor_set_enabled(
7182 &self,
7183 frame: Frame,
7184 module_id: String,
7185 enabled: bool,
7186 requester: Principal,
7187 ) -> Result<Vec<Frame>, RouterError> {
7188 let operation_lock = self.supervisor.operation_lock();
7189 let _operation_guard = operation_lock.lock().await;
7190 let Some(module) = self.supervisor.get(&module_id) else {
7191 return Ok(vec![control_error_frame(
7192 &frame,
7193 "unknown_module",
7194 format!("module_id '{module_id}' is not supervised"),
7195 )?]);
7196 };
7197
7198 self.route_outages.mark_operator_action(&module_id);
7201 let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
7202 Ok(applied) => applied,
7203 Err(err) => {
7204 self.route_outages
7205 .operator_action_ended_unrefused(&module_id);
7206 return Ok(vec![control_error_frame(
7207 &frame,
7208 "target_unavailable",
7209 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
7210 )?]);
7211 }
7212 };
7213 if !applied {
7214 self.route_outages
7217 .operator_action_ended_unrefused(&module_id);
7218 }
7219
7220 self.capability_evaluator.wake_deadline_loop();
7221 self.refresh_capability_requirements();
7222 let response = ClientControlResponse::SupervisorAck { module_id, applied };
7223 Ok(vec![control_response_body_frame(
7224 &frame,
7225 &response,
7226 "ClientControlResponse::SupervisorAck",
7227 )?])
7228 }
7229
7230 async fn handle_supervisor_health_probe(
7231 &self,
7232 ctx: &RouteCtx,
7233 frame: Frame,
7234 module_id: String,
7235 ) -> Result<Vec<Frame>, RouterError> {
7236 self.refresh_capability_requirements();
7237 let Some(registration) = self
7238 .registry
7239 .get_module(&module_id)
7240 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
7241 else {
7242 return Ok(vec![control_error_frame(
7243 &frame,
7244 "unknown_module",
7245 format!("module_id '{module_id}' is not registered"),
7246 )?]);
7247 };
7248
7249 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
7262 {
7263 return Ok(vec![control_error_frame(
7264 &frame,
7265 "health_not_advertised",
7266 format!("module_id '{module_id}' did not advertise health.check"),
7267 )?]);
7268 }
7269
7270 self.observe_control_wait(
7271 ctx,
7272 Some(®istration.manifest.module_id),
7273 ops::SUPERVISOR_HEALTH_PROBE,
7274 async {
7275 let deadline = Instant::now() + self.health_probe_timeout;
7276 let pending = match self.forwarding.begin_module_control_rpc_for(
7277 &module_id,
7278 MODULE_CONTROL_OP_HEALTH_CHECK,
7279 deadline,
7280 ) {
7281 Ok(pending) => pending,
7282 Err(err) => {
7283 return Ok(vec![control_error_frame(
7284 &frame,
7285 forwarding_error_code(&err),
7286 err.to_string(),
7287 )?])
7288 }
7289 };
7290
7291 let PendingModuleControlRpc {
7292 endpoint,
7293 module_sink,
7294 negotiated_ver,
7295 corr: probe_corr,
7296 receiver,
7297 } = pending;
7298 let mut guard =
7299 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
7300 let probe_body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {})
7301 .map_err(|err| {
7302 RouterError::backend(
7303 0,
7304 frame.header.corr,
7305 format!("failed to encode health.check request: {err}"),
7306 )
7307 })?;
7308 let probe_frame = Frame::build_with_version(
7309 negotiated_ver,
7310 FrameType::Request,
7311 control_flags(),
7312 0,
7313 0,
7314 probe_corr,
7315 probe_body,
7316 )
7317 .map_err(RouterError::FrameBuild)?;
7318
7319 match timeout_at(deadline, module_sink.send(probe_frame)).await {
7322 Ok(Ok(())) => {}
7323 Ok(Err(err)) => {
7324 return Ok(vec![control_error_frame(
7325 &frame,
7326 "target_unavailable",
7327 err.to_string(),
7328 )?]);
7329 }
7330 Err(_) => {
7331 return Ok(vec![control_error_frame(
7332 &frame,
7333 "module_timeout",
7334 format!(
7335 "module_id '{module_id}' did not answer health.check within {:?}",
7336 self.health_probe_timeout
7337 ),
7338 )?]);
7339 }
7340 }
7341
7342 match timeout_at(deadline, receiver).await {
7343 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
7344 guard.disarm();
7345 let Some(report) = response.health_report() else {
7346 return Ok(vec![control_error_frame(
7347 &frame,
7348 "invalid_control_body",
7349 "health.check RPC returned a non-health response",
7350 )?]);
7351 };
7352 let HealthReport {
7357 status,
7358 detail,
7359 metrics,
7360 } = report;
7361 let capability_detail = self
7362 .capability_evaluator
7363 .required_problem_detail(&module_id);
7364 let response = ClientControlResponse::SupervisorHealthProbe {
7365 module_id,
7366 status,
7367 detail: append_capability_problem_detail(detail, capability_detail),
7368 metrics,
7369 };
7370 Ok(vec![control_response_body_frame(
7371 &frame,
7372 &response,
7373 "ClientControlResponse::SupervisorHealthProbe",
7374 )?])
7375 }
7376 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
7377 guard.disarm();
7378 Ok(vec![control_relayed_error_body_frame(&frame, body)?])
7383 }
7384 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
7385 guard.disarm();
7386 Ok(vec![control_error_frame(
7387 &frame,
7388 "target_unavailable",
7389 message,
7390 )?])
7391 }
7392 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
7393 guard.disarm();
7394 Ok(vec![control_error_frame(
7395 &frame,
7396 "invalid_control_body",
7397 message,
7398 )?])
7399 }
7400 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
7401 guard.disarm();
7402 Ok(vec![control_error_frame(
7403 &frame,
7404 "invalid_control_body",
7405 format!("expected module-control op '{expected}', got '{actual}'"),
7406 )?])
7407 }
7408 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
7409 guard.disarm();
7410 Ok(vec![control_error_frame(
7411 &frame,
7412 "module_timeout",
7413 format!(
7414 "module_id '{module_id}' answered health.check after {:?}",
7415 self.health_probe_timeout
7416 ),
7417 )?])
7418 }
7419 Ok(Err(_)) => Ok(vec![control_error_frame(
7420 &frame,
7421 "target_unavailable",
7422 "health.check waiter was canceled before the module responded",
7423 )?]),
7424 Err(_) => Ok(vec![control_error_frame(
7425 &frame,
7426 "module_timeout",
7427 format!(
7428 "module_id '{module_id}' did not answer health.check within {:?}",
7429 self.health_probe_timeout
7430 ),
7431 )?]),
7432 }
7433 },
7434 )
7435 .await
7436 }
7437
7438 fn supervisor_status(
7439 &self,
7440 module_id: &str,
7441 corr: u64,
7442 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
7443 self.supervisor
7444 .get(module_id)
7445 .map(|module| {
7446 let warming = module.is_warming_for_control("status").map_err(|err| {
7447 RouterError::backend(
7448 0,
7449 corr,
7450 format!(
7451 "failed to read supervisor warming state for module_id '{module_id}': {err}"
7452 ),
7453 )
7454 })?;
7455 module.status_for_control("status").map_err(|err| {
7456 RouterError::backend(
7457 0,
7458 corr,
7459 format!(
7460 "failed to read supervisor status for module_id '{module_id}': {err}"
7461 ),
7462 )
7463 }).map(|status| (status, warming))
7464 })
7465 .transpose()
7466 }
7467
7468 fn guard_module_control_op(
7469 &self,
7470 frame: &Frame,
7471 module_id: &str,
7472 op: &str,
7473 ) -> Result<Option<Frame>, RouterError> {
7474 if self.module_grants_op(module_id, op, frame.header.corr)? {
7475 return Ok(None);
7476 }
7477
7478 Ok(Some(control_error_frame(
7479 frame,
7480 "op_not_allowed",
7481 format!("module_id '{module_id}' did not grant control op '{op}'"),
7482 )?))
7483 }
7484
7485 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
7486 let Some(registration) = self
7487 .registry
7488 .get_module(module_id)
7489 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
7490 else {
7491 return Ok(false);
7492 };
7493 Ok(module_registration_grants_op(®istration.control_ops, op))
7494 }
7495
7496 fn handle_status_update(
7497 &self,
7498 endpoint: ModuleEndpointId,
7499 frame: Frame,
7500 ) -> Result<Vec<Frame>, RouterError> {
7501 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
7502 Ok(update) => update,
7503 Err(err) => {
7504 if is_known_module_push_op(&frame.body) {
7509 return Ok(vec![control_error_frame(
7510 &frame,
7511 "invalid_control_body",
7512 format!("malformed module control push body: {err}"),
7513 )?]);
7514 }
7515 return Ok(Vec::new());
7516 }
7517 };
7518
7519 match update {
7520 ModuleControlPush::RouteStatus {
7521 route_channel,
7522 route_epoch,
7523 status,
7524 } => {
7525 self.forwarding
7526 .cache_status(endpoint, route_channel, route_epoch, status)
7527 .map_err(RouterError::Forwarding)?;
7528 }
7529 }
7530 Ok(Vec::new())
7531 }
7532
7533 fn handle_route_poll(
7534 &self,
7535 ctx: &RouteCtx,
7536 frame: Frame,
7537 route_channel: u16,
7538 route_epoch: u32,
7539 kind: PollKind,
7540 ) -> Result<Vec<Frame>, RouterError> {
7541 let snapshot = self
7542 .forwarding
7543 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
7544 .map_err(RouterError::Forwarding)?;
7545 let response = match (kind, snapshot) {
7546 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
7547 ClientControlResponse::RoutePoll {
7548 route_channel,
7549 route_epoch,
7550 status,
7551 live: None,
7552 }
7553 }
7554 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
7555 route_channel,
7556 route_epoch,
7557 status: None,
7558 live: None,
7559 },
7560 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
7561 let live = self
7587 .process_liveness
7588 .as_ref()
7589 .and_then(|source| source.process_live(&module_id))
7590 .unwrap_or(true);
7591 ClientControlResponse::RoutePoll {
7592 route_channel,
7593 route_epoch,
7594 status: None,
7595 live: Some(live),
7596 }
7597 }
7598 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
7599 route_channel,
7600 route_epoch,
7601 status: None,
7602 live: Some(false),
7603 },
7604 };
7605
7606 Ok(vec![control_response_body_frame(
7607 &frame,
7608 &response,
7609 "ClientControlResponse::RoutePoll",
7610 )?])
7611 }
7612
7613 pub(crate) fn observe_module_control_completion(
7614 &self,
7615 completion: ModuleControlRpcCompletion,
7616 ) -> bool {
7617 match completion {
7618 ModuleControlRpcCompletion::Unknown => false,
7619 ModuleControlRpcCompletion::Settled => true,
7620 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
7621 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
7622 info!(
7623 module_id = %module_id,
7624 latency_ms,
7625 "late health.check answer proves the module is alive"
7626 );
7627 match self
7628 .supervisor
7629 .record_late_health_answer(&module_id, latency_ms)
7630 {
7631 Ok(true) => {}
7632 Ok(false) => debug!(
7633 module_id = %module_id,
7634 latency_ms,
7635 "late health.check answer has no active supervisor snapshot"
7636 ),
7637 Err(err) => warn!(
7638 module_id = %module_id,
7639 latency_ms,
7640 error = %err,
7641 "failed to record late health.check answer"
7642 ),
7643 }
7644 true
7645 }
7646 }
7647 }
7648
7649 fn refuse_to_end_module_connection_for_a_client(
7670 &self,
7671 module_connection_id: ConnectionId,
7672 corr: u64,
7673 err: ForwardingError,
7674 ) -> Result<(), RouterError> {
7675 if let ForwardingError::ConnectionClosing { connection_id } = err {
7676 if connection_id != module_connection_id {
7677 warn!(
7678 module_connection_id = module_connection_id.get(),
7679 client_connection_id = connection_id.get(),
7680 corr,
7681 "dropping a route.bind response for a closing client; the module connection keeps serving"
7682 );
7683 return Ok(());
7684 }
7685 }
7686 Err(RouterError::Forwarding(err))
7687 }
7688
7689 fn handle_module_relay_response(
7690 &self,
7691 connection_id: ConnectionId,
7692 frame: Frame,
7693 ) -> Result<Vec<Frame>, RouterError> {
7694 let mut secondary_error = None;
7695 let outcome = match frame.header.ty {
7696 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
7697 Ok(probe) if probe.op == "route.bind" => {
7698 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
7699 Ok(ModuleControlResponse::RouteBindAck {}) => {
7700 RouteBindRelayOutcome::Accepted
7701 }
7702 Ok(other) => {
7703 let message =
7704 format!("route.bind response carried unexpected body: {other:?}");
7705 secondary_error = Some(control_error_frame(
7706 &frame,
7707 "invalid_control_body",
7708 message.clone(),
7709 )?);
7710 RouteBindRelayOutcome::ModuleGone(message)
7711 }
7712 Err(err) => {
7713 let message = format!("malformed route.bind response body: {err}");
7714 secondary_error = Some(control_error_frame(
7715 &frame,
7716 "invalid_control_body",
7717 message.clone(),
7718 )?);
7719 RouteBindRelayOutcome::ModuleGone(message)
7720 }
7721 }
7722 }
7723 Ok(probe) => {
7724 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
7725 {
7726 Ok(response) => ModuleControlRpcOutcome::Response(response),
7727 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7728 "malformed {} response body: {err}",
7729 probe.op
7730 )),
7731 };
7732 let completion = self
7733 .forwarding
7734 .complete_module_control_rpc(
7735 connection_id,
7736 frame.header.corr,
7737 Some(&probe.op),
7738 outcome,
7739 )
7740 .map_err(RouterError::Forwarding)?;
7741 if !self.observe_module_control_completion(completion) {
7742 debug!(
7743 connection_id = connection_id.get(),
7744 corr = frame.header.corr,
7745 op = %probe.op,
7746 "dropping late or unknown module-control RPC response"
7747 );
7748 }
7749 return Ok(Vec::new());
7750 }
7751 Err(err) => {
7752 if let Some(expected_op) = self
7753 .forwarding
7754 .pending_module_control_op(connection_id, frame.header.corr)
7755 .map_err(RouterError::Forwarding)?
7756 {
7757 let completion = self
7758 .forwarding
7759 .complete_module_control_rpc(
7760 connection_id,
7761 frame.header.corr,
7762 None,
7763 ModuleControlRpcOutcome::MalformedResponse(format!(
7764 "malformed {expected_op} response body: {err}"
7765 )),
7766 )
7767 .map_err(RouterError::Forwarding)?;
7768 if !self.observe_module_control_completion(completion) {
7769 debug!(
7770 connection_id = connection_id.get(),
7771 corr = frame.header.corr,
7772 "dropping late malformed module-control RPC response"
7773 );
7774 }
7775 return Ok(Vec::new());
7776 }
7777 let message = format!("malformed route.bind response body: {err}");
7778 secondary_error = Some(control_error_frame(
7779 &frame,
7780 "invalid_control_body",
7781 message.clone(),
7782 )?);
7783 RouteBindRelayOutcome::ModuleGone(message)
7784 }
7785 },
7786 FrameType::Error => {
7787 if self
7788 .forwarding
7789 .pending_module_control_op(connection_id, frame.header.corr)
7790 .map_err(RouterError::Forwarding)?
7791 .is_some()
7792 {
7793 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
7794 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
7795 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7796 "malformed module-control ERROR body: {err}"
7797 )),
7798 };
7799 let completion = self
7800 .forwarding
7801 .complete_module_control_rpc(
7802 connection_id,
7803 frame.header.corr,
7804 None,
7805 outcome,
7806 )
7807 .map_err(RouterError::Forwarding)?;
7808 if !self.observe_module_control_completion(completion) {
7809 debug!(
7810 connection_id = connection_id.get(),
7811 corr = frame.header.corr,
7812 "dropping late or unknown module-control RPC error"
7813 );
7814 }
7815 return Ok(Vec::new());
7816 }
7817 match serde_json::from_slice::<ErrorBody>(&frame.body) {
7818 Ok(body) => RouteBindRelayOutcome::Rejected(body),
7819 Err(err) => {
7820 let message = format!("malformed route.bind ERROR body: {err}");
7821 secondary_error = Some(control_error_frame(
7822 &frame,
7823 "invalid_control_body",
7824 message.clone(),
7825 )?);
7826 RouteBindRelayOutcome::ModuleGone(message)
7827 }
7828 }
7829 }
7830 ty => {
7831 return Ok(vec![control_error_frame(
7832 &frame,
7833 "unsupported_control_frame",
7834 format!("unsupported module channel-0 frame {ty:?}"),
7835 )?])
7836 }
7837 };
7838
7839 let settled =
7840 self.forwarding
7841 .complete_pending_relay(connection_id, frame.header.corr, outcome);
7842 let completion = match settled {
7843 Ok(completion) => completion,
7844 Err(err) => {
7845 self.refuse_to_end_module_connection_for_a_client(
7846 connection_id,
7847 frame.header.corr,
7848 err,
7849 )?;
7850 return Ok(secondary_error.into_iter().collect());
7851 }
7852 };
7853 if let Some(target) = completion.abandoned.as_ref() {
7854 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
7855 }
7856 if !completion.settled {
7857 debug!(
7858 connection_id = connection_id.get(),
7859 corr = frame.header.corr,
7860 frame_type = ?frame.header.ty,
7861 "dropping late or unknown route.bind relay response"
7862 );
7863 }
7864 Ok(secondary_error.into_iter().collect())
7865 }
7866
7867 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
7868 debug!(connection_id = connection_id.get(), "handling GOODBYE");
7869 self.cleanup_connection_with_end_reason(
7873 connection_id,
7874 RegistrationEndReason::ExplicitGoodbye,
7875 )
7876 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
7877 Ok(Vec::new())
7878 }
7879}
7880
7881impl Default for ControlHandler {
7882 fn default() -> Self {
7883 Self::new(Arc::new(Registry::default()))
7884 }
7885}
7886
7887impl crate::supervise::SwapPromotionObserver for ControlHandler {
7888 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
7889 self.apply_registration_capabilities(registration);
7890 }
7891}
7892
7893fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
7894 CapabilityRequirementStatus {
7895 consumer: status.consumer,
7896 capability: status.capability,
7897 need: match status.need {
7898 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
7899 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
7900 },
7901 verdict: status.verdict.as_str().to_string(),
7902 episode_seq: status.episode_seq,
7903 config_satisfiable: status.config_satisfiable,
7904 runtime_available: status.runtime_available,
7905 detail: status.detail,
7906 }
7907}
7908
7909fn append_capability_problem_detail(
7910 detail: Option<String>,
7911 capability_detail: Option<String>,
7912) -> Option<String> {
7913 match (detail, capability_detail) {
7914 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
7915 (Some(detail), None) => Some(detail),
7916 (None, Some(capability_detail)) => Some(capability_detail),
7917 (None, None) => None,
7918 }
7919}
7920
7921fn subc_ops() -> Vec<String> {
7922 SUBC_CONTROL_OPS
7923 .iter()
7924 .map(|op| (*op).to_string())
7925 .collect()
7926}
7927
7928fn module_subc_ops() -> Vec<String> {
7929 SUBC_CONTROL_OPS
7930 .iter()
7931 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
7932 .map(|op| (*op).to_string())
7933 .collect()
7934}
7935
7936#[cfg(test)]
7937fn module_baseline_control_ops() -> Vec<String> {
7938 MODULE_BASELINE_CONTROL_OPS
7939 .iter()
7940 .map(|op| (*op).to_string())
7941 .collect()
7942}
7943
7944fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
7945 let mut seen = HashSet::new();
7946 let mut effective = Vec::new();
7947 for op in MODULE_BASELINE_CONTROL_OPS {
7948 if seen.insert((*op).to_string()) {
7949 effective.push((*op).to_string());
7950 }
7951 }
7952 for op in declared.unwrap_or_default() {
7953 if seen.insert(op.clone()) {
7954 effective.push(op);
7955 }
7956 }
7957 effective
7958}
7959
7960fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
7961 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
7962}
7963
7964fn target_module_id(target: &RouteTarget) -> &str {
7965 match target {
7966 RouteTarget::ToolProvider { module_id }
7967 | RouteTarget::ManagementSurface { module_id }
7968 | RouteTarget::InternalService { module_id, .. } => module_id,
7969 }
7970}
7971
7972fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
7973 roles.iter().any(|role| match (target, role) {
7974 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
7975 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
7976 (
7977 RouteTarget::InternalService { service_id, .. },
7978 ProviderRole::InternalService {
7979 service_id: provided,
7980 ..
7981 },
7982 ) => service_id == provided,
7983 _ => false,
7984 })
7985}
7986
7987fn is_routable_role(role: &ProviderRole) -> bool {
7988 matches!(
7989 role,
7990 ProviderRole::ToolProvider { .. }
7991 | ProviderRole::ManagementSurface { .. }
7992 | ProviderRole::InternalService { .. }
7993 )
7994}
7995
7996#[derive(Debug, Clone, Copy, PartialEq, Eq)]
7997enum ControlRequestBodyError {
7998 UnknownOp,
7999 InvalidBody,
8000}
8001
8002#[derive(Debug, Deserialize)]
8003struct ControlOpProbe {
8004 op: String,
8005}
8006
8007const MODULE_PUSH_OPS: &[&str] = &["route.status"];
8010
8011fn is_known_module_push_op(body: &[u8]) -> bool {
8012 serde_json::from_slice::<ControlOpProbe>(body)
8013 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
8014 .unwrap_or(false)
8015}
8016
8017fn is_known_module_request_op(body: &[u8]) -> bool {
8018 serde_json::from_slice::<ControlOpProbe>(body)
8019 .map(|probe| is_module_to_subc_op(&probe.op))
8020 .unwrap_or(false)
8021}
8022
8023fn is_module_to_subc_op(op: &str) -> bool {
8024 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
8025}
8026
8027fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
8028 debug!(
8029 op = %op,
8030 connection_id = connection_id.get(),
8031 corr,
8032 "control dispatch"
8033 );
8034}
8035
8036fn log_slow_control_dispatch(
8037 dispatch_started_at: Option<StdInstant>,
8038 op: &'static str,
8039 connection_id: ConnectionId,
8040 corr: u64,
8041 route_open_timings: Option<&RouteOpenTimings>,
8042) {
8043 let Some(dispatch_started_at) = dispatch_started_at else {
8044 return;
8045 };
8046 let elapsed = Instant::from_std(dispatch_started_at).elapsed();
8048 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
8049 if let Some(timings) = route_open_timings {
8050 warn!(
8051 op = %op,
8052 connection_id = connection_id.get(),
8053 corr,
8054 elapsed_ms = elapsed.as_millis() as u64,
8055 module_id = ?timings.module_id,
8056 principal = %timings.principal,
8057 admission_ms = timings.admission.as_millis() as u64,
8058 project_root_ms = timings.project_root.as_millis() as u64,
8059 egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
8060 bind_send_ms = timings.bind_send.as_millis() as u64,
8061 bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
8062 "slow control dispatch"
8063 );
8064 return;
8065 }
8066 warn!(
8067 op = %op,
8068 connection_id = connection_id.get(),
8069 corr,
8070 elapsed_ms = elapsed.as_millis() as u64,
8071 "slow control dispatch"
8072 );
8073 }
8074}
8075
8076#[cfg(test)]
8078type TestCallerLookup =
8079 Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
8080
8081fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
8083 match request {
8084 ClientControlRequest::SupervisorRestart { module_id, .. } => {
8085 Some(PrivilegedOperation::Restart { module_id })
8086 }
8087 ClientControlRequest::SupervisorSwap { module_id, .. } => {
8088 Some(PrivilegedOperation::Swap { module_id })
8089 }
8090 ClientControlRequest::SupervisorReload { module_id } => {
8091 Some(PrivilegedOperation::Reload { module_id })
8092 }
8093 ClientControlRequest::SupervisorRescan { preview } => {
8094 if *preview {
8095 None
8096 } else {
8097 Some(PrivilegedOperation::Rescan)
8098 }
8099 }
8100 ClientControlRequest::SupervisorRescanChecked {
8105 preview,
8106 candidate,
8107 expect,
8108 } => (!*preview && candidate.is_none() && expect.is_some())
8109 .then_some(PrivilegedOperation::Rescan),
8110 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
8111 Some(PrivilegedOperation::SetEnabled {
8112 module_id,
8113 enabled: *enabled,
8114 })
8115 }
8116 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
8117 Some(PrivilegedOperation::ReleaseReserved { module_id })
8118 }
8119 ClientControlRequest::ServerDescribe {}
8120 | ClientControlRequest::DaemonShutdownIntent { .. }
8123 | ClientControlRequest::CatalogList { .. }
8124 | ClientControlRequest::RouteOpen { .. }
8125 | ClientControlRequest::RoutePoll { .. }
8126 | ClientControlRequest::SupervisorList {}
8127 | ClientControlRequest::SupervisorHealth {}
8128 | ClientControlRequest::SupervisorHealthProbe { .. }
8129 | ClientControlRequest::SupervisorRoutes { .. }
8130 | ClientControlRequest::SupervisorProvenance { .. }
8131 | ClientControlRequest::SupervisorStderrTail { .. }
8132 | ClientControlRequest::SupervisorTerminals { .. }
8133 | ClientControlRequest::SupervisorSpawnSnapshot {}
8134 | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
8135 }
8136}
8137
8138fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
8139 match request {
8140 ClientControlRequest::DaemonShutdownIntent { .. } => ops::DAEMON_SHUTDOWN_INTENT,
8141 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
8142 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
8143 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
8144 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
8145 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
8146 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
8147 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
8148 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
8149 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
8150 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
8151 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
8152 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
8153 ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
8154 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
8155 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
8156 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
8157 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
8158 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
8159 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
8160 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
8161 }
8162}
8163
8164struct RouteOpenCaller {
8172 principal: String,
8176 scope: String,
8179 harness: String,
8182}
8183
8184impl RouteOpenCaller {
8185 fn new(
8186 principal: Option<&Principal>,
8187 scope: Option<&ScopeSelector>,
8188 identity: &BindIdentity,
8189 ) -> Self {
8190 Self {
8191 principal: principal.map_or_else(|| "unattested".to_string(), principal_label),
8192 scope: scope.map_or_else(|| "none".to_string(), route_open_requested_scope_label),
8193 harness: bounded_log_text(&identity.harness),
8194 }
8195 }
8196
8197 fn unknown() -> Self {
8200 Self {
8201 principal: "unknown".to_string(),
8202 scope: "unknown".to_string(),
8203 harness: "unknown".to_string(),
8204 }
8205 }
8206}
8207
8208const MAX_LOGGED_TEXT_CHARS: usize = 64;
8211
8212fn bounded_log_text(value: &str) -> String {
8216 let mut bounded: String = value.chars().take(MAX_LOGGED_TEXT_CHARS).collect();
8217 if value.chars().nth(MAX_LOGGED_TEXT_CHARS).is_some() {
8218 bounded.push('…');
8219 }
8220 format!("{bounded:?}")
8221}
8222
8223fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
8229 use subc_protocol::scope::ScopeKind;
8230 let kind = match stamp.kind {
8231 ScopeKind::Head => "head",
8232 ScopeKind::Worker => "worker",
8233 ScopeKind::Ephemeral => "ephemeral",
8234 };
8235 format!(
8236 "{kind}/{}/{}@{}",
8237 principal_label(&stamp.owner),
8238 bounded_log_text(&stamp.scope_ref),
8239 stamp.scope_epoch
8240 )
8241}
8242
8243fn route_open_requested_scope_label(selector: &ScopeSelector) -> String {
8250 let epoch = selector
8251 .scope_epoch
8252 .map_or_else(|| "none".to_string(), |epoch| epoch.to_string());
8253 format!(
8254 "requested/{}/{}@{epoch}",
8255 bounded_log_text(&principal_label(&selector.owner)),
8256 bounded_log_text(&selector.scope_ref),
8257 )
8258}
8259
8260fn principal_label(principal: &Principal) -> String {
8261 match principal {
8262 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
8263 Principal::Direct => "direct".to_string(),
8264 other => format!("{other:?}"),
8265 }
8266}
8267
8268fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
8269 match request {
8270 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
8271 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
8272 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
8273 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
8274 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
8275 ModuleControlRequestFromModule::AgentCommands { .. } => "supervisor.agent_commands",
8276 _ => "unsupported",
8277 }
8278}
8279
8280fn parse_client_control_request(
8281 body: &[u8],
8282) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
8283 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
8284 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
8285 Ok(probe)
8286 if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
8287 || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
8288 {
8289 ControlRequestBodyError::InvalidBody
8290 }
8291 Ok(_) => ControlRequestBodyError::UnknownOp,
8292 Err(_) => ControlRequestBodyError::InvalidBody,
8293 };
8294 (err, classification)
8295 })
8296}
8297
8298fn parse_module_control_request_from_module(
8299 body: &[u8],
8300) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
8301 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
8302 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
8303 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
8304 Ok(_) => ControlRequestBodyError::UnknownOp,
8305 Err(_) => ControlRequestBodyError::InvalidBody,
8306 };
8307 (err, classification)
8308 })
8309}
8310
8311#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
8312enum ProviderRoleKind {
8313 ToolProvider,
8314 PipelineStage,
8315 ManagementSurface,
8316 InternalService,
8317}
8318
8319fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
8320 match role {
8321 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
8322 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
8323 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
8324 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
8325 }
8326}
8327
8328fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
8329 roles.iter().map(provider_role_kind).collect()
8330}
8331
8332const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
8335
8336#[derive(Debug, Default, PartialEq, Eq)]
8338struct ScopeOutcomeCounts {
8339 created: usize,
8340 replaced: usize,
8341 updated: usize,
8342 unchanged: usize,
8343 refused: usize,
8344}
8345
8346impl ScopeOutcomeCounts {
8347 fn of(results: &[ScopeRecordResult]) -> Self {
8348 let mut counts = Self::default();
8349 for result in results {
8350 let slot = match result.outcome {
8351 ScopeRecordOutcome::Created => &mut counts.created,
8352 ScopeRecordOutcome::Replaced => &mut counts.replaced,
8353 ScopeRecordOutcome::Updated => &mut counts.updated,
8354 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
8355 ScopeRecordOutcome::Refused => &mut counts.refused,
8356 };
8357 *slot += 1;
8358 }
8359 counts
8360 }
8361}
8362
8363#[cfg(test)]
8364mod scope_outcome_count_tests {
8365 use super::*;
8366
8367 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
8368 ScopeRecordResult {
8369 scope_ref: "r".to_string(),
8370 scope_epoch: 1,
8371 outcome,
8372 code: None,
8373 message: None,
8374 version: None,
8375 parent_state: None,
8376 }
8377 }
8378
8379 #[test]
8382 fn every_outcome_is_counted_in_its_own_field() {
8383 let results = [
8384 result(ScopeRecordOutcome::Created),
8385 result(ScopeRecordOutcome::Created),
8386 result(ScopeRecordOutcome::Replaced),
8387 result(ScopeRecordOutcome::Updated),
8388 result(ScopeRecordOutcome::Unchanged),
8389 result(ScopeRecordOutcome::Refused),
8390 result(ScopeRecordOutcome::Refused),
8391 result(ScopeRecordOutcome::Refused),
8392 ];
8393 assert_eq!(
8394 ScopeOutcomeCounts::of(&results),
8395 ScopeOutcomeCounts {
8396 created: 2,
8397 replaced: 1,
8398 updated: 1,
8399 unchanged: 1,
8400 refused: 3,
8401 }
8402 );
8403 }
8404}
8405
8406fn capability_census_trigger(
8410 old: Option<&CapabilityDeclarations>,
8411 new: Option<&CapabilityDeclarations>,
8412) -> bool {
8413 let old_provides = old
8414 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
8415 .unwrap_or_default();
8416 let old_denies = old
8417 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
8418 .unwrap_or_default();
8419 let new = new.cloned().unwrap_or(CapabilityDeclarations {
8420 provides: Vec::new(),
8421 requires: Vec::new(),
8422 must_never_reach: Vec::new(),
8423 });
8424
8425 new.provides
8426 .iter()
8427 .any(|capability| !old_provides.contains(capability))
8428 || new
8429 .must_never_reach
8430 .iter()
8431 .any(|capability| !old_denies.contains(capability))
8432}
8433
8434fn denied_capability<'a>(
8437 opening_manifest: &'a ModuleManifest,
8438 target_manifest: &ModuleManifest,
8439) -> Option<&'a str> {
8440 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
8441 let target_capabilities = target_manifest.capabilities.as_ref()?;
8442 opening_capabilities
8443 .must_never_reach
8444 .iter()
8445 .find(|denied| {
8446 target_capabilities
8447 .provides
8448 .iter()
8449 .any(|provided| provided == *denied)
8450 })
8451 .map(String::as_str)
8452}
8453
8454fn catalog_update_frozen_field_message(
8455 registered: &ModuleManifest,
8456 provides: &[ProviderRole],
8457) -> Option<String> {
8458 let old_has_provides = !registered.provides.is_empty();
8459 let new_has_provides = !provides.is_empty();
8460 if old_has_provides != new_has_provides {
8461 return Some(format!(
8462 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
8463 registered.module_id
8464 ));
8465 }
8466
8467 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
8468 return Some(format!(
8469 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
8470 registered.module_id
8471 ));
8472 }
8473
8474 let registered_concurrency = manifest_concurrency(registered);
8475 let mut candidate = registered.clone();
8476 candidate.provides = provides.to_vec();
8477 let candidate_concurrency = manifest_concurrency(&candidate);
8478 if candidate_concurrency != registered_concurrency {
8479 return Some(format!(
8480 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
8481 registered.module_id, registered_concurrency, candidate_concurrency
8482 ));
8483 }
8484
8485 None
8488}
8489
8490fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
8491 manifest.provides.iter().any(is_routable_role)
8492}
8493
8494fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
8500 manifest
8501 .provides
8502 .iter()
8503 .find_map(|provider| match provider {
8504 ProviderRole::ToolProvider { concurrency, .. }
8505 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
8506 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
8507 })
8508 .unwrap_or(Concurrency::ModuleManaged)
8509}
8510
8511fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
8518 let has_management_surface = manifest
8519 .provides
8520 .iter()
8521 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
8522 if !has_management_surface {
8523 return false;
8524 }
8525 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
8526 return false;
8527 };
8528 let Some(provides) = raw
8529 .get("manifest")
8530 .and_then(|manifest| manifest.get("provides"))
8531 .and_then(serde_json::Value::as_array)
8532 else {
8533 return false;
8534 };
8535 provides.iter().any(|role| {
8540 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
8541 && role.get("concurrency").is_none()
8542 })
8543}
8544
8545fn negotiate_version(peer_version: u8) -> Result<u8, String> {
8546 if peer_version != PROTOCOL_VERSION {
8547 return Err(format!(
8548 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
8549 ));
8550 }
8551 Ok(PROTOCOL_VERSION)
8552}
8553
8554fn pong(frame: &Frame) -> Result<Frame, RouterError> {
8555 Frame::build_with_version(
8556 response_version(frame),
8557 FrameType::Pong,
8558 frame.header.flags,
8559 0,
8560 0,
8561 frame.header.corr,
8562 Vec::new(),
8563 )
8564 .map_err(RouterError::FrameBuild)
8565}
8566
8567pub(crate) fn control_error_frame(
8568 frame: &Frame,
8569 code: &'static str,
8570 message: impl Into<String>,
8571) -> Result<Frame, RouterError> {
8572 control_error_body_frame(
8573 frame,
8574 ErrorBody {
8575 code: code.to_string(),
8576 message: message.into(),
8577 detail: None,
8578 },
8579 )
8580}
8581
8582fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
8590 control_error_frame_with_flags(frame, error, daemon_error_flags())
8591}
8592
8593fn control_relayed_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
8599 control_error_frame_with_flags(frame, error, control_flags())
8600}
8601
8602fn control_error_frame_with_flags(
8603 frame: &Frame,
8604 error: ErrorBody,
8605 flags: Flags,
8606) -> Result<Frame, RouterError> {
8607 let body = serde_json::to_vec(&error).map_err(|err| {
8608 RouterError::backend(
8609 0,
8610 frame.header.corr,
8611 format!("failed to encode control ERROR: {err}"),
8612 )
8613 })?;
8614
8615 Frame::build_with_version(
8616 response_version(frame),
8617 FrameType::Error,
8618 flags,
8619 0,
8620 0,
8621 frame.header.corr,
8622 body,
8623 )
8624 .map_err(RouterError::FrameBuild)
8625}
8626
8627pub(crate) fn control_response_body_frame<T: Serialize>(
8628 frame: &Frame,
8629 reply: &T,
8630 label: &'static str,
8631) -> Result<Frame, RouterError> {
8632 let body = serde_json::to_vec(reply).map_err(|err| {
8633 RouterError::backend(
8634 0,
8635 frame.header.corr,
8636 format!("failed to encode {label}: {err}"),
8637 )
8638 })?;
8639
8640 Frame::build_with_version(
8641 response_version(frame),
8642 FrameType::Response,
8643 control_flags(),
8644 0,
8645 0,
8646 frame.header.corr,
8647 body,
8648 )
8649 .map_err(RouterError::FrameBuild)
8650}
8651
8652fn forwarding_error_code(err: &ForwardingError) -> &'static str {
8682 match err {
8683 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
8684 ForwardingError::NoModuleConnection => "target_unavailable",
8685 ForwardingError::ModuleReloading { .. } => "module_reloading",
8686 ForwardingError::ClientRouteChannelExhausted { .. }
8687 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
8688 ForwardingError::StaleModuleEndpoint
8689 | ForwardingError::UnknownReservation { .. }
8690 | ForwardingError::ConnectionClosing { .. }
8691 | ForwardingError::ClientEgressClosed { .. }
8692 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
8693 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
8696 ForwardingError::RelayCorrelationExhausted
8697 | ForwardingError::RouteOpenBuild(_)
8698 | ForwardingError::Poisoned => "forwarding_error",
8699 }
8700}
8701
8702fn response_version(frame: &Frame) -> u8 {
8703 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
8704 frame.header.ver
8705 } else {
8706 PROTOCOL_VERSION
8707 }
8708}
8709
8710fn control_flags() -> Flags {
8711 Flags::new(false, Priority::Passive, false)
8712}
8713
8714fn send_goodbye_target_best_effort(
8719 counters: &DaemonCounters,
8720 target: &GoodbyeTarget,
8721 context: &'static str,
8722) {
8723 let Ok(frame) = Frame::build_with_version(
8724 target.negotiated_ver,
8725 FrameType::Goodbye,
8726 control_flags(),
8727 target.channel,
8728 target.epoch,
8729 0,
8730 Vec::new(),
8731 ) else {
8732 return;
8733 };
8734 crate::forwarding::send_module_route_goodbye(
8735 counters,
8736 &target.sink,
8737 frame,
8738 target.module_id.as_deref(),
8739 context,
8740 );
8741}
8742
8743pub(crate) fn send_route_control_pushes(
8744 forwarding: &ForwardingTable,
8745 routes: Vec<EndpointRoute>,
8746 push: ClientControlPush,
8747) {
8748 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
8749 for route in routes {
8750 let target = route.goodbye_target;
8751 if let Some((existing, channels)) = targets
8752 .iter_mut()
8753 .find(|(existing, _)| existing.connection_id == target.connection_id)
8754 {
8755 debug_assert_eq!(
8756 existing.negotiated_ver, target.negotiated_ver,
8757 "one connection cannot negotiate multiple frame versions"
8758 );
8759 if !channels.contains(&target.channel) {
8760 channels.push(target.channel);
8761 }
8762 continue;
8763 }
8764 let channel = target.channel;
8765 targets.push((target, vec![channel]));
8766 }
8767 for (target, mut channels) in targets {
8768 channels.sort_unstable();
8769 let mut push = push.clone();
8770 match &mut push {
8771 ClientControlPush::RouteClosing {
8772 channels: covered, ..
8773 }
8774 | ClientControlPush::RouteClosed {
8775 channels: covered, ..
8776 } => *covered = channels,
8777 }
8778 let body = match serde_json::to_vec(&push) {
8779 Ok(body) => body,
8780 Err(err) => {
8781 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
8782 continue;
8783 }
8784 };
8785 let frame = match Frame::build_with_version(
8786 target.negotiated_ver,
8787 FrameType::Push,
8788 control_flags(),
8789 0,
8790 0,
8791 0,
8792 body.clone(),
8793 ) {
8794 Ok(frame) => frame,
8795 Err(err) => {
8796 warn!(
8797 route_channel = target.channel,
8798 error = %err,
8799 "failed to build route lifecycle control PUSH frame"
8800 );
8801 continue;
8802 }
8803 };
8804 if let Err(err) = target.sink.try_send(frame) {
8805 if target.close_on_delivery_failure() {
8806 warn!(
8807 target_connection_id = target.connection_id.get(),
8808 route_channel = target.channel,
8809 error = %err,
8810 "route lifecycle control PUSH was not delivered to client; closing target connection"
8811 );
8812 let _ = forwarding.escalate_client_delivery_failure(
8813 target.connection_id,
8814 target.channel,
8815 target.epoch,
8816 CloseReason::new(
8817 "route_lifecycle_push_delivery_failed",
8818 format!(
8819 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
8820 target.channel
8821 ),
8822 ),
8823 crate::forwarding::UndeliveredFrame {
8824 module_id: target.module_id.as_deref(),
8825 sink: &target.sink,
8826 },
8827 );
8828 }
8829 }
8830 }
8831}
8832
8833#[cfg(test)]
8834mod tests {
8835 include!("control/agent_commands_tests.rs");
8836 include!("control/resource_revision_tests.rs");
8837 #[tokio::test]
8838 async fn route_busy_wait_attribution_resolves_targets_instead_of_trusting_selectors() {
8839 let registry = Arc::new(Registry::default());
8840 let handler = ControlHandler::new(registry);
8841 assert_eq!(
8842 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
8843 None
8844 );
8845 let (ctx, mut rx) = route_ctx(ConnectionId::new(901));
8846 hello_via_sink(
8847 &handler,
8848 &ctx,
8849 &mut rx,
8850 hello_frame("daemon", PROTOCOL_VERSION, 1),
8851 )
8852 .await;
8853 assert_eq!(
8854 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
8855 Some("daemon".into())
8856 );
8857 assert_eq!(
8858 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("missing")),
8859 None
8860 );
8861 assert_eq!(
8862 handler.resolve_reader_wait_target(ops::SUPERVISOR_TERMINALS, Some("daemon")),
8863 None,
8864 "a registered module is not necessarily supervised"
8865 );
8866 assert_eq!(
8867 handler.resolve_reader_wait_target(ops::SUPERVISOR_PROVENANCE, None),
8868 None,
8869 "aggregate provenance has no single target"
8870 );
8871 assert_eq!(
8872 handler.resolve_reader_wait_target(ops::APP_OPEN, Some("daemon")),
8873 None
8874 );
8875 }
8876
8877 mod peer_manifest {
8878 include!("peer_manifest_tests.rs");
8879 }
8880 mod activity_read {
8881 include!("activity_read_tests.rs");
8882 }
8883
8884 mod lifecycle_activity {
8885 include!("control_lifecycle_tests.rs");
8886 }
8887 #[cfg(unix)]
8888 #[tokio::test]
8889 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
8890 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
8891 let path = dir.join("subc.jsonc");
8892 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
8893 "program":"/bin/sleep","args":["60"],"protocol":"none",
8894 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
8895 }}}).to_string()).unwrap();
8896 let mut configured = crate::daemon_config::load(&path)
8897 .unwrap()
8898 .unwrap()
8899 .modules
8900 .pop()
8901 .unwrap();
8902 let registry = std::sync::Arc::new(crate::Registry::default());
8903 let handle = crate::SupervisorHandle::new();
8904 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
8905 .with_handle(handle.clone());
8906 let module = supervisor
8907 .supervise_configured_with_health(
8908 configured.module_spec(),
8909 true,
8910 configured.health.clone(),
8911 None,
8912 configured.restart,
8913 )
8914 .unwrap();
8915 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
8916 let before = module.status().unwrap().pid;
8917 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
8918 configured.health.cadence = std::time::Duration::from_secs(3600);
8919 let health_only = handler
8920 .reconcile_supervised_modules(
8921 &supervisor,
8922 vec![configured.clone()],
8923 false,
8924 Principal::Direct,
8925 )
8926 .await
8927 .unwrap();
8928 assert!(
8929 health_only.0.changed_pending_reload.is_empty(),
8930 "health policy is already applied live"
8931 );
8932 assert_eq!(module.status().unwrap().pid, before);
8933 assert_eq!(
8934 module.configuration().unwrap().1.http,
8935 configured.health.http
8936 );
8937 configured.args = vec!["61".into()];
8938 let launch = handler
8939 .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
8940 .await
8941 .unwrap();
8942 assert_eq!(launch.0.changed_pending_reload, ["stock"]);
8943 assert_eq!(
8944 module.status().unwrap().pid,
8945 before,
8946 "a launch edit is stored until reload"
8947 );
8948 module.drain().await.unwrap();
8949 }
8950 use cortexkit_test_support::ScratchDir;
8951 use std::{
8952 collections::BTreeMap,
8953 fmt,
8954 path::PathBuf,
8955 sync::{Arc, Mutex},
8956 time::Duration,
8957 };
8958
8959 use serde_json::{json, Value};
8960 use subc_protocol::{
8961 manifest::{
8962 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
8963 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
8964 },
8965 session::HealthStatus,
8966 FrameType,
8967 };
8968
8969 use super::*;
8970 use crate::{
8971 forwarding::{DataRoute, DataRouteState},
8972 registry::ChannelState,
8973 router::FrameSink,
8974 stderr_tail::DEFAULT_MAX_LINE_BYTES,
8975 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
8976 RouteCtx, Router,
8977 };
8978 use tokio::{
8979 sync::mpsc,
8980 time::{sleep, Instant},
8981 };
8982 use tracing::{
8983 field::{Field, Visit},
8984 Event, Subscriber,
8985 };
8986 use tracing_subscriber::{layer::Context, prelude::*, Layer};
8987
8988 fn fake_aft_stub_path() -> PathBuf {
9007 let mut path = std::env::current_exe().expect("current_exe available in tests");
9008 path.pop(); path.pop(); path.push(if cfg!(windows) {
9011 "fake-aft-stub.exe"
9012 } else {
9013 "fake-aft-stub"
9014 });
9015 assert!(
9016 path.exists(),
9017 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
9018 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
9019 path.display()
9020 );
9021 path
9022 }
9023
9024 fn client_retries(code: &str) -> bool {
9029 subc_protocol::error_codes::is_retryable_route_open(code)
9030 }
9031
9032 #[test]
9043 fn retryability_of_forwarding_codes_matches_the_failure() {
9044 let transient = [
9047 ForwardingError::NoModuleConnection,
9048 ForwardingError::ModuleReloading {
9049 module_id: "m".into(),
9050 },
9051 ForwardingError::StaleModuleEndpoint,
9052 ForwardingError::UnknownReservation {
9053 client_channel: 1,
9054 module_channel: 1,
9055 },
9056 ForwardingError::ConnectionClosing {
9057 connection_id: ConnectionId::new(1),
9058 },
9059 ForwardingError::ClientEgressClosed {
9060 connection_id: ConnectionId::new(1),
9061 },
9062 ForwardingError::ModuleEgressUnavailable {
9063 connection_id: ConnectionId::new(1),
9064 },
9065 ];
9066 for err in transient {
9067 let code = forwarding_error_code(&err);
9068 assert!(
9069 client_retries(code),
9070 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
9071 );
9072 }
9073
9074 let permanent = [
9079 ForwardingError::ConnectionRoleConflict {
9080 connection_id: ConnectionId::new(1),
9081 },
9082 ForwardingError::ClientRouteChannelExhausted {
9083 connection_id: ConnectionId::new(1),
9084 },
9085 ForwardingError::ModuleRouteChannelExhausted {
9086 endpoint: ModuleEndpointId {
9087 connection_id: ConnectionId::new(1),
9088 generation: 1,
9089 },
9090 },
9091 ForwardingError::RelayCorrelationExhausted,
9092 ForwardingError::RouteOpenBuild("x".into()),
9093 ForwardingError::Poisoned,
9094 ];
9095 for err in permanent {
9096 let code = forwarding_error_code(&err);
9097 assert!(
9098 !client_retries(code),
9099 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
9100 );
9101 }
9102 }
9103
9104 #[tokio::test]
9132 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
9133 let handler = ControlHandler::default();
9134 let frame =
9135 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
9136
9137 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
9141 assert_eq!(
9142 stamped,
9143 Principal::Direct,
9144 "a caller that proved nothing must not be stamped as a supervised module"
9145 );
9146
9147 let forged = handler
9152 .route_open_principal(
9153 &frame,
9154 Some(ConsumerIdentity {
9155 module_id: "aft".to_string(),
9156 launch_nonce: "not-a-real-nonce".to_string(),
9157 }),
9158 )
9159 .unwrap();
9160 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
9161 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
9162 }
9163
9164 #[test]
9181 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
9182 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
9183 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
9184 let ClientControlRequest::RouteOpen {
9185 consumer_identity, ..
9186 } = parsed
9187 else {
9188 panic!("route.open body must parse as RouteOpen");
9189 };
9190 assert_eq!(
9191 consumer_identity,
9192 Some(ConsumerIdentity {
9193 module_id: "aft".to_string(),
9194 launch_nonce: "n".to_string(),
9195 }),
9196 "the wire field name must reach the value route_open_principal reads"
9197 );
9198 }
9199
9200 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
9201 ModuleManifest::builder(module_id, "0.1.0")
9202 .protocol_ver(protocol_ver)
9203 .provides(vec![ProviderRole::ToolProvider {
9204 tools: vec![Tool {
9205 name: "read".to_string(),
9206 description: None,
9207 execution_mode: ExecutionMode::Pure,
9208 schema: json!({"type": "object"}),
9209 }],
9210 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
9211 concurrency: Concurrency::ModuleManaged,
9212 emits_push: true,
9213 sub_supervises: true,
9214 }])
9215 .build()
9216 }
9217
9218 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
9219 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
9220 }
9221
9222 fn hello_frame_with_control_ops(
9223 module_id: &str,
9224 protocol_ver: u8,
9225 corr: u64,
9226 control_ops: Option<Vec<String>>,
9227 ) -> Frame {
9228 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
9229 }
9230
9231 fn hello_frame_with_nonce(
9232 module_id: &str,
9233 protocol_ver: u8,
9234 corr: u64,
9235 launch_nonce: Option<&str>,
9236 ) -> Frame {
9237 hello_frame_full(
9238 module_id,
9239 protocol_ver,
9240 corr,
9241 None,
9242 launch_nonce.map(ToOwned::to_owned),
9243 )
9244 }
9245
9246 fn hello_frame_full(
9247 module_id: &str,
9248 protocol_ver: u8,
9249 corr: u64,
9250 control_ops: Option<Vec<String>>,
9251 launch_nonce: Option<String>,
9252 ) -> Frame {
9253 let body = serde_json::to_vec(&ModuleHelloBody {
9254 manifest: manifest(module_id, protocol_ver),
9255 protocol_ver,
9256 control_ops,
9257 launch_nonce,
9258 })
9259 .unwrap();
9260 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
9261 }
9262
9263 fn hello_frame_with_rich_ui(
9264 module_id: &str,
9265 protocol_ver: u8,
9266 corr: u64,
9267 launch_nonce: Option<&str>,
9268 rich_ui: Value,
9269 ) -> Frame {
9270 let mut body = serde_json::to_value(ModuleHelloBody {
9271 manifest: manifest(module_id, protocol_ver),
9272 protocol_ver,
9273 control_ops: None,
9274 launch_nonce: launch_nonce.map(str::to_string),
9275 })
9276 .unwrap();
9277 body["manifest"]["rich_ui"] = rich_ui;
9278 Frame::build(
9279 FrameType::Hello,
9280 control_flags(),
9281 0,
9282 0,
9283 corr,
9284 serde_json::to_vec(&body).unwrap(),
9285 )
9286 .unwrap()
9287 }
9288
9289 fn non_routable_hello_frame_with_control_ops(
9290 module_id: &str,
9291 corr: u64,
9292 control_ops: Option<Vec<String>>,
9293 ) -> Frame {
9294 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
9295 manifest.provides.clear();
9296 let body = serde_json::to_vec(&ModuleHelloBody {
9297 manifest,
9298 protocol_ver: PROTOCOL_VERSION,
9299 control_ops,
9300 launch_nonce: None,
9301 })
9302 .unwrap();
9303 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
9304 }
9305
9306 fn capability_grammar_hello_frame(
9307 capabilities: Value,
9308 runtime_computed: Option<Value>,
9309 corr: u64,
9310 ) -> Frame {
9311 let mut body = serde_json::to_value(ModuleHelloBody {
9312 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
9313 protocol_ver: PROTOCOL_VERSION,
9314 control_ops: None,
9315 launch_nonce: None,
9316 })
9317 .expect("HELLO body serializes");
9318 body["manifest"]["capabilities"] = capabilities;
9319 if let Some(runtime_computed) = runtime_computed {
9320 body["runtime_computed"] = runtime_computed;
9321 }
9322 Frame::build(
9323 FrameType::Hello,
9324 control_flags(),
9325 0,
9326 0,
9327 corr,
9328 serde_json::to_vec(&body).expect("HELLO body reserializes"),
9329 )
9330 .expect("HELLO frame builds")
9331 }
9332
9333 fn channel_request(channel: u16, corr: u64) -> Frame {
9334 Frame::build(
9335 FrameType::Request,
9336 Flags::new(true, Priority::Interactive, false),
9337 channel,
9338 0,
9339 corr,
9340 b"opaque".to_vec(),
9341 )
9342 .unwrap()
9343 }
9344
9345 fn route_ctx(
9346 connection_id: ConnectionId,
9347 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
9348 let (tx, rx) = mpsc::channel(8);
9349 (
9350 RouteCtx {
9351 connection_id,
9352 egress: FrameSink::new(tx),
9353 },
9354 rx,
9355 )
9356 }
9357
9358 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
9359 serde_json::from_slice(&frame.body).unwrap()
9360 }
9361
9362 async fn hello_via_sink(
9367 handler: &ControlHandler,
9368 ctx: &RouteCtx,
9369 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9370 hello: Frame,
9371 ) -> Frame {
9372 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
9373 assert!(
9374 replies.is_empty(),
9375 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
9376 );
9377 let ack = rx
9378 .try_recv()
9379 .expect("HELLO_ACK is queued on the module sink")
9380 .frame;
9381 assert_eq!(ack.header.ty, FrameType::HelloAck);
9382 ack
9383 }
9384
9385 fn parse_error(frame: &Frame) -> Value {
9386 serde_json::from_slice(&frame.body).unwrap()
9387 }
9388
9389 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
9390 serde_json::from_slice(&frame.body).unwrap()
9391 }
9392
9393 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
9394 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
9395 route_channel,
9396 route_epoch: 0,
9397 kind,
9398 })
9399 .unwrap();
9400 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9401 }
9402
9403 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
9404 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
9405 module_id: module_id.to_string(),
9406 })
9407 .unwrap();
9408 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9409 }
9410
9411 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
9412 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
9413 }
9414
9415 fn legacy_route_open_frame(corr: u64, module_id: &str, root: ScratchDir) -> Frame {
9417 let path = ProjectRootId::from_path_allowing_missing(root.path()).unwrap();
9418 let mut frame = route_open_frame(corr, module_id, root);
9419 let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
9420 body["identity"]["project_root"] = serde_json::json!(path.as_path());
9421 frame.body = serde_json::to_vec(&body).unwrap();
9422 frame
9423 }
9424
9425 fn route_open_frame_with_consumer_capabilities(
9426 corr: u64,
9427 module_id: &str,
9428 _project_root: ScratchDir,
9429 consumer_capabilities: Option<Vec<String>>,
9430 ) -> Frame {
9431 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9432 busy_refusals: None,
9433 target: RouteTarget::ToolProvider {
9434 module_id: module_id.to_string(),
9435 },
9436 identity: BindIdentity::new("unit".to_string(), "session".to_string()),
9437 consumer_identity: None,
9438 consumer_capabilities,
9439 role_versions: None,
9440 admission_facts: None,
9441 scope: None,
9442 })
9443 .unwrap();
9444 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9445 }
9446
9447 fn route_open_frame_with_role_versions(
9448 corr: u64,
9449 module_id: &str,
9450 _project_root: ScratchDir,
9451 role_versions: Option<BTreeMap<String, String>>,
9452 ) -> Frame {
9453 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9454 busy_refusals: None,
9455 target: RouteTarget::ToolProvider {
9456 module_id: module_id.to_string(),
9457 },
9458 identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
9459 consumer_identity: None,
9460 consumer_capabilities: None,
9461 role_versions,
9462 admission_facts: None,
9463 scope: None,
9464 })
9465 .unwrap();
9466 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9467 }
9468
9469 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
9470 entries
9471 .iter()
9472 .map(|(role, version)| (role.to_string(), version.to_string()))
9473 .collect()
9474 }
9475
9476 fn route_open_frame_with_admission_facts(
9477 corr: u64,
9478 module_id: &str,
9479 _project_root: ScratchDir,
9480 consumer_identity: Option<subc_control::ConsumerIdentity>,
9481 facts: Option<Value>,
9482 ) -> Frame {
9483 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9484 busy_refusals: None,
9485 target: RouteTarget::ToolProvider {
9486 module_id: module_id.to_string(),
9487 },
9488 identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
9489 consumer_identity,
9490 consumer_capabilities: None,
9491 role_versions: None,
9492 admission_facts: facts,
9493 scope: None,
9494 })
9495 .unwrap();
9496 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9497 }
9498
9499 #[derive(Clone, Default)]
9500 struct EventCapture {
9501 events: Arc<Mutex<Vec<CapturedEvent>>>,
9502 }
9503
9504 #[derive(Clone, Debug)]
9505 struct CapturedEvent {
9506 target: String,
9507 level: tracing::Level,
9508 fields: BTreeMap<String, String>,
9509 }
9510
9511 impl EventCapture {
9512 fn events(&self) -> Vec<CapturedEvent> {
9513 self.events.lock().unwrap().clone()
9514 }
9515 }
9516
9517 impl<S> Layer<S> for EventCapture
9518 where
9519 S: Subscriber,
9520 {
9521 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
9522 let mut visitor = EventFieldVisitor::default();
9523 event.record(&mut visitor);
9524 self.events.lock().unwrap().push(CapturedEvent {
9525 target: event.metadata().target().to_string(),
9526 level: *event.metadata().level(),
9527 fields: visitor.fields,
9528 });
9529 }
9530 }
9531
9532 #[derive(Default)]
9533 struct EventFieldVisitor {
9534 fields: BTreeMap<String, String>,
9535 }
9536
9537 impl Visit for EventFieldVisitor {
9538 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
9539 self.fields
9540 .insert(field.name().to_string(), format!("{value:?}"));
9541 }
9542 }
9543
9544 fn health_response(corr: u64, status: HealthStatus) -> Frame {
9545 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
9546 status,
9547 detail: Some("warming".to_string()),
9548 metrics: Some(json!({"queue_depth": 3})),
9549 })
9550 .unwrap();
9551 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
9552 }
9553
9554 fn route_bind_ack(corr: u64) -> Frame {
9555 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
9556 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
9557 }
9558
9559 fn unique_project_root(label: &str) -> ScratchDir {
9560 ScratchDir::new(label)
9561 }
9562
9563 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
9564 match parse_route_poll(frame) {
9565 ClientControlResponse::RoutePoll {
9566 status: None,
9567 live: Some(live),
9568 ..
9569 } => assert_eq!(live, expected_live),
9570 other => panic!("unexpected route.poll response: {other:?}"),
9571 }
9572 }
9573
9574 fn bind_liveness_route(
9575 registry: &Registry,
9576 forwarding: &ForwardingTable,
9577 module_id: &str,
9578 ) -> (RouteCtx, u16, u32) {
9579 let module_connection = ConnectionId::new(101);
9580 let client_connection = ConnectionId::new(202);
9581 let registration = registry
9582 .register_with_control_ops(
9583 manifest(module_id, PROTOCOL_VERSION),
9584 PROTOCOL_VERSION,
9585 module_connection,
9586 module_baseline_control_ops(),
9587 )
9588 .unwrap();
9589 let (module_tx, _module_rx) = mpsc::channel(8);
9590 let endpoint = forwarding
9591 .register_module_connection(
9592 module_connection,
9593 module_id.to_string(),
9594 PROTOCOL_VERSION,
9595 manifest_concurrency(®istration.manifest),
9596 FrameSink::new(module_tx),
9597 )
9598 .unwrap();
9599 let (client_ctx, _client_rx) = route_ctx(client_connection);
9600 let pending = forwarding
9601 .begin_route_bind_relay_for_test(
9602 client_connection,
9603 client_ctx.egress.clone(),
9604 1,
9605 module_id,
9606 )
9607 .unwrap();
9608 assert_eq!(pending.endpoint, endpoint);
9609 let route_channel = pending.client_channel;
9610 let route_epoch = pending.client_epoch;
9611 forwarding
9612 .complete_pending_relay(
9613 module_connection,
9614 pending.corr,
9615 RouteBindRelayOutcome::Accepted,
9616 )
9617 .unwrap();
9618 (client_ctx, route_channel, route_epoch)
9619 }
9620
9621 struct FakeProcessLiveness {
9622 live: Option<bool>,
9623 }
9624
9625 impl ModuleProcessLiveness for FakeProcessLiveness {
9626 fn process_live(&self, _module_id: &str) -> Option<bool> {
9627 self.live
9628 }
9629 }
9630
9631 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9632 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
9633 {
9634 let registry = Arc::new(Registry::default());
9635 let supervisor_handle = SupervisorHandle::new();
9636 let supervisor = Supervisor::new_for_test(
9637 Arc::clone(®istry),
9638 RestartPolicy::new(1, Duration::from_millis(10)),
9639 )
9640 .with_handle(supervisor_handle.clone());
9641 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
9642 let module = supervisor
9643 .spawn(ModuleSpec {
9644 carry_containment: false,
9645 module_id: "stderr-tail-wire".to_string(),
9646 program: fake_aft_stub_path(),
9647 args: Vec::new(),
9648 env: vec![
9649 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
9650 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
9651 ],
9652 reserved: false,
9653 reserved_prefixes: Vec::new(),
9654 protocol: ModuleProtocol::Subc,
9655 overlap: Default::default(),
9656 })
9657 .unwrap();
9658
9659 let deadline = Instant::now() + Duration::from_secs(5);
9660 loop {
9661 let tail = module.stderr_tail(None, None);
9662 if tail
9663 .entries
9664 .iter()
9665 .any(|entry| matches!(entry, TailEntry::ProcessStart))
9666 && tail.entries.iter().any(|entry| {
9667 matches!(
9668 entry,
9669 TailEntry::Line {
9670 truncated: true,
9671 ..
9672 }
9673 )
9674 })
9675 {
9676 break;
9677 }
9678 assert!(
9679 Instant::now() < deadline,
9680 "module did not produce a truncated line and restart boundary: {tail:?}"
9681 );
9682 sleep(Duration::from_millis(10)).await;
9683 }
9684
9685 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9686 let request = ClientControlRequest::SupervisorStderrTail {
9687 module_id: "stderr-tail-wire".to_string(),
9688 max_lines: None,
9689 max_bytes: None,
9690 };
9691 let frame = Frame::build(
9692 FrameType::Request,
9693 control_flags(),
9694 0,
9695 0,
9696 1,
9697 serde_json::to_vec(&request).unwrap(),
9698 )
9699 .unwrap();
9700 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9701 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
9702 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
9703 serde_json::from_slice(&responses[0].body).unwrap()
9704 else {
9705 panic!("expected supervisor.stderr_tail response");
9706 };
9707
9708 assert!(
9709 tail.entries
9710 .iter()
9711 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
9712 "the control response lost the restart boundary"
9713 );
9714 let Some(StderrTailEntry::Line {
9715 text,
9716 truncated,
9717 at_ms,
9718 }) = tail.entries.iter().find(|entry| {
9719 matches!(
9720 entry,
9721 StderrTailEntry::Line {
9722 truncated: true,
9723 ..
9724 }
9725 )
9726 })
9727 else {
9728 panic!("the control response lost the truncated line");
9729 };
9730 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
9731 assert!(*truncated);
9732 assert!(
9733 at_ms.is_some(),
9734 "the control response lost the line's capture time"
9735 );
9736 }
9737
9738 #[tokio::test(flavor = "current_thread")]
9743 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
9744 let dir = ScratchDir::new("terminals-off-worker");
9745 let journal_path = dir.join("terminals.jsonl");
9746 let registry = Arc::new(Registry::default());
9747 let supervisor_handle = SupervisorHandle::new();
9748 let supervisor =
9749 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
9750 .with_handle(supervisor_handle.clone())
9751 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
9752 let module = supervisor
9753 .spawn(ModuleSpec {
9754 carry_containment: false,
9755 module_id: "terminal-off-worker".to_string(),
9756 program: fake_aft_stub_path(),
9757 args: Vec::new(),
9758 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
9759 reserved: false,
9760 reserved_prefixes: Vec::new(),
9761 protocol: ModuleProtocol::Subc,
9762 overlap: Default::default(),
9763 })
9764 .unwrap();
9765 let deadline = Instant::now() + Duration::from_secs(5);
9766 while module.terminal_history().entries.len() != 2 {
9767 assert!(Instant::now() < deadline, "module did not record two exits");
9768 sleep(Duration::from_millis(10)).await;
9769 }
9770
9771 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
9772 let handler =
9773 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
9774 let frame = Frame::build(
9775 FrameType::Request,
9776 control_flags(),
9777 0,
9778 0,
9779 1,
9780 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
9781 module_id: "terminal-off-worker".to_string(),
9782 })
9783 .unwrap(),
9784 )
9785 .unwrap();
9786 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9787 let spawned_at = std::time::Instant::now();
9788 let read = tokio::spawn({
9789 let handler = Arc::clone(&handler);
9790 async move { handler.handle_control_frame(&ctx, frame).await }
9791 });
9792 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
9795 .await
9796 .unwrap()
9797 .expect("the history read reached its pause");
9798 let elapsed = spawned_at.elapsed();
9799 assert!(
9800 elapsed < Duration::from_secs(2) && !read.is_finished(),
9801 "this task could not run while the history read was paused \
9802 (resumed after {elapsed:?}, read finished: {})",
9803 read.is_finished()
9804 );
9805
9806 drop(release);
9807 let responses = read.await.unwrap().unwrap();
9808 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9809 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
9810 panic!("expected supervisor.terminals response");
9811 };
9812 assert_eq!(terminals.entries.len(), 2);
9813 assert_eq!(terminals.journal_skipped_lines, 0);
9814 assert_eq!(terminals.journal_read_errors, 0);
9815 }
9816
9817 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9818 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
9819 let registry = Arc::new(Registry::default());
9820 let supervisor_handle = SupervisorHandle::new();
9821 let supervisor =
9822 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
9823 .with_handle(supervisor_handle.clone());
9824 let module = supervisor
9825 .spawn(ModuleSpec {
9826 carry_containment: false,
9827 module_id: "terminal-golden".to_string(),
9828 program: fake_aft_stub_path(),
9829 args: Vec::new(),
9830 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
9831 reserved: false,
9832 reserved_prefixes: Vec::new(),
9833 protocol: ModuleProtocol::Subc,
9834 overlap: Default::default(),
9835 })
9836 .unwrap();
9837
9838 let deadline = Instant::now() + Duration::from_secs(5);
9839 while module.terminal_history().entries.len() != 2 {
9840 assert!(
9841 Instant::now() < deadline,
9842 "module did not retain two terminal exits: {:?}",
9843 module.terminal_history()
9844 );
9845 sleep(Duration::from_millis(10)).await;
9846 }
9847
9848 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9849 let request = ClientControlRequest::SupervisorTerminals {
9850 module_id: "terminal-golden".to_string(),
9851 };
9852 let frame = Frame::build(
9853 FrameType::Request,
9854 control_flags(),
9855 0,
9856 0,
9857 1,
9858 serde_json::to_vec(&request).unwrap(),
9859 )
9860 .unwrap();
9861 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9862 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
9863 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9864 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
9865 panic!("expected supervisor.terminals response");
9866 };
9867 assert_eq!(terminals.entries.len(), 2);
9868 assert_eq!(terminals.dropped, 0);
9869
9870 let mut rendered = serde_json::to_value(response).unwrap();
9871 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
9874 for (index, entry) in rendered["entries"]
9875 .as_array_mut()
9876 .expect("terminal response entries array")
9877 .iter_mut()
9878 .enumerate()
9879 {
9880 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
9881 }
9882
9883 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9884 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
9885 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
9886 if std::env::var_os("UPDATE_GOLDEN").is_some() {
9887 std::fs::write(&golden_path, &serialized).unwrap();
9888 }
9889 let expected: Value =
9890 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
9891 assert_eq!(rendered, expected);
9892 }
9893
9894 #[test]
9895 fn hello_registers_manifest_and_returns_ack() {
9896 let registry = Arc::new(Registry::default());
9897 let handler = ControlHandler::new(Arc::clone(®istry));
9898 let conn = ConnectionId::new(1);
9899
9900 let responses = handler
9901 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
9902 .unwrap();
9903
9904 assert_eq!(responses.len(), 1);
9905 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
9906 assert_eq!(responses[0].header.channel, 0);
9907 assert_eq!(responses[0].header.corr, 7);
9908 let ack = parse_ack(&responses[0]);
9909 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9910 assert!(ack
9911 .subc_capabilities
9912 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
9913 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
9914 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
9915 assert!(ack
9916 .subc_ops
9917 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
9918 assert!(ack
9919 .subc_ops
9920 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
9921
9922 let registration = registry.get_module("aft").unwrap().unwrap();
9923 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
9924 assert_eq!(registration.state, ChannelState::Active);
9925 assert_eq!(registration.connection_id, conn);
9926 assert_eq!(registration.control_ops, module_baseline_control_ops());
9927 assert!(registration.manifest.display_name.is_none());
9928 }
9929
9930 #[test]
9931 fn hello_display_name_refuses_invalid_manifest_by_field() {
9932 let registry = Arc::new(Registry::default());
9933 let handler = ControlHandler::new(Arc::clone(®istry));
9934 for (index, invalid) in [
9935 json!(""),
9936 json!("a".repeat(41)),
9937 json!(" Routing"),
9938 json!("a\nb"),
9939 json!("a\u{202e}b"),
9940 json!(7),
9941 ]
9942 .into_iter()
9943 .enumerate()
9944 {
9945 let module_id = format!("bad-display-name-{index}");
9946 let mut body = serde_json::to_value(ModuleHelloBody {
9947 manifest: manifest(&module_id, PROTOCOL_VERSION),
9948 protocol_ver: PROTOCOL_VERSION,
9949 control_ops: None,
9950 launch_nonce: None,
9951 })
9952 .unwrap();
9953 body["manifest"]["display_name"] = invalid;
9954 let hello = Frame::build(
9955 FrameType::Hello,
9956 control_flags(),
9957 0,
9958 0,
9959 7,
9960 serde_json::to_vec(&body).unwrap(),
9961 )
9962 .unwrap();
9963 let response = handler
9964 .handle_control(ConnectionId::new(100 + index as u64), hello)
9965 .unwrap();
9966 assert_eq!(response[0].header.ty, FrameType::Error);
9967 let error = parse_error(&response[0]);
9968 assert_eq!(error["code"], "invalid_manifest");
9969 assert!(error["message"].as_str().unwrap().contains("display_name"));
9970 assert!(registry.get_module(&module_id).unwrap().is_none());
9971 }
9972 }
9973
9974 #[test]
9975 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
9976 let invalid_identifiers = [
9977 ("case_change", "credentials-Provider/v1"),
9978 ("leading_zero", "credentials-provider/v01"),
9979 ("trailing_hyphen", "credentials-provider-/v1"),
9980 ("consecutive_hyphens", "credentials--provider/v1"),
9981 ("uppercase", "Credentials-provider/v1"),
9982 ("missing_v", "credentials-provider/1"),
9983 ("whitespace", "credentials provider/v1"),
9984 ("zero_version", "credentials-provider/v0"),
9985 ("out_of_range_version", "credentials-provider/v4294967296"),
9986 (
9987 "overlength_name",
9988 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
9989 ),
9990 ];
9991 let mut cases = invalid_identifiers
9992 .into_iter()
9993 .map(|(name, identifier)| {
9994 (
9995 format!("identifier_{name}"),
9996 "capabilities.provides[0]".to_string(),
9997 identifier.to_string(),
9998 json!({ "provides": [identifier] }),
9999 None,
10000 )
10001 })
10002 .collect::<Vec<_>>();
10003 cases.extend([
10004 (
10005 "unknown_need".to_string(),
10006 "capabilities.requires[0].need".to_string(),
10007 "deferred".to_string(),
10008 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
10009 None,
10010 ),
10011 (
10012 "duplicate_provides".to_string(),
10013 "capabilities.provides[1]".to_string(),
10014 "credentials-provider/v1".to_string(),
10015 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
10016 None,
10017 ),
10018 (
10019 "duplicate_must_never_reach".to_string(),
10020 "capabilities.must_never_reach[1]".to_string(),
10021 "credentials-provider/v1".to_string(),
10022 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
10023 None,
10024 ),
10025 (
10026 "duplicate_requires_same_need".to_string(),
10027 "capabilities.requires[1]".to_string(),
10028 "credentials-provider/v1".to_string(),
10029 json!({ "requires": [
10030 { "capability": "credentials-provider/v1", "need": "required" },
10031 { "capability": "credentials-provider/v1", "need": "required" }
10032 ] }),
10033 None,
10034 ),
10035 (
10036 "duplicate_requires_conflicting_need".to_string(),
10037 "capabilities.requires[1]".to_string(),
10038 "credentials-provider/v1".to_string(),
10039 json!({ "requires": [
10040 { "capability": "credentials-provider/v1", "need": "required" },
10041 { "capability": "credentials-provider/v1", "need": "optional" }
10042 ] }),
10043 None,
10044 ),
10045 (
10046 "capabilities_root_pointer".to_string(),
10047 "runtime_computed[0]".to_string(),
10048 "/capabilities".to_string(),
10049 json!({}),
10050 Some(json!(["/capabilities"])),
10051 ),
10052 (
10053 "capabilities_descendant_pointer".to_string(),
10054 "runtime_computed[0]".to_string(),
10055 "/capabilities/provides".to_string(),
10056 json!({}),
10057 Some(json!(["/capabilities/provides"])),
10058 ),
10059 (
10060 "malformed_pointer_without_leading_slash".to_string(),
10061 "runtime_computed[0]".to_string(),
10062 "capabilities".to_string(),
10063 json!({}),
10064 Some(json!(["capabilities"])),
10065 ),
10066 (
10067 "malformed_pointer_escape".to_string(),
10068 "runtime_computed[0]".to_string(),
10069 "/roles/~2/tools".to_string(),
10070 json!({}),
10071 Some(json!(["/roles/~2/tools"])),
10072 ),
10073 (
10074 "unknown_capabilities_field".to_string(),
10075 "capabilities.future".to_string(),
10076 "<array>".to_string(),
10077 json!({ "future": [] }),
10078 None,
10079 ),
10080 ]);
10081
10082 for (index, (name, field, value, capabilities, runtime_computed)) in
10083 cases.into_iter().enumerate()
10084 {
10085 let registry = Arc::new(Registry::default());
10086 let handler = ControlHandler::new(Arc::clone(®istry));
10087 let response = handler
10088 .handle_control(
10089 ConnectionId::new((index + 1) as u64),
10090 capability_grammar_hello_frame(
10091 capabilities,
10092 runtime_computed,
10093 index as u64 + 1,
10094 ),
10095 )
10096 .expect("invalid HELLO returns a refusal");
10097
10098 assert_eq!(response.len(), 1, "{name} must emit one refusal");
10099 let error = parse_error(&response[0]);
10100 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
10101 let message = error["message"]
10102 .as_str()
10103 .expect("error message is a string");
10104 assert!(
10105 message.contains(&field),
10106 "{name}: field missing from {message}"
10107 );
10108 assert!(
10109 message.contains(&value),
10110 "{name}: value missing from {message}"
10111 );
10112 assert_eq!(
10113 registry
10114 .active_registration_count()
10115 .expect("registry reads"),
10116 0,
10117 "{name}: refused HELLO must not create a catalog entry"
10118 );
10119 }
10120 }
10121
10122 #[test]
10123 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
10124 let registry = Arc::new(Registry::default());
10125 let handler = ControlHandler::new(Arc::clone(®istry));
10126 let capabilities = json!({
10127 "provides": ["credentials-provider/v1"],
10128 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
10129 "must_never_reach": ["federation-transport/v1"]
10130 });
10131 let response = handler
10132 .handle_control(
10133 ConnectionId::new(99),
10134 capability_grammar_hello_frame(
10135 capabilities.clone(),
10136 Some(json!(["/roles/0/tools"])),
10137 99,
10138 ),
10139 )
10140 .expect("valid HELLO registers");
10141 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10142
10143 let request = Frame::build(
10144 FrameType::Request,
10145 control_flags(),
10146 0,
10147 0,
10148 100,
10149 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
10150 .expect("catalog request serializes"),
10151 )
10152 .expect("catalog request frame builds");
10153 let response = handler
10154 .handle_catalog_list(request, None)
10155 .expect("catalog list succeeds");
10156 let ClientControlResponse::CatalogList { modules, .. } =
10157 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
10158 else {
10159 panic!("catalog request must return catalog.list");
10160 };
10161 assert_eq!(modules.len(), 1);
10162 assert_eq!(
10163 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
10164 capabilities
10165 );
10166 }
10167
10168 #[test]
10169 fn catalog_list_mirrors_management_operation_description() {
10170 let registry = Arc::new(Registry::default());
10171 let handler = ControlHandler::new(Arc::clone(®istry));
10172 let description = "List managed records and return their identifiers and metadata.";
10173 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
10174 manifest.provides = vec![ProviderRole::ManagementSurface {
10175 operations: vec![ManagementOperation {
10176 name: "records.list".to_string(),
10177 kind: ManagementOperationKind::Query,
10178 description: Some(description.to_string()),
10179 }],
10180 config_schema: json!({"type": "object"}),
10181 observability: vec![ObservabilitySurface {
10182 name: "records.stats".to_string(),
10183 kind: ObservabilityKind::Snapshot,
10184 }],
10185 identity_scope: vec![IdentityScope::Project],
10186 concurrency: Concurrency::ModuleManaged,
10187 }];
10188 registry
10189 .register_with_control_ops(
10190 manifest,
10191 PROTOCOL_VERSION,
10192 ConnectionId::new(99),
10193 Vec::new(),
10194 )
10195 .expect("described management manifest registers");
10196
10197 let request = Frame::build(
10198 FrameType::Request,
10199 control_flags(),
10200 0,
10201 0,
10202 100,
10203 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
10204 .expect("catalog request serializes"),
10205 )
10206 .expect("catalog request frame builds");
10207 let response = handler
10208 .handle_catalog_list(request, None)
10209 .expect("catalog list succeeds");
10210 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
10211 assert_eq!(
10212 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
10213 "catalog.list must preserve the declared operation description verbatim"
10214 );
10215 }
10216
10217 #[test]
10218 fn catalog_list_mirrors_registered_operation_delivery() {
10219 let registry = Arc::new(Registry::default());
10220 let handler = ControlHandler::new(Arc::clone(®istry));
10221 let delivery =
10222 json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
10223 let mut body = serde_json::to_value(ModuleHelloBody {
10224 manifest: manifest("watch-management", PROTOCOL_VERSION),
10225 protocol_ver: PROTOCOL_VERSION,
10226 control_ops: None,
10227 launch_nonce: None,
10228 })
10229 .unwrap();
10230 body["manifest"]["provides"] = json!([{
10231 "role": "management_surface",
10232 "operations": [
10233 {"name": "status", "kind": "query"},
10234 {"name": "future", "kind": "query"},
10235 {"name": "reset", "kind": "mutate"}
10236 ],
10237 "config_schema": {}, "observability": [], "identity_scope": []
10238 }]);
10239 body["manifest"]["operation_delivery"] = delivery.clone();
10240 let hello = Frame::build(
10241 FrameType::Hello,
10242 control_flags(),
10243 0,
10244 0,
10245 99,
10246 serde_json::to_vec(&body).unwrap(),
10247 )
10248 .unwrap();
10249 let response = handler
10250 .handle_control(ConnectionId::new(99), hello)
10251 .unwrap();
10252 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10253
10254 let (_, registrations) = registry.list_modules().unwrap();
10255 let registered = ®istrations[0].manifest;
10256 assert!(registered.is_watch_operation("status"));
10257 assert!(!registered.is_watch_operation("future"));
10258 assert!(!registered.is_watch_operation("reset"));
10259 assert!(!registered.is_watch_operation("missing"));
10260
10261 let request = Frame::build(
10262 FrameType::Request,
10263 control_flags(),
10264 0,
10265 0,
10266 100,
10267 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10268 )
10269 .unwrap();
10270 let response = handler.handle_catalog_list(request, None).unwrap();
10271 let body: Value = serde_json::from_slice(&response[0].body).unwrap();
10272 assert_eq!(body["modules"][0]["operation_delivery"], delivery);
10273 let ClientControlResponse::CatalogList { modules, .. } =
10274 serde_json::from_slice(&response[0].body).unwrap()
10275 else {
10276 panic!("expected catalog.list");
10277 };
10278 assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
10279 }
10280
10281 #[test]
10282 fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
10283 let registry = Arc::new(Registry::default());
10284 let supervisor = SupervisorHandle::new();
10285 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
10286 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
10287 let descriptor = json!({
10288 "schema": "ckios.richui.manifest/1",
10289 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
10290 "bytes": 4096,
10291 "chunks": 4
10292 });
10293 let connection = ConnectionId::new(201);
10294 let hello = handler
10295 .handle_control(
10296 connection,
10297 hello_frame_with_rich_ui(
10298 "vault",
10299 PROTOCOL_VERSION,
10300 201,
10301 Some("the-real-nonce"),
10302 descriptor.clone(),
10303 ),
10304 )
10305 .unwrap();
10306 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10307
10308 let request =
10309 Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
10310 let provides = registry
10311 .get_module_by_connection(connection)
10312 .unwrap()
10313 .unwrap()
10314 .manifest
10315 .provides;
10316 handler
10317 .handle_catalog_update(connection, request, provides, None, Some(false))
10318 .unwrap();
10319
10320 let request = Frame::build(
10321 FrameType::Request,
10322 control_flags(),
10323 0,
10324 0,
10325 203,
10326 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10327 )
10328 .unwrap();
10329 let response = handler.handle_catalog_list(request, None).unwrap();
10330 let ClientControlResponse::CatalogList { modules, .. } =
10331 serde_json::from_slice(&response[0].body).unwrap()
10332 else {
10333 panic!("expected catalog.list");
10334 };
10335 assert_eq!(
10336 serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
10337 descriptor
10338 );
10339 assert!(modules[0].nonce_verified);
10340 }
10341
10342 fn hello_with_role_declarations(declarations: Value) -> Frame {
10343 let mut body = serde_json::to_value(ModuleHelloBody {
10344 manifest: manifest("role-test", PROTOCOL_VERSION),
10345 protocol_ver: PROTOCOL_VERSION,
10346 control_ops: None,
10347 launch_nonce: None,
10348 })
10349 .unwrap();
10350 body["manifest"]["role_declarations"] = declarations;
10351 Frame::build(
10352 FrameType::Hello,
10353 control_flags(),
10354 0,
10355 0,
10356 301,
10357 serde_json::to_vec(&body).unwrap(),
10358 )
10359 .unwrap()
10360 }
10361
10362 fn role_declarations_catalog(handler: &ControlHandler) -> (Value, CatalogEntry) {
10363 let request = Frame::build(
10364 FrameType::Request,
10365 control_flags(),
10366 0,
10367 0,
10368 302,
10369 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10370 )
10371 .unwrap();
10372 let response = handler.handle_catalog_list(request, None).unwrap();
10373 let raw: Value = serde_json::from_slice(&response[0].body).unwrap();
10374 let ClientControlResponse::CatalogList { mut modules, .. } =
10375 serde_json::from_slice(&response[0].body).unwrap()
10376 else {
10377 panic!("expected catalog.list");
10378 };
10379 (raw["modules"][0].clone(), modules.remove(0))
10380 }
10381
10382 #[test]
10383 fn catalog_list_mirrors_role_declarations_through_hello_and_catalog_update() {
10384 let registry = Arc::new(Registry::default());
10385 let handler = ControlHandler::new(Arc::clone(®istry));
10386 let declarations = json!({
10387 "routing/v1": {"consumers": [{"name": "chat", "default_demand": 2}]},
10388 "future/v2": {"unknown": [null, true, {"nested": "λ"}]}
10389 });
10390 let connection = ConnectionId::new(301);
10391 let response = handler
10392 .handle_control(
10393 connection,
10394 hello_with_role_declarations(declarations.clone()),
10395 )
10396 .unwrap();
10397 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10398 let registered = registry
10399 .get_module_by_connection(connection)
10400 .unwrap()
10401 .unwrap()
10402 .manifest;
10403 assert_eq!(
10407 serde_json::to_value(®istered.role_declarations).unwrap(),
10408 declarations
10409 );
10410 let (raw, entry) = role_declarations_catalog(&handler);
10411 assert_eq!(raw["role_declarations"], declarations);
10412 assert_eq!(entry.role_declarations, registered.role_declarations);
10413
10414 let request =
10415 Frame::build(FrameType::Request, control_flags(), 0, 0, 303, Vec::new()).unwrap();
10416 handler
10417 .handle_catalog_update(connection, request, registered.provides, None, Some(false))
10418 .unwrap();
10419 let (raw, entry) = role_declarations_catalog(&handler);
10420 assert_eq!(raw["role_declarations"], declarations);
10421 assert_eq!(entry.role_declarations, registered.role_declarations);
10422 }
10423
10424 fn assert_hello_drops_role_declarations_with_warning(declarations: Value, reason: &str) {
10425 let registry = Arc::new(Registry::default());
10426 let handler = ControlHandler::new(Arc::clone(®istry));
10427 let capture = EventCapture::default();
10428 let _subscriber =
10429 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10430 let response = handler
10431 .handle_control(
10432 ConnectionId::new(301),
10433 hello_with_role_declarations(declarations),
10434 )
10435 .unwrap();
10436 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10437 assert!(registry
10438 .get_module_by_connection(ConnectionId::new(301))
10439 .unwrap()
10440 .unwrap()
10441 .manifest
10442 .role_declarations
10443 .is_empty());
10444 let (raw, entry) = role_declarations_catalog(&handler);
10445 assert!(raw.get("role_declarations").is_none());
10446 assert!(entry.role_declarations.is_empty());
10447 let warnings: Vec<_> = capture
10448 .events()
10449 .into_iter()
10450 .filter(|event| {
10451 event.fields.get("message").map(String::as_str)
10452 == Some("dropping whole role_declarations map from HELLO")
10453 })
10454 .collect();
10455 assert_eq!(warnings.len(), 1);
10456 assert_eq!(warnings[0].level, tracing::Level::WARN);
10457 assert_eq!(warnings[0].fields["module_id"], "\"role-test\"");
10458 assert_eq!(warnings[0].fields["reason"], reason);
10459 }
10460
10461 #[test]
10462 fn hello_non_object_role_declarations_drop_whole_map_and_warn() {
10463 for value in [
10464 json!(null),
10465 json!(false),
10466 json!(0),
10467 json!("opaque"),
10468 json!([]),
10469 ] {
10470 assert_hello_drops_role_declarations_with_warning(
10471 json!({"routing/v1": {}, "broken/v1": value}),
10472 "role_declarations values must be objects",
10473 );
10474 }
10475 assert_hello_drops_role_declarations_with_warning(
10476 json!([]),
10477 "role_declarations must be an object",
10478 );
10479 }
10480
10481 #[test]
10482 fn hello_bad_role_declaration_key_drops_whole_map_and_warns() {
10483 assert_hello_drops_role_declarations_with_warning(
10484 json!({"routing/v1": {}, "bad key": {}}),
10485 "role_declarations key must use capability-name grammar",
10486 );
10487 }
10488
10489 #[test]
10490 fn hello_oversized_role_declarations_drop_whole_map_and_warn() {
10491 assert_hello_drops_role_declarations_with_warning(
10492 json!({"routing/v1": {}, "overflow/v1": {"padding": "x".repeat(16_384)}}),
10493 "role_declarations exceeds 16384 compact JSON bytes",
10494 );
10495 }
10496
10497 fn hello_with_flow_operations(capabilities: Value, flow_operations: Option<Value>) -> Frame {
10498 let mut body = serde_json::to_value(ModuleHelloBody {
10499 manifest: manifest("flow-ops", PROTOCOL_VERSION),
10500 protocol_ver: PROTOCOL_VERSION,
10501 control_ops: None,
10502 launch_nonce: None,
10503 })
10504 .unwrap();
10505 body["manifest"]["provides"] = json!([{
10506 "role": "management_surface",
10507 "operations": [
10508 {"name": "diagnostics.findings", "kind": "query"},
10509 {"name": "reset", "kind": "mutate"}
10510 ],
10511 "config_schema": {}, "observability": [], "identity_scope": []
10512 }]);
10513 body["manifest"]["capabilities"] = json!({ "provides": capabilities });
10514 if let Some(flow_operations) = flow_operations {
10515 body["manifest"]["flow_operations"] = flow_operations;
10516 }
10517 Frame::build(
10518 FrameType::Hello,
10519 control_flags(),
10520 0,
10521 0,
10522 401,
10523 serde_json::to_vec(&body).unwrap(),
10524 )
10525 .unwrap()
10526 }
10527
10528 #[test]
10529 fn catalog_list_mirrors_flow_operations_unchanged() {
10530 for (capability, flow_operations) in [
10533 ("flow-scopes/v1", json!(["reset", "diagnostics.findings"])),
10534 ("call-modes/v1", json!(["diagnostics.findings"])),
10535 ("flow-scopes/v1", json!([])),
10536 ] {
10537 let registry = Arc::new(Registry::default());
10538 let handler = ControlHandler::new(Arc::clone(®istry));
10539 let connection = ConnectionId::new(401);
10540 let response = handler
10541 .handle_control(
10542 connection,
10543 hello_with_flow_operations(json!([capability]), Some(flow_operations.clone())),
10544 )
10545 .unwrap();
10546 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10547 let registered = registry
10548 .get_module_by_connection(connection)
10549 .unwrap()
10550 .unwrap()
10551 .manifest;
10552 assert_eq!(
10553 serde_json::to_value(®istered.flow_operations).unwrap(),
10554 flow_operations
10555 );
10556 let (raw, entry) = role_declarations_catalog(&handler);
10557 assert_eq!(raw["flow_operations"], flow_operations);
10558 assert_eq!(entry.flow_operations, registered.flow_operations);
10559 }
10560
10561 let registry = Arc::new(Registry::default());
10562 let handler = ControlHandler::new(Arc::clone(®istry));
10563 let response = handler
10564 .handle_control(
10565 ConnectionId::new(401),
10566 hello_with_flow_operations(json!(["flow-scopes/v1"]), None),
10567 )
10568 .unwrap();
10569 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10570 let (raw, entry) = role_declarations_catalog(&handler);
10571 assert!(raw.get("flow_operations").is_none());
10572 assert_eq!(entry.flow_operations, None);
10573 }
10574
10575 fn assert_hello_drops_flow_operations_with_warning(
10576 capabilities: Value,
10577 flow_operations: Value,
10578 reason: &str,
10579 ) {
10580 let registry = Arc::new(Registry::default());
10581 let handler = ControlHandler::new(Arc::clone(®istry));
10582 let capture = EventCapture::default();
10583 let _subscriber =
10584 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10585 let response = handler
10586 .handle_control(
10587 ConnectionId::new(401),
10588 hello_with_flow_operations(capabilities, Some(flow_operations)),
10589 )
10590 .unwrap();
10591 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10592 let registered = registry
10593 .get_module_by_connection(ConnectionId::new(401))
10594 .unwrap()
10595 .expect("the module still registers")
10596 .manifest;
10597 assert_eq!(registered.module_id, "flow-ops");
10598 assert_eq!(registered.flow_operations, None);
10599 let (raw, entry) = role_declarations_catalog(&handler);
10600 assert_eq!(raw["module_id"], "flow-ops");
10601 assert!(raw.get("flow_operations").is_none());
10602 assert_eq!(entry.flow_operations, None);
10603 let warnings: Vec<_> = capture
10604 .events()
10605 .into_iter()
10606 .filter(|event| {
10607 event.fields.get("message").map(String::as_str)
10608 == Some("dropping flow_operations from HELLO")
10609 })
10610 .collect();
10611 assert_eq!(warnings.len(), 1);
10612 assert_eq!(warnings[0].level, tracing::Level::WARN);
10613 assert_eq!(warnings[0].fields["module_id"], "\"flow-ops\"");
10614 assert_eq!(warnings[0].fields["reason"], reason);
10615 }
10616
10617 #[test]
10618 fn hello_flow_operations_naming_unknown_operation_is_dropped_with_warning() {
10619 assert_hello_drops_flow_operations_with_warning(
10620 json!(["flow-scopes/v1"]),
10621 json!(["diagnostics.findings", "missing"]),
10622 "flow_operations names 'missing', which is not a declared management operation",
10623 );
10624 }
10625
10626 #[test]
10627 fn hello_flow_operations_without_flow_capability_is_dropped_with_warning() {
10628 assert_hello_drops_flow_operations_with_warning(
10629 json!([]),
10630 json!(["diagnostics.findings"]),
10631 "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
10632 );
10633 assert_hello_drops_flow_operations_with_warning(
10635 json!(["agent-run-scopes/v1"]),
10636 json!([]),
10637 "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
10638 );
10639 }
10640
10641 #[test]
10642 fn hello_malformed_flow_operations_is_dropped_with_warning() {
10643 assert_hello_drops_flow_operations_with_warning(
10644 json!(["flow-scopes/v1"]),
10645 json!(["reset", "reset"]),
10646 "flow_operations names 'reset' more than once",
10647 );
10648 assert_hello_drops_flow_operations_with_warning(
10649 json!(["flow-scopes/v1"]),
10650 json!(vec!["reset"; 257]),
10651 "flow_operations lists 257 names, more than 256",
10652 );
10653 assert_hello_drops_flow_operations_with_warning(
10654 json!(["flow-scopes/v1"]),
10655 json!("reset"),
10656 "flow_operations must be an array of operation names",
10657 );
10658 }
10659
10660 #[test]
10661 fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
10662 let registry = Arc::new(Registry::default());
10663 let handler = ControlHandler::new(registry.clone());
10664 let descriptor = json!({
10665 "schema": "ckios.richui.manifest/1",
10666 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
10667 "bytes": 128,
10668 "chunks": 2
10669 });
10670 let malformed = [
10671 ("schema", json!("")),
10672 ("schema", json!("a".repeat(65))),
10673 ("schema", json!("Bad schema")),
10674 (
10675 "manifest_digest",
10676 json!(format!("sha256:{}", "A".repeat(64))),
10677 ),
10678 ("manifest_digest", json!("sha256:abc")),
10679 ("bytes", json!(0)),
10680 ("bytes", json!(16_777_217)),
10681 ("bytes", json!("128")),
10682 ("chunks", json!(0)),
10683 ("chunks", json!(1025)),
10684 ("chunks", json!("2")),
10685 ];
10686 for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
10687 let mut invalid = descriptor.clone();
10688 invalid[field] = invalid_value;
10689 let response = handler
10690 .handle_control(
10691 ConnectionId::new(210 + index as u64),
10692 hello_frame_with_rich_ui(
10693 &format!("bad-rich-ui-{index}"),
10694 PROTOCOL_VERSION,
10695 210 + index as u64,
10696 None,
10697 invalid,
10698 ),
10699 )
10700 .unwrap();
10701 assert_eq!(response[0].header.ty, FrameType::Error);
10702 let error = parse_error(&response[0]);
10703 assert_eq!(error["code"], "invalid_manifest");
10704 assert!(error["message"]
10705 .as_str()
10706 .unwrap()
10707 .contains(&format!("rich_ui.{field}")));
10708 assert!(registry
10709 .get_module(&format!("bad-rich-ui-{index}"))
10710 .unwrap()
10711 .is_none());
10712 }
10713 }
10714
10715 #[test]
10716 fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
10717 let registry = Arc::new(Registry::default());
10718 let handler = ControlHandler::new(Arc::clone(®istry));
10719 let connection = ConnectionId::new(219);
10720 registry
10721 .register_with_control_ops(
10722 ModuleManifest::builder("invalid-rich-ui", "0.1.0")
10723 .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
10724 "bad schema",
10725 format!("sha256:{}", "a".repeat(64)),
10726 128,
10727 2,
10728 )))
10729 .build(),
10730 PROTOCOL_VERSION,
10731 connection,
10732 Vec::new(),
10733 )
10734 .unwrap();
10735 let request =
10736 Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
10737 let response = handler
10738 .handle_catalog_update(connection, request, Vec::new(), None, None)
10739 .unwrap();
10740 assert_eq!(response[0].header.ty, FrameType::Error);
10741 let error = parse_error(&response[0]);
10742 assert_eq!(error["code"], "invalid_manifest");
10743 assert!(error["message"]
10744 .as_str()
10745 .unwrap()
10746 .contains("rich_ui.schema"));
10747 }
10748
10749 #[test]
10750 fn swap_candidate_hello_is_nonce_verified_after_promotion() {
10751 let registry = Arc::new(Registry::default());
10752 let supervisor = SupervisorHandle::new();
10753 supervisor.open_swap("provider", "candidate-nonce".to_string());
10754 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
10755 let connection = ConnectionId::new(218);
10756 let hello = handler
10757 .handle_control(
10758 connection,
10759 hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
10760 )
10761 .unwrap();
10762 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10763 assert!(registry.list_modules().unwrap().1.is_empty());
10764
10765 registry.promote_candidate("provider").unwrap().unwrap();
10766 let request = Frame::build(
10767 FrameType::Request,
10768 control_flags(),
10769 0,
10770 0,
10771 219,
10772 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10773 )
10774 .unwrap();
10775 let response = handler.handle_catalog_list(request, None).unwrap();
10776 let ClientControlResponse::CatalogList { modules, .. } =
10777 serde_json::from_slice(&response[0].body).unwrap()
10778 else {
10779 panic!("expected catalog.list");
10780 };
10781 assert_eq!(modules.len(), 1);
10782 assert!(modules[0].nonce_verified);
10783 }
10784
10785 #[test]
10786 fn catalog_list_with_provenance_mirrors_declared_build_bytes() {
10787 for provenance_json in [
10788 r#"{"build_git_sha":"0123456789abcdef0123456789abcdef01234567","build_lock_digest":"abababababababababababababababababababababababababababababababab","wire_crate_version":"0.34.0","store_schema_version":"7","launch_nonce_source":"fd"}"#,
10789 r#"{"build_git_sha_absence_reason":"no_git_dir","build_lock_digest":"abababababababababababababababababababababababababababababababab"}"#,
10790 ] {
10791 let registry = Arc::new(Registry::default());
10792 let handler = ControlHandler::new(registry);
10793 let hello = Frame::build(
10794 FrameType::Hello,
10795 control_flags(),
10796 0,
10797 0,
10798 220,
10799 serde_json::to_vec(&ModuleHelloBody {
10800 manifest: ModuleManifest::builder("built", "0.1.0")
10801 .provenance(Some(serde_json::from_str(provenance_json).unwrap()))
10802 .build(),
10803 protocol_ver: PROTOCOL_VERSION,
10804 control_ops: None,
10805 launch_nonce: None,
10806 })
10807 .unwrap(),
10808 )
10809 .unwrap();
10810 assert_eq!(
10811 handler
10812 .handle_control(ConnectionId::new(220), hello)
10813 .unwrap()[0]
10814 .header
10815 .ty,
10816 FrameType::HelloAck
10817 );
10818 let request = Frame::build(
10819 FrameType::Request,
10820 control_flags(),
10821 0,
10822 0,
10823 221,
10824 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10825 )
10826 .unwrap();
10827 let response = handler.handle_catalog_list(request, None).unwrap();
10828 let ClientControlResponse::CatalogList { modules, .. } =
10829 serde_json::from_slice(&response[0].body).unwrap()
10830 else {
10831 panic!("expected catalog.list");
10832 };
10833 assert_eq!(modules.len(), 1);
10834 assert_eq!(
10835 serde_json::to_vec(&modules[0]).unwrap(),
10836 format!(
10837 r#"{{"module_id":"built","ready":true,"module_version":"0.1.0","provenance":{provenance_json},"roles":[],"control_ops":["route.bind","route.status"]}}"#
10838 )
10839 .as_bytes()
10840 );
10841 }
10842 }
10843
10844 #[test]
10845 fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
10846 let registry = Arc::new(Registry::default());
10847 let handler = ControlHandler::new(Arc::clone(®istry));
10848 let hello = Frame::build(
10849 FrameType::Hello,
10850 control_flags(),
10851 0,
10852 0,
10853 220,
10854 serde_json::to_vec(&ModuleHelloBody {
10855 manifest: ModuleManifest::builder("plain", "0.1.0").build(),
10856 protocol_ver: PROTOCOL_VERSION,
10857 control_ops: None,
10858 launch_nonce: Some("unverified".to_string()),
10859 })
10860 .unwrap(),
10861 )
10862 .unwrap();
10863 assert_eq!(
10864 handler
10865 .handle_control(ConnectionId::new(220), hello)
10866 .unwrap()[0]
10867 .header
10868 .ty,
10869 FrameType::HelloAck
10870 );
10871 let request = Frame::build(
10872 FrameType::Request,
10873 control_flags(),
10874 0,
10875 0,
10876 221,
10877 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10878 )
10879 .unwrap();
10880 let response = handler.handle_catalog_list(request, None).unwrap();
10881 let wire: serde_json::Value = serde_json::from_slice(&response[0].body).unwrap();
10882 assert!(wire["modules"][0].get("provenance").is_none());
10883 let ClientControlResponse::CatalogList { modules, .. } =
10884 serde_json::from_slice(&response[0].body).unwrap()
10885 else {
10886 panic!("expected catalog.list");
10887 };
10888 assert_eq!(
10889 serde_json::to_vec(&modules[0]).unwrap(),
10890 br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
10891 );
10892 assert!(!modules[0].nonce_verified);
10893 assert!(modules[0].provenance.is_none());
10894 }
10895
10896 #[test]
10897 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
10898 let registry = Arc::new(Registry::default());
10899 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
10900 [("vault".to_string(), true), ("squatter".to_string(), true)],
10901 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
10902 );
10903 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
10904 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
10905 provides: vec!["credentials-provider/v1".to_string()],
10906 requires: Vec::new(),
10907 must_never_reach: Vec::new(),
10908 });
10909 let frame = Frame::build(
10910 FrameType::Hello,
10911 control_flags(),
10912 0,
10913 0,
10914 77,
10915 serde_json::to_vec(&ModuleHelloBody {
10916 manifest: squatter,
10917 protocol_ver: PROTOCOL_VERSION,
10918 control_ops: None,
10919 launch_nonce: None,
10920 })
10921 .expect("HELLO serializes"),
10922 )
10923 .expect("HELLO frame builds");
10924 let response = handler
10925 .handle_control(ConnectionId::new(77), frame)
10926 .expect("reserved claim receives a typed refusal");
10927 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
10928 assert_eq!(
10929 registry
10930 .active_registration_count()
10931 .expect("registry reads"),
10932 0,
10933 "a reserved capability refusal must not leave a catalog entry"
10934 );
10935 }
10936
10937 #[test]
10938 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
10939 for settlement in ["timeout", "inconclusive", "drop"] {
10940 let breakers = RouteBindBreakers::default();
10941 let RouteBindAdmission::Admitted {
10942 guard: mut old,
10943 probe: false,
10944 } = breakers.admit("prov")
10945 else {
10946 panic!("ordinary relay admitted")
10947 };
10948 let RouteBindAdmission::Admitted {
10949 guard: mut opener, ..
10950 } = breakers.admit("prov")
10951 else {
10952 panic!("second relay admitted")
10953 };
10954 assert!(
10955 !opener
10956 .record_timeout(1, Duration::ZERO)
10957 .unwrap()
10958 .reopened_after_probe
10959 );
10960 let RouteBindAdmission::Admitted {
10961 guard: mut probe,
10962 probe: true,
10963 } = breakers.admit("prov")
10964 else {
10965 panic!("one cooldown probe admitted")
10966 };
10967 match settlement {
10968 "timeout" => assert!(
10969 !old.record_timeout(1, Duration::ZERO)
10970 .unwrap()
10971 .reopened_after_probe
10972 ),
10973 "inconclusive" => old.record_inconclusive(),
10974 "drop" => drop(old),
10975 _ => unreachable!(),
10976 }
10977 assert!(
10978 matches!(
10979 breakers.admit("prov"),
10980 RouteBindAdmission::Refused {
10981 probe_in_flight: true,
10982 ..
10983 }
10984 ),
10985 "{settlement} of a pre-open relay cannot release the real probe"
10986 );
10987 assert!(
10988 probe
10989 .record_timeout(1, Duration::ZERO)
10990 .unwrap()
10991 .reopened_after_probe
10992 );
10993 assert!(matches!(
10994 breakers.admit("prov"),
10995 RouteBindAdmission::Admitted { probe: true, .. }
10996 ));
10997 }
10998 let breakers = RouteBindBreakers::default();
10999 let admit = || match breakers.admit("prov") {
11000 RouteBindAdmission::Admitted { guard, .. } => guard,
11001 _ => panic!("relay admitted"),
11002 };
11003 admit().record_timeout(1, Duration::ZERO);
11004 let mut old_probe = admit();
11005 breakers.reset_for_new_module_connection("prov");
11006 admit().record_timeout(1, Duration::ZERO);
11007 let _new_probe = admit();
11008 old_probe.record_inconclusive();
11009 assert!(matches!(
11010 breakers.admit("prov"),
11011 RouteBindAdmission::Refused {
11012 probe_in_flight: true,
11013 ..
11014 }
11015 ));
11016 }
11017
11018 #[tokio::test]
11019 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
11020 for candidate in [false, true] {
11021 let registry = Arc::new(Registry::default());
11022 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
11023 [("vault".to_string(), true), ("squatter".to_string(), true)],
11024 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
11025 );
11026 let conn = ConnectionId::new(77);
11027 let (ctx, mut rx) = route_ctx(conn);
11028 let initial = capability_manifest("squatter", &[], &[]);
11029 if candidate {
11030 registry
11031 .register_candidate_with_control_ops(
11032 initial.clone(),
11033 PROTOCOL_VERSION,
11034 conn,
11035 module_baseline_control_ops(),
11036 )
11037 .unwrap();
11038 handler
11039 .forwarding
11040 .register_candidate_module_connection(
11041 conn,
11042 "squatter".to_string(),
11043 PROTOCOL_VERSION,
11044 manifest_concurrency(&initial),
11045 ctx.egress.clone(),
11046 )
11047 .unwrap();
11048 } else {
11049 hello_via_sink(
11050 &handler,
11051 &ctx,
11052 &mut rx,
11053 hello_frame_with_manifest(initial.clone(), 1),
11054 )
11055 .await;
11056 }
11057 let response = handler
11058 .handle_control_frame(
11059 &ctx,
11060 catalog_update_with_capabilities_frame(
11061 2,
11062 capability_manifest("squatter", &["credentials-provider/v1"], &[])
11063 .capabilities
11064 .unwrap(),
11065 ),
11066 )
11067 .await
11068 .unwrap();
11069 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
11070 assert_eq!(
11071 registry
11072 .get_module_by_connection(conn)
11073 .unwrap()
11074 .unwrap()
11075 .manifest,
11076 initial
11077 );
11078 }
11079 }
11080
11081 #[test]
11082 fn server_describe_surfaces_required_capability_verdict_fields() {
11083 let registry = Arc::new(Registry::default());
11084 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
11085 [
11086 ("consumer".to_string(), true),
11087 ("provider".to_string(), false),
11088 ],
11089 BTreeMap::new(),
11090 );
11091 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
11092 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
11093 provides: Vec::new(),
11094 requires: vec![subc_protocol::manifest::CapabilityRequirement {
11095 capability: "credentials-provider/v1".to_string(),
11096 need: subc_protocol::manifest::CapabilityNeed::Required,
11097 }],
11098 must_never_reach: Vec::new(),
11099 });
11100 let hello = Frame::build(
11101 FrameType::Hello,
11102 control_flags(),
11103 0,
11104 0,
11105 78,
11106 serde_json::to_vec(&ModuleHelloBody {
11107 manifest: consumer,
11108 protocol_ver: PROTOCOL_VERSION,
11109 control_ops: None,
11110 launch_nonce: None,
11111 })
11112 .expect("HELLO serializes"),
11113 )
11114 .expect("HELLO frame builds");
11115 handler
11116 .handle_control(ConnectionId::new(78), hello)
11117 .expect("consumer registers");
11118 let describe = Frame::build(
11119 FrameType::Request,
11120 control_flags(),
11121 0,
11122 0,
11123 79,
11124 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
11125 .expect("request serializes"),
11126 )
11127 .expect("describe frame builds");
11128 let response = handler
11129 .handle_server_describe(describe)
11130 .expect("server.describe succeeds");
11131 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
11132 let requirement = &rendered["capability_requirements"][0];
11133 assert_eq!(requirement["consumer"], "consumer");
11134 assert_eq!(requirement["verdict"], "never_provided");
11135 assert_eq!(requirement["episode_seq"], 1);
11136 assert_eq!(requirement["config_satisfiable"], false);
11137 assert_eq!(requirement["runtime_available"], false);
11138 assert!(requirement["detail"]
11139 .as_str()
11140 .expect("detail string")
11141 .contains("credentials-provider/v1"));
11142 }
11143
11144 fn deregister_required_provider(
11150 daemon_shutdown_begun: bool,
11151 ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
11152 const CAPABILITY: &str = "project-identity/v1";
11153 let registry = Arc::new(Registry::default());
11154 let daemon_shutdown = DaemonShutdownFlag::default();
11155 let handler = ControlHandler::new(Arc::clone(®istry))
11156 .with_capability_config(
11157 [
11158 ("consumer".to_string(), true),
11159 ("provider".to_string(), true),
11160 ],
11161 BTreeMap::new(),
11162 )
11163 .with_daemon_shutdown_flag(daemon_shutdown.clone());
11164 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
11165 consumer.capabilities = Some(CapabilityDeclarations {
11166 provides: Vec::new(),
11167 requires: vec![subc_protocol::manifest::CapabilityRequirement {
11168 capability: CAPABILITY.to_string(),
11169 need: subc_protocol::manifest::CapabilityNeed::Required,
11170 }],
11171 must_never_reach: Vec::new(),
11172 });
11173 let provider = capability_manifest("provider", &[CAPABILITY], &[]);
11174 let provider_conn = ConnectionId::new(182);
11175 for (conn, manifest, corr) in [
11176 (ConnectionId::new(181), consumer, 181),
11177 (provider_conn, provider, 182),
11178 ] {
11179 let hello = handler
11180 .handle_control(conn, hello_frame_with_manifest(manifest, corr))
11181 .expect("module registers");
11182 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
11183 }
11184 assert_eq!(
11185 handler.capability_evaluator.verdict("consumer", CAPABILITY),
11186 Some(CapabilityVerdict::Provided),
11187 "the registered provider must satisfy the requirement first"
11188 );
11189 if daemon_shutdown_begun {
11190 daemon_shutdown.set_for_test();
11191 }
11192
11193 let capture = EventCapture::default();
11194 let _subscriber =
11195 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11196 handler
11197 .cleanup_connection(provider_conn)
11198 .expect("provider deregisters");
11199 let events = capture
11200 .events()
11201 .into_iter()
11202 .filter(|event| {
11203 event.fields.get("message").map(String::as_str) == Some("capability.requirement")
11204 })
11205 .collect();
11206 (
11207 events,
11208 handler.capability_evaluator.verdict("consumer", CAPABILITY),
11209 )
11210 }
11211
11212 #[test]
11213 fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
11214 let (events, verdict) = deregister_required_provider(true);
11215 assert_eq!(
11216 verdict,
11217 Some(CapabilityVerdict::NeverProvided),
11218 "shutdown must not change the verdict route.open refusals read"
11219 );
11220 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
11221 let event = &events[0];
11222 assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
11223 assert_eq!(
11224 event.fields.get("cause").map(String::as_str),
11225 Some("\"daemon_shutdown\""),
11226 "{event:?}"
11227 );
11228 assert_eq!(
11229 event.fields.get("verdict").map(String::as_str),
11230 Some("\"never_provided\""),
11231 "{event:?}"
11232 );
11233 }
11234
11235 #[test]
11236 fn required_provider_leaving_while_daemon_runs_still_logs_error() {
11237 let (events, verdict) = deregister_required_provider(false);
11238 assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
11239 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
11240 let event = &events[0];
11241 assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
11242 assert_eq!(event.fields.get("cause"), None, "{event:?}");
11243 assert_eq!(
11244 event.fields.get("verdict").map(String::as_str),
11245 Some("\"never_provided\""),
11246 "{event:?}"
11247 );
11248 }
11249
11250 #[test]
11251 fn catalog_list_omits_capabilities_for_legacy_manifest() {
11252 let registry = Arc::new(Registry::default());
11253 let handler = ControlHandler::new(Arc::clone(®istry));
11254 let hello = handler
11255 .handle_control(
11256 ConnectionId::new(101),
11257 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
11258 )
11259 .expect("legacy HELLO registers");
11260 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
11261
11262 let request = Frame::build(
11263 FrameType::Request,
11264 control_flags(),
11265 0,
11266 0,
11267 102,
11268 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
11269 .expect("catalog request serializes"),
11270 )
11271 .expect("catalog request frame builds");
11272 let response = handler
11273 .handle_catalog_list(request, None)
11274 .expect("catalog list succeeds");
11275 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
11276 assert!(
11277 body["modules"][0].get("capabilities").is_none(),
11278 "legacy manifest must retain an absent capabilities field on catalog.list"
11279 );
11280 }
11281
11282 #[test]
11283 fn hello_ack_omits_storage_when_no_storage_config() {
11284 let registry = Arc::new(Registry::default());
11285 let handler = ControlHandler::new(Arc::clone(®istry));
11286 let responses = handler
11287 .handle_control(
11288 ConnectionId::new(1),
11289 hello_frame("aft", PROTOCOL_VERSION, 7),
11290 )
11291 .unwrap();
11292 let ack = parse_ack(&responses[0]);
11293 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
11294 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
11295 }
11296
11297 #[tokio::test]
11298 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
11299 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
11300 let registry = Arc::new(Registry::default());
11301 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
11302 let responses = handler
11303 .handle_control(
11304 ConnectionId::new(1),
11305 hello_frame("aft", PROTOCOL_VERSION, 7),
11306 )
11307 .unwrap();
11308 let ack = parse_ack(&responses[0]);
11309 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
11310
11311 let described = handler
11312 .handle_control_frame(
11313 &route_ctx(ConnectionId::new(2)).0,
11314 Frame::build(
11315 FrameType::Request,
11316 control_flags(),
11317 0,
11318 0,
11319 9,
11320 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
11321 )
11322 .unwrap(),
11323 )
11324 .await
11325 .unwrap();
11326 let ClientControlResponse::ServerDescribe { machine_id, .. } =
11327 serde_json::from_slice(&described[0].body).unwrap()
11328 else {
11329 panic!("server.describe answered with another shape");
11330 };
11331 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
11332 }
11333
11334 #[test]
11335 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
11336 let registry = Arc::new(Registry::default());
11339 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
11340 crate::daemon_config::StorageConfig::Sqlite {
11341 data_home: std::path::PathBuf::from("/data"),
11342 },
11343 ));
11344
11345 let responses = handler
11346 .handle_control(
11347 ConnectionId::new(1),
11348 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
11349 )
11350 .unwrap();
11351 let ack = parse_ack(&responses[0]);
11352 assert_eq!(
11353 ack.storage,
11354 Some(serde_json::json!({
11355 "module_id": "alfonso-routing",
11356 "storage_namespace": "default",
11357 "isolation": { "kind": "module" },
11358 "backend": {
11359 "backend": "sqlite",
11360 "path": "/data/cortexkit/alfonso-routing/store.db"
11361 }
11362 })),
11363 "the delivered descriptor is the module's own sqlite store path"
11364 );
11365 }
11366
11367 #[test]
11368 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
11369 let registry = Arc::new(Registry::default());
11370 let handler = ControlHandler::new(Arc::clone(®istry));
11371 let conn = ConnectionId::new(1);
11372 let responses = handler
11373 .handle_control(
11374 conn,
11375 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
11376 )
11377 .unwrap();
11378 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
11379 let registration = registry.get_module("aft").unwrap().unwrap();
11380 assert_eq!(registration.control_ops, module_baseline_control_ops());
11381
11382 let frame =
11383 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
11384 assert!(handler
11385 .guard_module_control_op(&frame, "aft", "route.bind")
11386 .unwrap()
11387 .is_none());
11388 let error = handler
11389 .guard_module_control_op(&frame, "aft", "test.synthetic")
11390 .unwrap()
11391 .expect("synthetic ungranted op should be rejected");
11392 assert_eq!(error.header.ty, FrameType::Error);
11393 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
11394 }
11395
11396 #[test]
11397 fn hello_control_ops_some_adds_optional_grants() {
11398 let registry = Arc::new(Registry::default());
11399 let handler = ControlHandler::new(Arc::clone(®istry));
11400 handler
11401 .handle_control(
11402 ConnectionId::new(1),
11403 hello_frame_with_control_ops(
11404 "aft",
11405 PROTOCOL_VERSION,
11406 7,
11407 Some(vec![
11408 "future.synthetic".to_string(),
11409 "route.bind".to_string(),
11410 ]),
11411 ),
11412 )
11413 .unwrap();
11414 let registration = registry.get_module("aft").unwrap().unwrap();
11415 assert_eq!(
11416 registration.control_ops,
11417 vec![
11418 "route.bind".to_string(),
11419 "route.status".to_string(),
11420 "future.synthetic".to_string(),
11421 ]
11422 );
11423 let frame =
11424 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
11425 assert!(handler
11426 .guard_module_control_op(&frame, "aft", "future.synthetic")
11427 .unwrap()
11428 .is_none());
11429 }
11430
11431 #[tokio::test]
11432 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
11433 let registry = Arc::new(Registry::default());
11434 let forwarding = Arc::new(ForwardingTable::default());
11435 let handler =
11436 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11437 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
11438 hello_via_sink(
11439 &handler,
11440 &module_ctx,
11441 &mut module_rx,
11442 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
11443 )
11444 .await;
11445
11446 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
11447 let responses = handler
11448 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
11449 .await
11450 .unwrap();
11451 assert_eq!(responses.len(), 1);
11452 assert_eq!(responses[0].header.ty, FrameType::Error);
11453 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
11454 assert!(module_rx.try_recv().is_err());
11455 }
11456
11457 #[tokio::test]
11458 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
11459 let registry = Arc::new(Registry::default());
11460 let forwarding = Arc::new(ForwardingTable::default());
11461 let handler =
11462 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11463 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
11464 hello_via_sink(
11465 &handler,
11466 &module_ctx,
11467 &mut module_rx,
11468 hello_frame_with_control_ops(
11469 "aft",
11470 PROTOCOL_VERSION,
11471 7,
11472 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11473 ),
11474 )
11475 .await;
11476
11477 let project_root = unique_project_root("demux");
11478 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
11479 let route_handler = handler.clone();
11480 let route_task = tokio::spawn(async move {
11481 route_handler
11482 .handle_control_frame(
11483 &route_client_ctx,
11484 route_open_frame(100, "aft", project_root),
11485 )
11486 .await
11487 .unwrap()
11488 });
11489 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11490 .await
11491 .unwrap()
11492 .unwrap();
11493 assert!(matches!(
11494 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
11495 ModuleControlRequest::RouteBind { .. }
11496 ));
11497
11498 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
11499 let health_handler = handler.clone();
11500 let health_task = tokio::spawn(async move {
11501 health_handler
11502 .handle_control_frame(
11503 &health_client_ctx,
11504 supervisor_health_probe_frame(101, "aft"),
11505 )
11506 .await
11507 .unwrap()
11508 });
11509 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11510 .await
11511 .unwrap()
11512 .unwrap();
11513 assert_eq!(
11514 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
11515 ModuleControlRequest::HealthCheck {}
11516 );
11517
11518 handler
11519 .handle_control_frame(
11520 &module_ctx,
11521 health_response(health_frame.header.corr, HealthStatus::Degraded),
11522 )
11523 .await
11524 .unwrap();
11525 let health_response = health_task.await.unwrap();
11526 assert_eq!(health_response.len(), 1);
11527 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
11528 ClientControlResponse::SupervisorHealthProbe {
11529 module_id,
11530 status,
11531 detail,
11532 metrics,
11533 } => {
11534 assert_eq!(module_id, "aft");
11535 assert_eq!(status, HealthStatus::Degraded);
11536 assert_eq!(detail.as_deref(), Some("warming"));
11537 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
11538 }
11539 other => panic!("unexpected health response: {other:?}"),
11540 }
11541
11542 handler
11543 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11544 .await
11545 .unwrap();
11546 let route_response = route_task.await.unwrap();
11547 assert!(route_response.is_empty());
11548 let published = route_client_rx.recv().await.unwrap();
11549 assert!(matches!(
11550 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11551 ClientControlResponse::RouteOpen { .. }
11552 ));
11553 }
11554
11555 async fn relay_route_open(
11560 handler: &ControlHandler,
11561 client_connection: ConnectionId,
11562 client_egress: &FrameSink,
11563 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
11564 corr: u64,
11565 module_id: &str,
11566 project_root_label: &str,
11567 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
11568 let ctx = RouteCtx {
11569 connection_id: client_connection,
11570 egress: client_egress.clone(),
11571 };
11572 let handler = handler.clone();
11573 let project_root = unique_project_root(project_root_label);
11574 let module_id = module_id.to_string();
11575 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
11576 let task = tokio::spawn(async move {
11577 let _guard = tracing::dispatcher::set_default(&dispatch);
11578 handler
11579 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
11580 .await
11581 .unwrap()
11582 });
11583 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
11584 .await
11585 .expect("module receives the relayed route.bind")
11586 .expect("module egress is open");
11587 (task, bind.frame)
11588 }
11589
11590 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
11591 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
11592 ModuleControlRequest::RouteBind {
11593 route_channel,
11594 epoch,
11595 ..
11596 } => (route_channel, epoch),
11597 other => panic!("expected a route.bind request, got {other:?}"),
11598 }
11599 }
11600
11601 fn published_route(frame: &Frame) -> (u16, u32) {
11602 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
11603 ClientControlResponse::RouteOpen {
11604 route_channel,
11605 route_epoch,
11606 ..
11607 } => (route_channel, route_epoch),
11608 other => panic!("expected a route.open response, got {other:?}"),
11609 }
11610 }
11611
11612 #[tokio::test]
11632 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
11633 let registry = Arc::new(Registry::default());
11634 let forwarding = Arc::new(ForwardingTable::default());
11635 let handler =
11636 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11637
11638 let module_connection = ConnectionId::new(30);
11639 let (module_ctx, mut module_rx) = route_ctx(module_connection);
11640 hello_via_sink(
11641 &handler,
11642 &module_ctx,
11643 &mut module_rx,
11644 hello_frame("aft", PROTOCOL_VERSION, 7),
11645 )
11646 .await;
11647
11648 let dying_client = ConnectionId::new(31);
11649 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
11650
11651 let (first_task, first_bind) = relay_route_open(
11654 &handler,
11655 dying_client,
11656 &dying_ctx.egress,
11657 &mut module_rx,
11658 100,
11659 "aft",
11660 "closing-first",
11661 )
11662 .await;
11663 handler
11664 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
11665 .await
11666 .unwrap();
11667 assert!(first_task.await.unwrap().is_empty());
11668 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
11669
11670 let (second_task, second_bind) = relay_route_open(
11672 &handler,
11673 dying_client,
11674 &dying_ctx.egress,
11675 &mut module_rx,
11676 101,
11677 "aft",
11678 "closing-second",
11679 )
11680 .await;
11681 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
11682
11683 assert!(forwarding
11686 .escalate_client_delivery_failure(
11687 dying_client,
11688 first_channel,
11689 first_epoch,
11690 CloseReason::new(
11691 "module_to_client_delivery_failed",
11692 "client egress refused a module frame",
11693 ),
11694 crate::forwarding::UndeliveredFrame {
11695 module_id: None,
11696 sink: &dying_ctx.egress,
11697 },
11698 )
11699 .unwrap());
11700 assert!(!dying_ctx.egress.is_closed());
11701
11702 let ack = handler
11704 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
11705 .await;
11706 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
11707 if module_loop_error.is_some() {
11708 handler.cleanup_connection(module_connection).unwrap();
11712 }
11713 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11717 .await
11718 .ok()
11719 .flatten();
11720
11721 assert!(
11723 registry
11724 .get_module_by_connection(module_connection)
11725 .unwrap()
11726 .is_some(),
11727 "one client's closing connection ended the shared module connection: \
11728 {module_loop_error:?}"
11729 );
11730 let cotenant = ConnectionId::new(32);
11732 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
11733 let (cotenant_task, cotenant_bind) = relay_route_open(
11734 &handler,
11735 cotenant,
11736 &cotenant_ctx.egress,
11737 &mut module_rx,
11738 102,
11739 "aft",
11740 "closing-cotenant",
11741 )
11742 .await;
11743 handler
11744 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
11745 .await
11746 .unwrap();
11747 assert!(cotenant_task.await.unwrap().is_empty());
11748 let (cotenant_channel, cotenant_epoch) =
11749 published_route(&cotenant_rx.recv().await.unwrap());
11750 assert!(matches!(
11751 forwarding
11752 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
11753 .unwrap(),
11754 DataRoute::Client(DataRouteState::Bound(_))
11755 ));
11756
11757 let goodbye = post_ack_module_frame
11760 .expect("module receives a GOODBYE for the abandoned route channel");
11761 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11762 assert_eq!(goodbye.header.channel, abandoned_channel);
11763 assert_eq!(goodbye.header.epoch, abandoned_epoch);
11764
11765 assert!(dying_rx.try_recv().is_err());
11769 let second_response = second_task.await.unwrap();
11770 assert_eq!(second_response.len(), 1);
11771 assert_eq!(
11772 parse_error(&second_response[0])["code"],
11773 "target_unavailable"
11774 );
11775 }
11776
11777 async fn catalog_snapshot(
11780 handler: &ControlHandler,
11781 client_ctx: &RouteCtx,
11782 corr: u64,
11783 ) -> (u64, Vec<(String, bool)>) {
11784 let body =
11785 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
11786 let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
11787 let reply = handler
11788 .handle_control_frame(client_ctx, request)
11789 .await
11790 .unwrap()
11791 .pop()
11792 .unwrap();
11793 let ClientControlResponse::CatalogList {
11794 generation,
11795 modules,
11796 ..
11797 } = serde_json::from_slice(&reply.body).unwrap()
11798 else {
11799 panic!("not a catalog.list reply");
11800 };
11801 (
11802 generation,
11803 modules
11804 .into_iter()
11805 .map(|entry| (entry.module_id, entry.ready))
11806 .collect(),
11807 )
11808 }
11809
11810 fn hello_held_before_forwarding() -> (
11816 ControlHandler,
11817 Arc<std::sync::Barrier>,
11818 std::thread::JoinHandle<Vec<Frame>>,
11819 RouteCtx,
11820 mpsc::Receiver<crate::router::OutboundFrame>,
11821 ) {
11822 let hold = Arc::new(std::sync::Barrier::new(2));
11823 let handler = ControlHandler::with_forwarding(
11824 Arc::new(Registry::default()),
11825 Arc::new(ForwardingTable::default()),
11826 )
11827 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
11828 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
11829 let hello = {
11830 let handler = handler.clone();
11831 let module_ctx = module_ctx.clone();
11832 std::thread::spawn(move || {
11833 handler
11834 .handle_hello(
11835 module_ctx.connection_id,
11836 Some(module_ctx.egress.clone()),
11837 hello_frame("aft", PROTOCOL_VERSION, 7),
11838 )
11839 .unwrap()
11840 })
11841 };
11842 (handler, hold, hello, module_ctx, module_rx)
11843 }
11844
11845 #[tokio::test]
11854 async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
11855 let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
11856 let client = ConnectionId::new(41);
11857 let (client_ctx, mut client_rx) = route_ctx(client);
11858
11859 hold.wait();
11860 assert!(handler.registry.get_module("aft").unwrap().is_some());
11862 assert!(!handler
11863 .forwarding
11864 .has_live_module_connection("aft")
11865 .unwrap());
11866 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
11867 assert!(
11868 listed.is_empty(),
11869 "catalog.list listed a module route.open cannot reach yet: {listed:?}"
11870 );
11871 let refused = handler
11872 .handle_control_frame(
11873 &client_ctx,
11874 route_open_frame(301, "aft", unique_project_root("listed-window")),
11875 )
11876 .await
11877 .unwrap();
11878 assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
11879 let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
11880 let duplicate = handler
11881 .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
11882 .await
11883 .unwrap();
11884 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
11885
11886 hold.wait();
11887 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
11888 let ack = module_rx.try_recv().unwrap().frame;
11889 assert_eq!(ack.header.ty, FrameType::HelloAck);
11890 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
11891 assert_eq!(listed, vec![("aft".to_string(), true)]);
11892
11893 let (open_task, bind) = relay_route_open(
11894 &handler,
11895 client,
11896 &client_ctx.egress,
11897 &mut module_rx,
11898 303,
11899 "aft",
11900 "listed-after",
11901 )
11902 .await;
11903 assert!(handler
11904 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11905 .await
11906 .unwrap()
11907 .is_empty());
11908 assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
11909 published_route(&client_rx.try_recv().unwrap().frame);
11910 }
11911
11912 #[tokio::test]
11919 async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
11920 let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
11921 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11922 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
11923
11924 hold.wait();
11925 let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
11926 assert!(listed.is_empty());
11927 assert_eq!(
11928 during, before,
11929 "the generation moved while the module was not listed"
11930 );
11931
11932 hold.wait();
11933 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
11934 let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
11935 assert_eq!(listed, vec![("aft".to_string(), true)]);
11936 assert_eq!(
11937 after,
11938 before + 1,
11939 "the module appeared without exactly one generation bump"
11940 );
11941 }
11942
11943 #[tokio::test]
11948 async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
11949 let registry = Arc::new(Registry::default());
11950 let handler = ControlHandler::with_forwarding(
11951 Arc::clone(®istry),
11952 Arc::new(ForwardingTable::default()),
11953 );
11954 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11955 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
11956 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
11957 drop(module_rx);
11958
11959 let replies = handler
11960 .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
11961 .await
11962 .unwrap();
11963 assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
11964 assert!(registry.get_module("aft").unwrap().is_none());
11965 assert!(registry
11966 .get_module_by_connection(module_ctx.connection_id)
11967 .unwrap()
11968 .is_none());
11969 assert_eq!(
11970 catalog_snapshot(&handler, &client_ctx, 301).await,
11971 (before, Vec::new())
11972 );
11973 }
11974
11975 async fn route_open_held_after_bind_send() -> (
11979 ControlHandler,
11980 Arc<tokio::sync::Barrier>,
11981 RouteCtx,
11982 mpsc::Receiver<crate::router::OutboundFrame>,
11983 ) {
11984 let hold = Arc::new(tokio::sync::Barrier::new(2));
11985 let handler = ControlHandler::with_forwarding(
11986 Arc::new(Registry::default()),
11987 Arc::new(ForwardingTable::default()),
11988 )
11989 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
11990 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
11991 hello_via_sink(
11992 &handler,
11993 &module_ctx,
11994 &mut module_rx,
11995 hello_frame("aft", PROTOCOL_VERSION, 7),
11996 )
11997 .await;
11998 (handler, hold, module_ctx, module_rx)
11999 }
12000
12001 #[tokio::test]
12008 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
12009 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
12010 let client = ConnectionId::new(41);
12011 let (client_ctx, mut client_rx) = route_ctx(client);
12012
12013 let (open_task, bind) = relay_route_open(
12014 &handler,
12015 client,
12016 &client_ctx.egress,
12017 &mut module_rx,
12018 200,
12019 "aft",
12020 "acked-before-mark",
12021 )
12022 .await;
12023 let (module_channel, module_epoch) = route_bind_channel(&bind);
12024 hold.wait().await;
12026
12027 assert!(handler
12029 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
12030 .await
12031 .unwrap()
12032 .is_empty());
12033 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
12034
12035 hold.wait().await;
12036 assert!(
12037 open_task.await.unwrap().is_empty(),
12038 "route.open answers accepted"
12039 );
12040
12041 match module_rx.try_recv() {
12044 Err(mpsc::error::TryRecvError::Empty) => {}
12045 Ok(outbound) => panic!(
12046 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
12047 outbound.frame.header.ty,
12048 outbound.frame.header.channel,
12049 outbound.frame.header.epoch
12050 ),
12051 Err(err) => panic!("module egress closed: {err}"),
12052 }
12053
12054 let router = Router::with_control_handler(Arc::new(handler.clone()));
12057 let request = Frame::build(
12058 FrameType::Request,
12059 Flags::new(true, Priority::Interactive, false),
12060 client_channel,
12061 client_epoch,
12062 201,
12063 b"first request".to_vec(),
12064 )
12065 .unwrap();
12066 router
12067 .route_for_connection(&client_ctx, request)
12068 .await
12069 .unwrap();
12070 let delivered = module_rx.try_recv().unwrap().frame;
12071 assert_eq!(delivered.header.ty, FrameType::Request);
12072 assert_eq!(
12073 (delivered.header.channel, delivered.header.epoch),
12074 (module_channel, module_epoch)
12075 );
12076 assert_eq!(delivered.body, b"first request");
12077 }
12078
12079 #[tokio::test]
12085 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
12086 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
12087 let client = ConnectionId::new(42);
12088 let (client_ctx, _client_rx) = route_ctx(client);
12089
12090 let (open_task, bind) = relay_route_open(
12091 &handler,
12092 client,
12093 &client_ctx.egress,
12094 &mut module_rx,
12095 300,
12096 "aft",
12097 "released-before-mark",
12098 )
12099 .await;
12100 let (module_channel, module_epoch) = route_bind_channel(&bind);
12101 hold.wait().await;
12102
12103 handler.cleanup_connection(client).unwrap();
12104 assert!(
12105 module_rx.try_recv().is_err(),
12106 "client teardown must not GOODBYE a bind route.open is still queueing"
12107 );
12108
12109 hold.wait().await;
12110 let response = open_task.await.unwrap();
12111 assert_eq!(response.len(), 1);
12112 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12113
12114 let goodbye = module_rx
12115 .try_recv()
12116 .expect("module is told to drop the binding the daemon abandoned")
12117 .frame;
12118 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
12119 assert_eq!(
12120 (goodbye.header.channel, goodbye.header.epoch),
12121 (module_channel, module_epoch)
12122 );
12123 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
12124 }
12125
12126 #[tokio::test]
12131 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
12132 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
12133 let client = ConnectionId::new(43);
12134 let (client_ctx, client_rx) = route_ctx(client);
12135
12136 let (open_task, bind) = relay_route_open(
12137 &handler,
12138 client,
12139 &client_ctx.egress,
12140 &mut module_rx,
12141 400,
12142 "aft",
12143 "gone-client-before-mark",
12144 )
12145 .await;
12146 let (module_channel, module_epoch) = route_bind_channel(&bind);
12147 hold.wait().await;
12148
12149 drop(client_rx);
12150 assert!(client_ctx.egress.is_closed());
12151 handler
12152 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
12153 .await
12154 .unwrap();
12155
12156 hold.wait().await;
12157 let response = open_task.await.unwrap();
12158 assert_eq!(response.len(), 1);
12159 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12160
12161 let goodbye = module_rx
12162 .try_recv()
12163 .expect("module is told to drop the binding it acked")
12164 .frame;
12165 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
12166 assert_eq!(
12167 (goodbye.header.channel, goodbye.header.epoch),
12168 (module_channel, module_epoch)
12169 );
12170 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
12171 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
12172 }
12173
12174 #[test]
12181 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
12182 let handler = ControlHandler::default();
12183 let module_connection = ConnectionId::new(30);
12184 let client_connection = ConnectionId::new(31);
12185
12186 handler
12187 .refuse_to_end_module_connection_for_a_client(
12188 module_connection,
12189 77,
12190 ForwardingError::ConnectionClosing {
12191 connection_id: client_connection,
12192 },
12193 )
12194 .expect("a closing client must never end the module connection");
12195
12196 assert!(matches!(
12197 handler.refuse_to_end_module_connection_for_a_client(
12198 module_connection,
12199 78,
12200 ForwardingError::ConnectionClosing {
12201 connection_id: module_connection,
12202 },
12203 ),
12204 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
12205 connection_id
12206 })) if connection_id == module_connection
12207 ));
12208 assert!(matches!(
12209 handler.refuse_to_end_module_connection_for_a_client(
12210 module_connection,
12211 79,
12212 ForwardingError::Poisoned,
12213 ),
12214 Err(RouterError::Forwarding(ForwardingError::Poisoned))
12215 ));
12216 assert!(matches!(
12217 handler.refuse_to_end_module_connection_for_a_client(
12218 module_connection,
12219 80,
12220 ForwardingError::StaleModuleEndpoint,
12221 ),
12222 Err(RouterError::Forwarding(
12223 ForwardingError::StaleModuleEndpoint
12224 ))
12225 ));
12226 }
12227
12228 #[tokio::test]
12241 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
12242 let registry = Arc::new(Registry::default());
12243 let forwarding = Arc::new(ForwardingTable::default());
12244 let supervisor = SupervisorHandle::new();
12245 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12246 let handler =
12247 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12248 .with_supervisor(supervisor);
12249
12250 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
12251 hello_via_sink(
12252 &handler,
12253 &target_ctx,
12254 &mut target_rx,
12255 hello_frame("target", PROTOCOL_VERSION, 1),
12256 )
12257 .await;
12258
12259 let wrong_nonce = handler
12263 .handle_control_frame(
12264 &route_ctx(ConnectionId::new(91)).0,
12265 route_open_frame_with_admission_facts(
12266 20,
12267 "target",
12268 unique_project_root("admission-facts"),
12269 Some(subc_control::ConsumerIdentity {
12270 module_id: "fed".to_string(),
12271 launch_nonce: "not-the-real-nonce".to_string(),
12272 }),
12273 None,
12274 ),
12275 )
12276 .await
12277 .unwrap();
12278 assert_eq!(
12279 parse_error(&wrong_nonce[0])["code"],
12280 "bad_consumer_identity",
12281 "a mismatched launch nonce must be refused, not stamped Reserved"
12282 );
12283
12284 let never_spawned = handler
12288 .handle_control_frame(
12289 &route_ctx(ConnectionId::new(92)).0,
12290 route_open_frame_with_admission_facts(
12291 21,
12292 "target",
12293 unique_project_root("admission-facts"),
12294 Some(subc_control::ConsumerIdentity {
12295 module_id: "never-spawned".to_string(),
12296 launch_nonce: "any-nonce".to_string(),
12297 }),
12298 None,
12299 ),
12300 )
12301 .await
12302 .unwrap();
12303 assert_eq!(
12304 parse_error(&never_spawned[0])["code"],
12305 "bad_consumer_identity",
12306 "an unspawned module_id must be refused rather than accepted for lack of a record"
12307 );
12308 }
12309
12310 #[tokio::test]
12333 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
12334 let registry = Arc::new(Registry::default());
12335 let forwarding = Arc::new(ForwardingTable::default());
12336 let supervisor = SupervisorHandle::new();
12337 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12338 let handler =
12339 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12340 .with_supervisor(supervisor);
12341
12342 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
12343 hello_via_sink(
12344 &handler,
12345 &target_ctx,
12346 &mut target_rx,
12347 hello_frame("target", PROTOCOL_VERSION, 1),
12348 )
12349 .await;
12350
12351 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
12352 let route_handler = handler.clone();
12353 let route_task = tokio::spawn(async move {
12354 route_handler
12355 .handle_control_frame(
12356 &client_ctx,
12357 route_open_frame_with_admission_facts(
12358 30,
12359 "target",
12360 unique_project_root("admission-facts"),
12361 Some(subc_control::ConsumerIdentity {
12362 module_id: "fed".to_string(),
12363 launch_nonce: "fed-nonce".to_string(),
12364 }),
12365 None,
12366 ),
12367 )
12368 .await
12369 .unwrap()
12370 });
12371
12372 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12379 .await
12380 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
12381 .expect("module control channel closed before route.bind");
12382 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12383 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
12384 panic!("expected route.bind")
12385 };
12386 assert_eq!(
12387 principal,
12388 Some(Principal::Reserved {
12389 module_id: "fed".to_string()
12390 }),
12391 "a correctly attested consumer must be stamped Reserved for its own id"
12392 );
12393
12394 handler
12395 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12396 .await
12397 .unwrap();
12398 assert!(route_task.await.unwrap().is_empty());
12399 assert!(
12400 matches!(
12401 serde_json::from_slice::<ClientControlResponse>(
12402 &client_rx.recv().await.unwrap().body
12403 )
12404 .unwrap(),
12405 ClientControlResponse::RouteOpen { .. }
12406 ),
12407 "the route must actually open, not merely avoid an error"
12408 );
12409 }
12410
12411 #[tokio::test(start_paused = true)]
12412 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
12413 let registry = Arc::new(Registry::default());
12414 let forwarding = Arc::new(ForwardingTable::default());
12415 let supervisor = SupervisorHandle::new();
12416 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12417 let handler =
12418 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12419 .with_supervisor(supervisor);
12420
12421 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
12422 hello_via_sink(
12423 &handler,
12424 &target_ctx,
12425 &mut target_rx,
12426 hello_frame("target", PROTOCOL_VERSION, 1),
12427 )
12428 .await;
12429
12430 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
12431 let direct_handler = handler.clone();
12432 let direct_open = tokio::spawn(async move {
12433 direct_handler
12434 .handle_control_frame(
12435 &direct_ctx,
12436 route_open_frame(2, "target", unique_project_root("route-census-direct")),
12437 )
12438 .await
12439 .unwrap()
12440 });
12441 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12442 .await
12443 .expect("no direct route.bind within 5s")
12444 .expect("target control channel closed before direct route.bind");
12445 handler
12446 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
12447 .await
12448 .unwrap();
12449 assert!(direct_open.await.unwrap().is_empty());
12450 let _ = direct_rx.recv().await.unwrap();
12451
12452 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
12453 let reserved_handler = handler.clone();
12454 let reserved_open = tokio::spawn(async move {
12455 reserved_handler
12456 .handle_control_frame(
12457 &reserved_ctx,
12458 route_open_frame_with_admission_facts(
12459 3,
12460 "target",
12461 unique_project_root("admission-facts"),
12462 Some(ConsumerIdentity {
12463 module_id: "fed".to_string(),
12464 launch_nonce: "fed-nonce".to_string(),
12465 }),
12466 None,
12467 ),
12468 )
12469 .await
12470 .unwrap()
12471 });
12472 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12473 .await
12474 .expect("no reserved route.bind within 5s")
12475 .expect("target control channel closed before reserved route.bind");
12476 handler
12477 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
12478 .await
12479 .unwrap();
12480 assert!(reserved_open.await.unwrap().is_empty());
12481 let _ = reserved_rx.recv().await.unwrap();
12482
12483 forwarding
12484 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
12485 .unwrap();
12486 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
12487 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
12488 module_id: Some("target".to_string()),
12489 })
12490 .unwrap();
12491 let census_frame =
12492 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
12493 let response = handler
12494 .handle_control_frame(&census_ctx, census_frame)
12495 .await
12496 .unwrap()
12497 .pop()
12498 .unwrap();
12499 let actual: Value = serde_json::from_slice(&response.body).unwrap();
12500 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
12501 assert!(matches!(
12502 decoded,
12503 ClientControlResponse::SupervisorRoutes { .. }
12504 ));
12505 let routes = actual["modules"][0]["routes"].as_array().unwrap();
12506 assert_eq!(routes.len(), 2);
12507 assert!(routes.iter().all(|route| route["draining"] == true));
12508 assert!(
12511 routes.iter().all(|route| route["drain_reason"] == "reload"),
12512 "draining routes must name the drain's reason: {routes:?}"
12513 );
12514 assert!(routes.iter().any(|route| {
12515 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
12516 }));
12517 assert!(routes.iter().any(|route| {
12518 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
12519 }));
12520
12521 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
12522 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
12523 if std::env::var_os("UPDATE_GOLDEN").is_some() {
12524 std::fs::write(
12525 &golden_path,
12526 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
12527 )
12528 .unwrap();
12529 }
12530 let expected: Value =
12531 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
12532 assert_eq!(actual, expected);
12533 }
12534
12535 async fn query_live_roots(
12536 handler: &ControlHandler,
12537 module_ctx: &RouteCtx,
12538 ) -> ModuleControlResponseToModule {
12539 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
12540 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
12541 let response = handler
12542 .handle_control_frame(module_ctx, frame)
12543 .await
12544 .unwrap()
12545 .pop()
12546 .unwrap();
12547 serde_json::from_slice(&response.body).unwrap()
12548 }
12549
12550 #[tokio::test(start_paused = true)]
12551 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
12552 let registry = Arc::new(Registry::default());
12553 let forwarding = Arc::new(ForwardingTable::default());
12554 let handler = ControlHandler::with_forwarding(registry, forwarding);
12555 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
12556 hello_via_sink(
12557 &handler,
12558 &target_ctx,
12559 &mut target_rx,
12560 hello_frame("target", PROTOCOL_VERSION, 1),
12561 )
12562 .await;
12563 let root = unique_project_root("live-roots-known");
12564 let path = ProjectRootId::from_path_allowing_missing(root.path())
12565 .unwrap()
12566 .as_path()
12567 .to_path_buf();
12568 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
12569 let open_handler = handler.clone();
12570 let opened = tokio::spawn(async move {
12571 open_handler
12572 .handle_control_frame(&client_ctx, legacy_route_open_frame(2, "target", root))
12573 .await
12574 .unwrap()
12575 });
12576 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12577 .await
12578 .unwrap()
12579 .unwrap();
12580 handler
12581 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
12582 .await
12583 .unwrap();
12584 assert!(opened.await.unwrap().is_empty());
12585 let _ = client_rx.recv().await.unwrap();
12586
12587 let root = unique_project_root("live-roots-pending");
12588 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
12589 .unwrap()
12590 .as_path()
12591 .to_path_buf();
12592 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
12593 let open_handler = handler.clone();
12594 let pending = tokio::spawn(async move {
12595 open_handler
12596 .handle_control_frame(&client_ctx, legacy_route_open_frame(3, "target", root))
12597 .await
12598 .unwrap()
12599 });
12600 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12601 .await
12602 .unwrap()
12603 .unwrap();
12604 let actual = query_live_roots(&handler, &target_ctx).await;
12605 let ModuleControlResponseToModule::LiveRoots {
12606 roots,
12607 unknown_root_bindings,
12608 total_bindings,
12609 } = actual
12610 else {
12611 panic!("expected live roots")
12612 };
12613 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
12614 assert_eq!(unknown_root_bindings, 0);
12615 assert_eq!(
12616 roots.len(),
12617 2,
12618 "root-known arm must retain each canonical root"
12619 );
12620 assert_eq!(
12621 total_bindings,
12622 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12623 );
12624 let counts = roots
12625 .iter()
12626 .map(|root| (root.project_root.clone(), root.bound, root.pending))
12627 .collect::<Vec<_>>();
12628 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
12629 expected.sort_by(|a, b| a.0.cmp(&b.0));
12630 assert_eq!(
12631 counts, expected,
12632 "roots must sort by path and count pending separately"
12633 );
12634 handler
12635 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
12636 .await
12637 .unwrap();
12638 assert!(pending.await.unwrap().is_empty());
12639 }
12640
12641 #[tokio::test(start_paused = true)]
12642 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
12643 let forwarding = Arc::new(ForwardingTable::default());
12644 let handler =
12645 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
12646 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
12647 hello_via_sink(
12648 &handler,
12649 &target_ctx,
12650 &mut target_rx,
12651 hello_frame("target", PROTOCOL_VERSION, 1),
12652 )
12653 .await;
12654 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
12655 let pending = forwarding
12656 .begin_route_bind_relay_for_test(
12657 client_ctx.connection_id,
12658 client_ctx.egress.clone(),
12659 2,
12660 "target",
12661 )
12662 .unwrap();
12663 forwarding
12664 .complete_pending_relay(
12665 target_ctx.connection_id,
12666 pending.corr,
12667 RouteBindRelayOutcome::Accepted,
12668 )
12669 .unwrap();
12670 let actual = query_live_roots(&handler, &target_ctx).await;
12671 let ModuleControlResponseToModule::LiveRoots {
12672 roots,
12673 unknown_root_bindings,
12674 total_bindings,
12675 } = actual
12676 else {
12677 panic!("expected live roots")
12678 };
12679 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
12680 assert_eq!(
12681 unknown_root_bindings, 1,
12682 "unknown-root arm must not read as no bindings"
12683 );
12684 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
12685 assert_eq!(
12686 total_bindings,
12687 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12688 );
12689 }
12690
12691 #[tokio::test(start_paused = true)]
12699 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
12700 let forwarding = Arc::new(ForwardingTable::default());
12701 let handler =
12702 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
12703 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
12704 let replies = handler
12705 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
12706 .await
12707 .unwrap();
12708 let queued_by_hello = module_rx.len();
12709
12710 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
12711 let open_handler = handler.clone();
12712 let open = tokio::spawn(async move {
12713 open_handler
12714 .handle_control_frame(
12715 &client_ctx,
12716 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
12717 )
12718 .await
12719 .unwrap()
12720 });
12721 let mut spins = 0;
12723 while module_rx.len() == queued_by_hello {
12724 spins += 1;
12725 assert!(spins < 10_000, "route.open never queued a route.bind");
12726 tokio::task::yield_now().await;
12727 }
12728
12729 for reply in replies {
12731 module_ctx.egress.send(reply).await.unwrap();
12732 }
12733
12734 let first = module_rx.recv().await.unwrap().frame;
12735 assert_eq!(
12736 first.header.ty,
12737 FrameType::HelloAck,
12738 "the first frame a registering module reads must be its HELLO_ACK"
12739 );
12740 assert_eq!(first.header.corr, 7);
12741 let second = module_rx.recv().await.unwrap().frame;
12742 assert_eq!(second.header.ty, FrameType::Request);
12743 assert!(
12744 matches!(
12745 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
12746 ModuleControlRequest::RouteBind { .. }
12747 ),
12748 "the route.bind follows the ack"
12749 );
12750 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
12751
12752 handler
12753 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
12754 .await
12755 .unwrap();
12756 assert!(open.await.unwrap().is_empty());
12757 let _ = client_rx.recv().await.unwrap();
12758 }
12759
12760 #[tokio::test(start_paused = true)]
12761 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
12762 let handler = ControlHandler::with_forwarding(
12763 Arc::new(Registry::default()),
12764 Arc::new(ForwardingTable::default()),
12765 );
12766 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
12767 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
12768 hello_via_sink(
12769 &handler,
12770 &first_ctx,
12771 &mut first_rx,
12772 hello_frame("first", PROTOCOL_VERSION, 1),
12773 )
12774 .await;
12775 hello_via_sink(
12776 &handler,
12777 &second_ctx,
12778 &mut second_rx,
12779 hello_frame("second", PROTOCOL_VERSION, 2),
12780 )
12781 .await;
12782 let root = unique_project_root("second-only");
12783 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
12784 let cloned = handler.clone();
12785 let open = tokio::spawn(async move {
12786 cloned
12787 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
12788 .await
12789 .unwrap()
12790 });
12791 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
12792 .await
12793 .unwrap()
12794 .unwrap();
12795 let first = query_live_roots(&handler, &first_ctx).await;
12796 let second = query_live_roots(&handler, &second_ctx).await;
12797 assert!(
12798 matches!(
12799 first,
12800 ModuleControlResponseToModule::LiveRoots {
12801 total_bindings: 0,
12802 ..
12803 }
12804 ),
12805 "cross-module scope must not expose another module's roots"
12806 );
12807 assert!(
12808 matches!(
12809 second,
12810 ModuleControlResponseToModule::LiveRoots {
12811 total_bindings: 1,
12812 ..
12813 }
12814 ),
12815 "second module must see its pending route"
12816 );
12817 handler
12818 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
12819 .await
12820 .unwrap();
12821 assert!(open.await.unwrap().is_empty());
12822 }
12823
12824 #[tokio::test(start_paused = true)]
12825 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
12826 let handler = ControlHandler::with_forwarding(
12827 Arc::new(Registry::default()),
12828 Arc::new(ForwardingTable::default()),
12829 );
12830 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
12831 hello_via_sink(
12832 &handler,
12833 &target_ctx,
12834 &mut target_rx,
12835 hello_frame("target", PROTOCOL_VERSION, 1),
12836 )
12837 .await;
12838 let actual = query_live_roots(&handler, &target_ctx).await;
12839 let ModuleControlResponseToModule::LiveRoots {
12840 roots,
12841 unknown_root_bindings,
12842 total_bindings,
12843 } = actual
12844 else {
12845 panic!("expected live roots")
12846 };
12847 assert!(roots.is_empty());
12848 assert_eq!(unknown_root_bindings, 0);
12849 assert_eq!(total_bindings, 0);
12850 assert_eq!(
12851 total_bindings,
12852 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12853 );
12854 }
12855
12856 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
12863 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
12864 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
12865 let text = std::fs::read_to_string(&path)
12866 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
12867 let vectors: Vec<(String, Value)> = text
12868 .lines()
12869 .filter(|line| !line.trim().is_empty())
12870 .map(|line| {
12871 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
12872 let id = entry["corpus_id"]
12873 .as_str()
12874 .expect("every vector carries a corpus_id")
12875 .to_string();
12876 (id, entry["package"].clone())
12877 })
12878 .collect();
12879 assert_eq!(
12883 vectors.len(),
12884 3,
12885 "vendored fed corpus changed size; re-sync from subc-federation"
12886 );
12887
12888 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
12903 let richest = vectors
12904 .iter()
12905 .filter_map(|(_, package)| package.as_object())
12906 .map(|object| {
12907 object
12908 .keys()
12909 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
12910 .count()
12911 })
12912 .max()
12913 .unwrap_or(0);
12914 assert!(
12915 richest >= 2,
12916 "vendored corpus no longer carries a package with unmodelled fields, \
12917 so the relay test can no longer distinguish a verbatim relay from a lossy one"
12918 );
12919
12920 vectors
12921 }
12922
12923 #[tokio::test]
12931 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
12932 for (corpus_id, package) in fed_admission_facts_vectors() {
12933 let registry = Arc::new(Registry::default());
12934 let forwarding = Arc::new(ForwardingTable::default());
12935 let supervisor = SupervisorHandle::new();
12936 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12937 let handler =
12938 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12939 .with_supervisor(supervisor)
12940 .with_admission_facts_config(
12941 Some("fed".to_string()),
12942 Some(vec!["target".to_string()]),
12943 );
12944
12945 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
12946 hello_via_sink(
12947 &handler,
12948 &target_ctx,
12949 &mut target_rx,
12950 hello_frame("target", PROTOCOL_VERSION, 1),
12951 )
12952 .await;
12953
12954 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
12955 let route_handler = handler.clone();
12956 let expected = package.clone();
12957 let route_task = tokio::spawn(async move {
12958 route_handler
12959 .handle_control_frame(
12960 &client_ctx,
12961 route_open_frame_with_admission_facts(
12962 20,
12963 "target",
12964 unique_project_root("admission-facts"),
12965 Some(subc_control::ConsumerIdentity {
12966 module_id: "fed".to_string(),
12967 launch_nonce: "fed-nonce".to_string(),
12968 }),
12969 Some(package),
12970 ),
12971 )
12972 .await
12973 .unwrap()
12974 });
12975
12976 let bind_frame = target_rx.recv().await.unwrap();
12977 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12978 let ModuleControlRequest::RouteBind {
12979 admission_facts, ..
12980 } = bind
12981 else {
12982 panic!("{corpus_id}: expected route.bind")
12983 };
12984 assert_eq!(
12985 admission_facts,
12986 Some(expected),
12987 "{corpus_id}: relay must not add, drop or reshape any field"
12988 );
12989
12990 handler
12991 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12992 .await
12993 .unwrap();
12994 route_task.await.unwrap();
12995 }
12996 }
12997
12998 #[tokio::test]
12999 async fn route_bind_never_relays_the_openers_build_class() {
13000 let registry = Arc::new(Registry::default());
13001 let forwarding = Arc::new(ForwardingTable::default());
13002 let handler =
13003 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13004 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(80));
13005 hello_via_sink(
13006 &handler,
13007 &target_ctx,
13008 &mut target_rx,
13009 hello_frame("target", PROTOCOL_VERSION, 1),
13010 )
13011 .await;
13012
13013 let mut identity = BindIdentity::new("unit".to_string(), "s".to_string());
13014 identity.build_class = Some(subc_protocol::BUILD_CLASS_PRODUCTION.to_string());
13015 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
13016 busy_refusals: None,
13017 target: RouteTarget::ToolProvider {
13018 module_id: "target".to_string(),
13019 },
13020 identity,
13021 consumer_identity: None,
13022 consumer_capabilities: None,
13023 role_versions: None,
13024 admission_facts: None,
13025 scope: None,
13026 })
13027 .unwrap();
13028 let open = Frame::build(FrameType::Request, control_flags(), 0, 0, 30, body).unwrap();
13029
13030 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(81));
13031 let route_handler = handler.clone();
13032 let route_task = tokio::spawn(async move {
13033 route_handler
13034 .handle_control_frame(&client_ctx, open)
13035 .await
13036 .unwrap()
13037 });
13038 let bind_frame = target_rx.recv().await.unwrap();
13039 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13040 let ModuleControlRequest::RouteBind { identity, .. } = bind else {
13041 panic!("expected route.bind")
13042 };
13043 assert_eq!(
13044 identity.build_class, None,
13045 "route.bind carried the build class"
13046 );
13047 assert!(
13048 !String::from_utf8_lossy(&bind_frame.body).contains("build_class"),
13049 "the route.bind bytes still name build_class"
13050 );
13051 handler
13052 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
13053 .await
13054 .unwrap();
13055 route_task.await.unwrap();
13056 }
13057
13058 #[tokio::test]
13059 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
13060 let registry = Arc::new(Registry::default());
13061 let forwarding = Arc::new(ForwardingTable::default());
13062 let supervisor = SupervisorHandle::new();
13063 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
13064 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
13065 let handler =
13066 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13067 .with_supervisor(supervisor)
13068 .with_admission_facts_config(
13069 Some("fed".to_string()),
13070 Some(vec!["target".to_string()]),
13071 );
13072
13073 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
13074 hello_via_sink(
13075 &handler,
13076 &target_ctx,
13077 &mut target_rx,
13078 hello_frame("target", PROTOCOL_VERSION, 1),
13079 )
13080 .await;
13081 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
13082 hello_via_sink(
13083 &handler,
13084 &other_ctx,
13085 &mut other_rx,
13086 hello_frame("other", PROTOCOL_VERSION, 2),
13087 )
13088 .await;
13089
13090 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
13091 let expected_facts = facts.clone();
13092 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
13093 let route_handler = handler.clone();
13094 let route_task = tokio::spawn(async move {
13095 route_handler
13096 .handle_control_frame(
13097 &client_ctx,
13098 route_open_frame_with_admission_facts(
13099 10,
13100 "target",
13101 unique_project_root("admission-facts"),
13102 Some(subc_control::ConsumerIdentity {
13103 module_id: "fed".to_string(),
13104 launch_nonce: "fed-nonce".to_string(),
13105 }),
13106 Some(facts.clone()),
13107 ),
13108 )
13109 .await
13110 .unwrap()
13111 });
13112 let bind_frame = target_rx.recv().await.unwrap();
13113 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13114 let ModuleControlRequest::RouteBind {
13115 admission_facts, ..
13116 } = bind
13117 else {
13118 panic!("expected route.bind")
13119 };
13120 assert_eq!(admission_facts, Some(expected_facts));
13121 handler
13122 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
13123 .await
13124 .unwrap();
13125 assert!(route_task.await.unwrap().is_empty());
13126 assert!(matches!(
13127 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
13128 .unwrap(),
13129 ClientControlResponse::RouteOpen { .. }
13130 ));
13131
13132 let direct = handler
13133 .handle_control_frame(
13134 &route_ctx(ConnectionId::new(73)).0,
13135 route_open_frame_with_admission_facts(
13136 11,
13137 "target",
13138 unique_project_root("admission-facts"),
13139 None,
13140 Some(json!({"x": 1})),
13141 ),
13142 )
13143 .await
13144 .unwrap();
13145 assert_eq!(
13146 parse_error(&direct[0])["code"],
13147 "admission_facts_not_permitted"
13148 );
13149
13150 let different_reserved = handler
13151 .handle_control_frame(
13152 &route_ctx(ConnectionId::new(77)).0,
13153 route_open_frame_with_admission_facts(
13154 15,
13155 "target",
13156 unique_project_root("admission-facts"),
13157 Some(subc_control::ConsumerIdentity {
13158 module_id: "other".to_string(),
13159 launch_nonce: "other-nonce".to_string(),
13160 }),
13161 Some(json!({"x": 1})),
13162 ),
13163 )
13164 .await
13165 .unwrap();
13166 assert_eq!(
13167 parse_error(&different_reserved[0])["code"],
13168 "admission_facts_not_permitted"
13169 );
13170
13171 let other_target = handler
13172 .handle_control_frame(
13173 &route_ctx(ConnectionId::new(74)).0,
13174 route_open_frame_with_admission_facts(
13175 12,
13176 "other",
13177 unique_project_root("admission-facts"),
13178 Some(subc_control::ConsumerIdentity {
13179 module_id: "fed".to_string(),
13180 launch_nonce: "fed-nonce".to_string(),
13181 }),
13182 Some(json!({"x": 1})),
13183 ),
13184 )
13185 .await
13186 .unwrap();
13187 assert_eq!(
13188 parse_error(&other_target[0])["code"],
13189 "admission_facts_target_not_allowed"
13190 );
13191
13192 let nonexistent = handler
13193 .handle_control_frame(
13194 &route_ctx(ConnectionId::new(75)).0,
13195 route_open_frame_with_admission_facts(
13196 13,
13197 "missing",
13198 unique_project_root("admission-facts"),
13199 None,
13200 Some(json!({"x": 1})),
13201 ),
13202 )
13203 .await
13204 .unwrap();
13205 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
13206
13207 let described = handler
13208 .handle_control_frame(
13209 &route_ctx(ConnectionId::new(76)).0,
13210 Frame::build(
13211 FrameType::Request,
13212 control_flags(),
13213 0,
13214 0,
13215 14,
13216 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13217 )
13218 .unwrap(),
13219 )
13220 .await
13221 .unwrap();
13222 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13223 serde_json::from_slice(&described[0].body).unwrap()
13224 else {
13225 panic!("expected server.describe response")
13226 };
13227 assert!(capabilities
13228 .iter()
13229 .any(|cap| cap == "admission_facts_relay_v1"));
13230 }
13231
13232 #[tokio::test]
13233 async fn admission_facts_without_configured_carrier_are_rejected() {
13234 let registry = Arc::new(Registry::default());
13235 let forwarding = Arc::new(ForwardingTable::default());
13236 let handler = ControlHandler::with_forwarding(registry, forwarding);
13237 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
13238 hello_via_sink(
13239 &handler,
13240 &target_ctx,
13241 &mut target_rx,
13242 hello_frame("target", PROTOCOL_VERSION, 1),
13243 )
13244 .await;
13245
13246 let responses = handler
13247 .handle_control_frame(
13248 &route_ctx(ConnectionId::new(79)).0,
13249 route_open_frame_with_admission_facts(
13250 16,
13251 "target",
13252 unique_project_root("admission-facts"),
13253 None,
13254 Some(json!({"x": 1})),
13255 ),
13256 )
13257 .await
13258 .unwrap();
13259 assert_eq!(
13260 parse_error(&responses[0])["code"],
13261 "admission_facts_not_permitted"
13262 );
13263 }
13264
13265 #[tokio::test]
13266 async fn route_open_relays_consumer_capabilities_verbatim() {
13267 let registry = Arc::new(Registry::default());
13268 let forwarding = Arc::new(ForwardingTable::default());
13269 let handler =
13270 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13271 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
13272 hello_via_sink(
13273 &handler,
13274 &module_ctx,
13275 &mut module_rx,
13276 hello_frame("aft", PROTOCOL_VERSION, 7),
13277 )
13278 .await;
13279
13280 let expected = vec!["elicitation".to_string(), "roots".to_string()];
13281 let expected_for_request = expected.clone();
13282 let project_root = unique_project_root("consumer-capabilities-present");
13283 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
13284 let route_handler = handler.clone();
13285 let route_task = tokio::spawn(async move {
13286 route_handler
13287 .handle_control_frame(
13288 &client_ctx,
13289 route_open_frame_with_consumer_capabilities(
13290 401,
13291 "aft",
13292 project_root,
13293 Some(expected_for_request),
13294 ),
13295 )
13296 .await
13297 .unwrap()
13298 });
13299 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13300 .await
13301 .unwrap()
13302 .unwrap();
13303 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13304 let ModuleControlRequest::RouteBind {
13305 consumer_capabilities,
13306 ..
13307 } = bind
13308 else {
13309 panic!("expected route.bind request, got {bind:?}");
13310 };
13311 assert_eq!(consumer_capabilities, Some(expected.clone()));
13312
13313 handler
13314 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13315 .await
13316 .unwrap();
13317 let route_response = route_task.await.unwrap();
13318 assert!(route_response.is_empty());
13319 let published = client_rx.recv().await.unwrap();
13320 assert!(matches!(
13321 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13322 ClientControlResponse::RouteOpen { .. }
13323 ));
13324 }
13325
13326 #[tokio::test]
13327 async fn route_open_without_consumer_capabilities_relays_none() {
13328 let registry = Arc::new(Registry::default());
13329 let forwarding = Arc::new(ForwardingTable::default());
13330 let handler =
13331 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13332 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
13333 hello_via_sink(
13334 &handler,
13335 &module_ctx,
13336 &mut module_rx,
13337 hello_frame("aft", PROTOCOL_VERSION, 7),
13338 )
13339 .await;
13340
13341 let project_root = unique_project_root("consumer-capabilities-absent");
13342 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
13343 let route_handler = handler.clone();
13344 let route_task = tokio::spawn(async move {
13345 route_handler
13346 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
13347 .await
13348 .unwrap()
13349 });
13350 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13351 .await
13352 .unwrap()
13353 .unwrap();
13354 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13355 let ModuleControlRequest::RouteBind {
13356 consumer_capabilities,
13357 ..
13358 } = bind
13359 else {
13360 panic!("expected route.bind request, got {bind:?}");
13361 };
13362 assert_eq!(consumer_capabilities, None);
13363
13364 handler
13365 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13366 .await
13367 .unwrap();
13368 let route_response = route_task.await.unwrap();
13369 assert!(route_response.is_empty());
13370 let published = client_rx.recv().await.unwrap();
13371 assert!(matches!(
13372 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13373 ClientControlResponse::RouteOpen { .. }
13374 ));
13375 }
13376
13377 async fn bind_role_versions_for(
13381 sent: Option<BTreeMap<String, String>>,
13382 connection: u64,
13383 ) -> Option<BTreeMap<String, String>> {
13384 let registry = Arc::new(Registry::default());
13385 let forwarding = Arc::new(ForwardingTable::default());
13386 let handler =
13387 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13388 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
13389 hello_via_sink(
13390 &handler,
13391 &module_ctx,
13392 &mut module_rx,
13393 hello_frame("aft", PROTOCOL_VERSION, 7),
13394 )
13395 .await;
13396 let project_root = unique_project_root("role-versions");
13397 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
13398 let route_handler = handler.clone();
13399 let route_task = tokio::spawn(async move {
13400 route_handler
13401 .handle_control_frame(
13402 &client_ctx,
13403 route_open_frame_with_role_versions(403, "aft", project_root, sent),
13404 )
13405 .await
13406 .unwrap()
13407 });
13408 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13409 .await
13410 .expect("a well-formed route.open reaches the module as a bind")
13411 .unwrap();
13412 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13413 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
13414 panic!("expected route.bind request, got {bind:?}");
13415 };
13416 handler
13417 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13418 .await
13419 .unwrap();
13420 assert!(route_task.await.unwrap().is_empty());
13421 let published = client_rx.recv().await.unwrap();
13422 assert!(matches!(
13423 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13424 ClientControlResponse::RouteOpen { .. }
13425 ));
13426 role_versions
13427 }
13428
13429 #[tokio::test]
13430 async fn route_open_relays_role_versions_verbatim() {
13431 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
13432 assert_eq!(
13433 bind_role_versions_for(Some(sent.clone()), 141).await,
13434 Some(sent)
13435 );
13436 }
13437
13438 #[tokio::test]
13441 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
13442 assert_eq!(bind_role_versions_for(None, 143).await, None);
13443 assert_eq!(
13444 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
13445 None
13446 );
13447 }
13448
13449 #[tokio::test]
13450 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
13451 let registry = Arc::new(Registry::default());
13452 let forwarding = Arc::new(ForwardingTable::default());
13453 let handler =
13454 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13455 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
13456 hello_via_sink(
13457 &handler,
13458 &module_ctx,
13459 &mut module_rx,
13460 hello_frame("aft", PROTOCOL_VERSION, 7),
13461 )
13462 .await;
13463
13464 let nine: BTreeMap<String, String> = (0..9)
13465 .map(|index| (format!("role-{index}"), "v1".to_string()))
13466 .collect();
13467 for (label, malformed) in [
13468 (
13469 "invalid role name",
13470 role_versions(&[("Tool_Provider", "v1")]),
13471 ),
13472 ("invalid version", role_versions(&[("tool-provider", "v0")])),
13473 ("nine entries", nine),
13474 ] {
13475 let responses = tokio::time::timeout(
13478 Duration::from_secs(1),
13479 handler.handle_control_frame(
13480 &route_ctx(ConnectionId::new(148)).0,
13481 route_open_frame_with_role_versions(
13482 404,
13483 "aft",
13484 unique_project_root("role-versions-malformed"),
13485 Some(malformed),
13486 ),
13487 ),
13488 )
13489 .await
13490 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
13491 .unwrap();
13492 assert_eq!(responses.len(), 1, "{label}");
13493 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
13494 let error = parse_error(&responses[0]);
13495 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
13496 assert_eq!(
13497 error["detail"]["field"], "role_versions",
13498 "{label}: {error}"
13499 );
13500 assert!(
13501 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
13502 "{label}: a malformed declaration is terminal"
13503 );
13504 assert!(
13505 module_rx.try_recv().is_err(),
13506 "{label}: the module must never see a bind"
13507 );
13508 }
13509 }
13510
13511 #[tokio::test]
13515 async fn route_role_versions_capability_is_advertised() {
13516 let handler = ControlHandler::new(Arc::new(Registry::default()));
13517 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
13518 let ack = hello_via_sink(
13519 &handler,
13520 &ctx,
13521 &mut rx,
13522 hello_frame("m", PROTOCOL_VERSION, 1),
13523 )
13524 .await;
13525 let ack = parse_ack(&ack);
13526 assert!(
13527 ack.subc_capabilities
13528 .iter()
13529 .any(|c| c == "route-role-versions/v1"),
13530 "{:?}",
13531 ack.subc_capabilities
13532 );
13533
13534 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13535 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13536 let reply = handler
13537 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
13538 .await
13539 .unwrap()
13540 .pop()
13541 .unwrap();
13542 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13543 serde_json::from_slice(&reply.body).unwrap()
13544 else {
13545 panic!("not a server.describe reply");
13546 };
13547 assert!(
13548 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
13549 "{capabilities:?}"
13550 );
13551 }
13552
13553 #[tokio::test]
13557 async fn catalog_events_capability_is_advertised() {
13558 let handler = ControlHandler::new(Arc::new(Registry::default()));
13559 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13560 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13561 let reply = handler
13562 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
13563 .await
13564 .unwrap()
13565 .pop()
13566 .unwrap();
13567 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13568 serde_json::from_slice(&reply.body).unwrap()
13569 else {
13570 panic!("not a server.describe reply");
13571 };
13572 assert!(
13573 capabilities
13574 .iter()
13575 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
13576 "{capabilities:?}"
13577 );
13578 }
13579
13580 #[tokio::test]
13581 async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
13582 let handler = ControlHandler::new(Arc::new(Registry::default()));
13583 let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
13584 let ack = hello_via_sink(
13585 &handler,
13586 &module,
13587 &mut module_rx,
13588 hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
13589 )
13590 .await;
13591 let ack = parse_ack(&ack);
13592 assert!(
13593 ack.subc_capabilities
13594 .iter()
13595 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
13596 "{:?}",
13597 ack.subc_capabilities
13598 );
13599 assert!(ack
13600 .subc_capabilities
13601 .iter()
13602 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
13603
13604 let (client, _client_rx) = route_ctx(ConnectionId::new(153));
13605 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13606 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13607 let reply = handler
13608 .handle_control_frame(&client, frame)
13609 .await
13610 .unwrap()
13611 .pop()
13612 .unwrap();
13613 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13614 serde_json::from_slice(&reply.body).unwrap()
13615 else {
13616 panic!("not a server.describe reply");
13617 };
13618 assert!(
13619 capabilities
13620 .iter()
13621 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
13622 "{capabilities:?}"
13623 );
13624 assert!(capabilities
13625 .iter()
13626 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
13627 }
13628
13629 #[tokio::test]
13630 async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
13631 let handler = ControlHandler::new(Arc::new(Registry::default()));
13632 let (module, mut rx) = route_ctx(ConnectionId::new(156));
13633 let ack = hello_via_sink(
13634 &handler,
13635 &module,
13636 &mut rx,
13637 hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
13638 )
13639 .await;
13640 assert!(parse_ack(&ack)
13641 .subc_capabilities
13642 .iter()
13643 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
13644 assert!(parse_ack(&ack)
13645 .subc_capabilities
13646 .iter()
13647 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
13648 let (client, _rx) = route_ctx(ConnectionId::new(157));
13649 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13650 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13651 let reply = handler
13652 .handle_control_frame(&client, frame)
13653 .await
13654 .unwrap()
13655 .pop()
13656 .unwrap();
13657 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13658 serde_json::from_slice(&reply.body).unwrap()
13659 else {
13660 panic!("server.describe reply")
13661 };
13662 assert!(capabilities
13663 .iter()
13664 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
13665 assert!(capabilities
13666 .iter()
13667 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
13668 }
13669
13670 #[tokio::test]
13671 async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
13672 let handler = ControlHandler::new(Arc::new(Registry::default()));
13673 let (module, mut rx) = route_ctx(ConnectionId::new(158));
13674 let ack = hello_via_sink(
13675 &handler,
13676 &module,
13677 &mut rx,
13678 hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
13679 )
13680 .await;
13681 assert!(parse_ack(&ack)
13682 .subc_capabilities
13683 .iter()
13684 .any(|cap| cap == "scope-workdir/v1"));
13685 let (client, _rx) = route_ctx(ConnectionId::new(159));
13686 let frame = Frame::build(
13687 FrameType::Request,
13688 control_flags(),
13689 0,
13690 0,
13691 5,
13692 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13693 )
13694 .unwrap();
13695 let reply = handler
13696 .handle_control_frame(&client, frame)
13697 .await
13698 .unwrap()
13699 .remove(0);
13700 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13701 serde_json::from_slice(&reply.body).unwrap()
13702 else {
13703 panic!("server.describe reply")
13704 };
13705 assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
13706 }
13707
13708 #[tokio::test]
13709 async fn worker_runs_capability_is_advertised_in_hello_ack_and_server_describe() {
13710 let handler = ControlHandler::new(Arc::new(Registry::default()));
13711 let (module, mut rx) = route_ctx(ConnectionId::new(160));
13712 let ack = hello_via_sink(
13713 &handler,
13714 &module,
13715 &mut rx,
13716 hello_frame("worker-run-owner", PROTOCOL_VERSION, 1),
13717 )
13718 .await;
13719 assert!(parse_ack(&ack)
13720 .subc_capabilities
13721 .iter()
13722 .any(|cap| cap == "worker-runs/v1"));
13723 let (client, _rx) = route_ctx(ConnectionId::new(161));
13724 let frame = Frame::build(
13725 FrameType::Request,
13726 control_flags(),
13727 0,
13728 0,
13729 5,
13730 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13731 )
13732 .unwrap();
13733 let reply = handler
13734 .handle_control_frame(&client, frame)
13735 .await
13736 .unwrap()
13737 .remove(0);
13738 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13739 serde_json::from_slice(&reply.body).unwrap()
13740 else {
13741 panic!("server.describe reply")
13742 };
13743 assert!(capabilities.iter().any(|cap| cap == "worker-runs/v1"));
13744 }
13745
13746 #[tokio::test]
13747 async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
13748 let handler = ControlHandler::new(Arc::new(Registry::default()));
13749 let (module, mut rx) = route_ctx(ConnectionId::new(154));
13750 let ack = hello_via_sink(
13751 &handler,
13752 &module,
13753 &mut rx,
13754 hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
13755 )
13756 .await;
13757 assert!(parse_ack(&ack)
13758 .subc_capabilities
13759 .iter()
13760 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
13761 let (client, _rx) = route_ctx(ConnectionId::new(155));
13762 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13763 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13764 let reply = handler
13765 .handle_control_frame(&client, frame)
13766 .await
13767 .unwrap()
13768 .pop()
13769 .unwrap();
13770 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13771 serde_json::from_slice(&reply.body).unwrap()
13772 else {
13773 panic!("not a server.describe reply")
13774 };
13775 assert!(capabilities
13776 .iter()
13777 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
13778 }
13779
13780 #[tokio::test]
13781 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
13782 let registry = Arc::new(Registry::default());
13783 let forwarding = Arc::new(ForwardingTable::default());
13784 let handler =
13785 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13786 .with_health_probe_timeout(Duration::from_secs(5));
13787 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
13788 hello_via_sink(
13789 &handler,
13790 &module_ctx,
13791 &mut module_rx,
13792 non_routable_hello_frame_with_control_ops(
13793 "mcp",
13794 300,
13795 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13796 ),
13797 )
13798 .await;
13799 assert!(registry
13800 .get_module("mcp")
13801 .unwrap()
13802 .unwrap()
13803 .manifest
13804 .provides
13805 .is_empty());
13806
13807 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
13808 let route_response = handler
13809 .handle_control_frame(
13810 &route_client_ctx,
13811 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
13812 )
13813 .await
13814 .unwrap();
13815 assert_eq!(route_response[0].header.ty, FrameType::Error);
13816 assert_eq!(
13817 parse_error(&route_response[0])["code"],
13818 "target_unavailable"
13819 );
13820 assert!(parse_error(&route_response[0])["message"]
13821 .as_str()
13822 .unwrap()
13823 .contains("does not provide the requested target"));
13824 assert!(module_rx.try_recv().is_err());
13825
13826 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
13827 let health_handler = handler.clone();
13828 let health_task = tokio::spawn(async move {
13829 health_handler
13830 .handle_control_frame(
13831 &health_client_ctx,
13832 supervisor_health_probe_frame(302, "mcp"),
13833 )
13834 .await
13835 .unwrap()
13836 });
13837 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13838 .await
13839 .unwrap()
13840 .unwrap();
13841 assert_eq!(
13842 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
13843 ModuleControlRequest::HealthCheck {}
13844 );
13845 handler
13846 .handle_control_frame(
13847 &module_ctx,
13848 health_response(health_frame.header.corr, HealthStatus::Ok),
13849 )
13850 .await
13851 .unwrap();
13852 let health_response = health_task.await.unwrap();
13853 assert_eq!(health_response[0].header.ty, FrameType::Response);
13854 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
13855 ClientControlResponse::SupervisorHealthProbe {
13856 module_id, status, ..
13857 } => {
13858 assert_eq!(module_id, "mcp");
13859 assert_eq!(status, HealthStatus::Ok);
13860 }
13861 other => panic!("unexpected health response: {other:?}"),
13862 }
13863
13864 forwarding
13869 .cleanup_connection(module_ctx.connection_id)
13870 .unwrap();
13871 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
13872 let cleanup_response = tokio::time::timeout(
13873 Duration::from_millis(200),
13874 handler.handle_control_frame(
13875 &cleanup_probe_ctx,
13876 supervisor_health_probe_frame(303, "mcp"),
13877 ),
13878 )
13879 .await
13880 .expect("probe should fail immediately when the control lane is gone")
13881 .unwrap();
13882 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
13883 assert_eq!(
13884 parse_error(&cleanup_response[0])["code"],
13885 "target_unavailable"
13886 );
13887 assert!(parse_error(&cleanup_response[0])["message"]
13888 .as_str()
13889 .unwrap()
13890 .contains("no module connection"));
13891
13892 handler
13893 .cleanup_connection(module_ctx.connection_id)
13894 .unwrap();
13895 }
13896
13897 #[tokio::test]
13898 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
13899 let registry = Arc::new(Registry::default());
13900 let supervisor_handle = SupervisorHandle::new();
13901 let supervisor =
13902 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
13903 .with_handle(supervisor_handle.clone())
13904 .with_connection_file_path(
13905 std::env::temp_dir()
13906 .join(format!("subc-route-open-warming-{}", std::process::id())),
13907 );
13908 let module = supervisor
13909 .supervise_configured(
13910 ModuleSpec {
13911 carry_containment: false,
13912 module_id: "warming".to_string(),
13913 program: fake_aft_stub_path(),
13914 args: Vec::new(),
13915 env: Vec::new(),
13916 reserved: false,
13917 reserved_prefixes: Vec::new(),
13918 protocol: ModuleProtocol::Subc,
13919 overlap: Default::default(),
13920 },
13921 true,
13922 )
13923 .unwrap();
13924 assert_eq!(module.state().unwrap(), ModuleState::Running);
13925
13926 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13927 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
13928 let response = handler
13929 .handle_control_frame(
13930 &ctx,
13931 route_open_frame(304, "warming", unique_project_root("warming")),
13932 )
13933 .await
13934 .unwrap();
13935 module.stop().await.unwrap();
13936
13937 assert_eq!(response[0].header.ty, FrameType::Error);
13938 let error = parse_error(&response[0]);
13939 assert_eq!(error["code"], "module_warming");
13940 assert!(error["message"]
13941 .as_str()
13942 .unwrap()
13943 .contains("state=running, enabled=true, live=false"));
13944 }
13945
13946 #[test]
13947 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
13948 let handler = ControlHandler::new(Arc::new(Registry::default()));
13949 let capture = EventCapture::default();
13950 let _subscriber =
13951 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13952 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
13953 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
13954 let pending = (0..limit).collect::<Vec<_>>();
13955 let response = handler
13956 .route_open_capacity_refusal(
13957 &ctx,
13958 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
13959 "busy",
13960 pending.len(),
13961 limit,
13962 )
13963 .unwrap();
13964 assert_eq!(parse_error(&response)["code"], "target_unavailable");
13965 let event = capture
13966 .events()
13967 .into_iter()
13968 .find(|event| {
13969 event.target == "control"
13970 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
13971 })
13972 .expect("connection admission refusal event");
13973 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
13974 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
13975 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
13978 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
13979 }
13980
13981 #[test]
13982 fn route_open_target_cap_logs_admission_reason_and_capacity() {
13983 let handler = ControlHandler::new(Arc::new(Registry::default()));
13984 let capture = EventCapture::default();
13985 let _subscriber =
13986 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13987 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
13988 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
13989 let guards = (0..limit)
13990 .map(|_| {
13991 handler
13992 .route_bind_concurrency
13993 .try_admit("busy", limit)
13994 .unwrap()
13995 })
13996 .collect::<Vec<_>>();
13997 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
13998 Err(in_flight) => in_flight,
13999 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
14000 };
14001 let response = handler
14002 .route_open_target_capacity_refusal(
14003 &ctx,
14004 &RouteOpenCaller::unknown(),
14005 &route_open_frame(397, "busy", unique_project_root("target-cap")),
14006 "busy",
14007 in_flight,
14008 )
14009 .unwrap();
14010 assert_eq!(parse_error(&response)["code"], "target_unavailable");
14011 let event = capture
14012 .events()
14013 .into_iter()
14014 .find(|event| {
14015 event.target == "control"
14016 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
14017 })
14018 .expect("target admission refusal event");
14019 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
14020 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
14021 drop(guards);
14022 }
14023
14024 #[cfg(unix)]
14025 #[tokio::test(start_paused = true)]
14026 async fn supervisor_list_shows_a_failed_modules_next_retry() {
14027 let registry = Arc::new(Registry::default());
14028 let handle = SupervisorHandle::new();
14029 let supervisor =
14030 Supervisor::new_for_test(registry.clone(), RestartPolicy::new(0, Duration::ZERO))
14031 .with_handle(handle.clone());
14032 let module = supervisor
14033 .spawn(crate::ModuleSpec {
14034 carry_containment: false,
14035 module_id: "retry-list".into(),
14036 program: PathBuf::from("/bin/sh"),
14037 args: vec!["-c".into(), "exit 17".into()],
14038 env: vec![],
14039 reserved: false,
14040 reserved_prefixes: vec![],
14041 protocol: ModuleProtocol::None,
14042 overlap: Default::default(),
14043 })
14044 .unwrap();
14045 let deadline = std::time::Instant::now() + Duration::from_secs(10);
14046 while module.state().unwrap() != crate::ModuleState::Failed {
14047 assert!(
14048 std::time::Instant::now() < deadline,
14049 "waiting for budget failure"
14050 );
14051 tokio::task::yield_now().await;
14052 }
14053 let handler = ControlHandler::new(registry).with_supervisor(handle);
14054 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14055 let replies = handler.handle_supervisor_list(frame).await.unwrap();
14056 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14057 assert_eq!(body["modules"][0]["state"], "failed");
14058 let retry = body["modules"][0]["next_retry_at_ms"]
14059 .as_u64()
14060 .expect("scheduled retry on the wire");
14061 let now = std::time::SystemTime::now()
14062 .duration_since(std::time::UNIX_EPOCH)
14063 .unwrap()
14064 .as_millis() as u64;
14065 assert!((599_000..=600_000).contains(&retry.saturating_sub(now)));
14066 module.set_enabled(false).await.unwrap();
14067 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
14068 let replies = handler.handle_supervisor_list(frame).await.unwrap();
14069 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14070 assert!(body["modules"][0].get("next_retry_at_ms").is_none());
14071 }
14072
14073 #[tokio::test]
14074 async fn slow_step_before_pid_read_does_not_spend_its_budget() {
14075 let _guard = crate::subtree_resources::fault_read_guard().await;
14076 let dir = cortexkit_test_support::ScratchDir::new("pid-read-budget");
14077 let supervisor =
14078 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
14079 let module = supervisor
14080 .spawn(crate::ModuleSpec {
14081 carry_containment: false,
14082 module_id: "pid-read-budget".into(),
14083 program: std::env::current_exe().unwrap(),
14084 args: vec![
14085 "--exact".into(),
14086 "control::tests::pid_resource_read_fixture".into(),
14087 ],
14088 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14089 .into_iter()
14090 .map(|key| (key.into(), dir.display().to_string()))
14091 .chain(std::iter::once((
14092 "SUBC_SUBTREE_PID_FIXTURE".into(),
14093 "1".into(),
14094 )))
14095 .collect(),
14096 reserved: false,
14097 reserved_prefixes: Vec::new(),
14098 protocol: ModuleProtocol::None,
14099 overlap: Default::default(),
14100 })
14101 .unwrap();
14102 let ready_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
14103 while !matches!(
14104 module.child_resource_usage(),
14105 subc_control::ChildResourceUsage::Measured(_)
14106 ) {
14107 assert!(
14108 tokio::time::Instant::now() < ready_deadline,
14109 "fixture process was not measurable"
14110 );
14111 tokio::time::sleep(Duration::from_millis(10)).await;
14112 }
14113 tokio::time::sleep(crate::subtree_resources::READ_TIMEOUT * 2).await;
14114 let reading = crate::subtree_resources::read_process(module.clone()).await;
14115 module.stop().await.unwrap();
14116 assert!(
14117 matches!(reading, subc_control::ChildResourceUsage::Measured(_)),
14118 "{reading:?}"
14119 );
14120 }
14121
14122 #[test]
14123 fn pid_resource_read_fixture() {
14124 if std::env::var_os("SUBC_SUBTREE_PID_FIXTURE").is_some() {
14125 loop {
14126 std::thread::park_timeout(Duration::from_secs(600));
14127 }
14128 }
14129 }
14130
14131 #[tokio::test]
14132 async fn disabled_module_tree_is_not_running_instead_of_permanently_pending() {
14133 let registry = Arc::new(Registry::default());
14134 let handle = SupervisorHandle::new();
14135 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
14136 .with_handle(handle.clone());
14137 supervisor
14138 .supervise_configured(
14139 crate::ModuleSpec {
14140 module_id: "disabled-resource-reading".into(),
14141 program: std::env::current_exe().unwrap(),
14142 args: Vec::new(),
14143 env: Vec::new(),
14144 reserved: false,
14145 reserved_prefixes: Vec::new(),
14146 protocol: ModuleProtocol::None,
14147 overlap: Default::default(),
14148 carry_containment: false,
14149 },
14150 false,
14151 )
14152 .unwrap();
14153 let handler = ControlHandler::new(registry).with_supervisor(handle);
14154 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14155 let replies = handler.handle_supervisor_list(frame).await.unwrap();
14156 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14157 assert!(body["modules"][0].get("tree").is_none());
14158 assert_eq!(body["modules"][0]["tree_unavailable"], "not_running");
14159 }
14160
14161 #[tokio::test]
14162 async fn supervisor_list_reads_process_resources_before_a_stalled_tree() {
14163 let _guard = crate::subtree_resources::fault_read_guard().await;
14164 let dir = cortexkit_test_support::ScratchDir::new("list-process-priority");
14165 let registry = Arc::new(Registry::default());
14166 let handle = SupervisorHandle::new();
14167 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
14168 .with_handle(handle.clone());
14169 let module = supervisor
14170 .spawn(crate::ModuleSpec {
14171 carry_containment: false,
14172 module_id: "list-process-priority".into(),
14173 program: std::env::current_exe().unwrap(),
14174 args: vec![
14175 "--exact".into(),
14176 "control::tests::pid_resource_read_fixture".into(),
14177 ],
14178 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14179 .into_iter()
14180 .map(|key| (key.into(), dir.display().to_string()))
14181 .chain(std::iter::once((
14182 "SUBC_SUBTREE_PID_FIXTURE".into(),
14183 "1".into(),
14184 )))
14185 .collect(),
14186 reserved: false,
14187 reserved_prefixes: Vec::new(),
14188 protocol: ModuleProtocol::None,
14189 overlap: Default::default(),
14190 })
14191 .unwrap();
14192 let ready_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
14193 loop {
14194 let image = module.running_image_agreement().await;
14195 if matches!(image, subc_control::RunningImageAgreement::Match { .. }) {
14196 break;
14197 }
14198 assert!(
14199 tokio::time::Instant::now() < ready_deadline,
14200 "the fixture process never became observable: {image:?}"
14201 );
14202 tokio::time::sleep(Duration::from_millis(10)).await;
14203 }
14204 let (target, release, _) = crate::subtree_resources::held();
14205 module.set_subtree_target_for_test(target);
14206 let handler = ControlHandler::new(registry).with_supervisor(handle);
14207 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14208 let replies = handler.handle_supervisor_list(frame).await.unwrap();
14209 drop(release);
14210 module.stop().await.unwrap();
14211 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14212 assert_eq!(
14213 body["modules"][0]["resources"]["status"], "measured",
14214 "the optional tree timeout must not discard the process reading: {body}"
14215 );
14216 assert!(body["modules"][0].get("tree").is_none());
14217 assert_eq!(body["modules"][0]["tree_unavailable"], "not_yet_measured");
14218 assert_eq!(
14219 body["modules"][0]["pending_reload"]["image"]["status"], "match",
14220 "the tree timeout must not hide a completed image observation"
14221 );
14222 }
14223
14224 #[cfg(any(target_os = "linux", target_os = "macos"))]
14225 #[tokio::test]
14226 async fn cold_explicit_provenance_completes_while_list_stays_bounded() {
14227 let mut handler = ControlHandler::new(Arc::new(Registry::default()))
14228 .with_daemon_provenance(
14229 std::process::id(),
14230 0,
14231 Some(std::env::current_exe().unwrap()),
14232 None,
14233 None,
14234 );
14235 let (probe, release, began) = ExecutableIdentityProbe::held();
14236 handler.daemon_provenance.probe = probe;
14237 let observe = |handler: ControlHandler, corr| {
14238 tokio::spawn(async move {
14239 let (ctx, _) = route_ctx(ConnectionId::new(corr));
14240 let frame =
14241 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, Vec::new())
14242 .unwrap();
14243 handler
14244 .handle_supervisor_provenance(&ctx, frame, None)
14245 .await
14246 .unwrap()
14247 })
14248 };
14249 let explicit = observe(handler.clone(), 114);
14250 began.await.unwrap();
14251 let concurrent = observe(handler.clone(), 115);
14252 tokio::time::sleep(Duration::from_millis(300)).await;
14255 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14256 let list = tokio::time::timeout(
14257 Duration::from_secs(1),
14258 handler.handle_supervisor_list(frame),
14259 )
14260 .await;
14261 let explicit_was_pending = !explicit.is_finished() && !concurrent.is_finished();
14262 drop(release);
14263 let replies = explicit.await.unwrap();
14264 let concurrent_replies = concurrent.await.unwrap();
14265 assert!(
14266 explicit_was_pending,
14267 "explicit provenance must await the full observation"
14268 );
14269 let list = list
14270 .expect("a concurrent explicit probe must not stall list")
14271 .unwrap();
14272 let list_body: Value = serde_json::from_slice(&list[0].body).unwrap();
14273 assert_eq!(list_body["op"], "supervisor.list");
14274 for replies in [replies, concurrent_replies] {
14275 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14276 assert_eq!(
14277 body["daemon"]["daemon_observed"]["running_image"]["status"],
14278 "match"
14279 );
14280 }
14281 }
14282
14283 #[tokio::test]
14284 async fn supervisor_list_slow_subtree_does_not_hold_connection_reader() {
14285 let _guard = crate::subtree_resources::fault_read_guard().await;
14286 let registry = Arc::new(Registry::default());
14287 let handle = SupervisorHandle::new();
14288 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
14289 .with_handle(handle.clone());
14290 let module = supervisor
14291 .supervise_configured(
14292 crate::ModuleSpec {
14293 carry_containment: false,
14294 module_id: "memory-read".into(),
14295 program: PathBuf::from("disabled-module"),
14296 args: Vec::new(),
14297 env: Vec::new(),
14298 reserved: false,
14299 reserved_prefixes: Vec::new(),
14300 protocol: ModuleProtocol::Subc,
14301 overlap: Default::default(),
14302 },
14303 true,
14304 )
14305 .unwrap();
14306 let (target, release, began) = crate::subtree_resources::held();
14307 let stalled_target = target.clone();
14310 let stalled = tokio::task::spawn_blocking(move || {
14311 stalled_target.tree(&subc_os::ProcessTable::default())
14312 });
14313 began.await.unwrap();
14314 module.set_subtree_target_for_test(target);
14315 let handler = ControlHandler::new(registry).with_supervisor(handle);
14316 let (ctx, mut rx) = route_ctx(ConnectionId::new(112));
14317 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14318 let replies = handler
14319 .handle_client_control_request(
14320 &ctx,
14321 frame,
14322 ClientControlRequest::SupervisorList {},
14323 &mut None,
14324 )
14325 .await
14326 .unwrap();
14327 assert!(
14328 replies.is_empty(),
14329 "list reply must be detached from the connection reader"
14330 );
14331 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14332 let replies = handler
14333 .handle_client_control_request(
14334 &ctx,
14335 frame,
14336 ClientControlRequest::SupervisorSpawnSnapshot {},
14337 &mut None,
14338 )
14339 .await
14340 .unwrap();
14341 assert_eq!(
14342 replies.len(),
14343 1,
14344 "another request must complete while the subtree read stalls"
14345 );
14346 let reply = tokio::time::timeout(Duration::from_secs(5), rx.recv())
14347 .await
14348 .unwrap()
14349 .unwrap();
14350 release.send(()).unwrap();
14351 drop(release);
14352 stalled.await.unwrap();
14353 let body: Value = serde_json::from_slice(&reply.frame.body).unwrap();
14354 assert!(
14355 body["modules"][0].get("tree").is_none(),
14356 "an unknown tree reading must be omitted after the bounded read expires"
14357 );
14358 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
14359 let replies = handler.handle_supervisor_list(frame).await.unwrap();
14360 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14361 assert_eq!(
14362 body["modules"][0]["tree"]["source"], "process_tree",
14363 "the next request must read afresh"
14364 );
14365 assert_eq!(body["modules"][0]["tree"]["process_count"], 1);
14366 }
14367
14368 #[tokio::test]
14369 async fn supervisor_list_reports_loaded_config_and_omits_unconfigured() {
14370 let registry = Arc::new(Registry::default());
14371 let handle = SupervisorHandle::new();
14372 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
14373 .with_handle(handle.clone());
14374 supervisor
14375 .supervise_configured(
14376 crate::ModuleSpec {
14377 carry_containment: false,
14378 module_id: "drift".into(),
14379 program: PathBuf::from("disabled-module"),
14380 args: Vec::new(),
14381 env: Vec::new(),
14382 reserved: false,
14383 reserved_prefixes: Vec::new(),
14384 protocol: ModuleProtocol::Subc,
14385 overlap: Default::default(),
14386 },
14387 false,
14388 )
14389 .unwrap();
14390 let handler = ControlHandler::new(registry).with_supervisor(handle);
14391 for configured in [None, Some(false), Some(true)] {
14392 handler.capability_evaluator.configure(
14393 configured.map(|enabled| ("drift".to_string(), enabled)),
14394 BTreeMap::new(),
14395 );
14396 let frame =
14397 Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14398 let replies = handler
14399 .handle_supervisor_set_enabled(frame, "drift".into(), false, Principal::Direct)
14400 .await
14401 .unwrap();
14402 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14403 assert!(body.get("configured_enabled").is_none());
14404 assert_eq!(body["op"], "supervisor.ack");
14405 let frame =
14406 Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14407 let replies = handler.handle_supervisor_list(frame).await.unwrap();
14408 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14409 assert_eq!(
14410 body["modules"][0].get("configured_enabled"),
14411 configured.map(Value::Bool).as_ref()
14412 );
14413 assert!(body["modules"][0].get("tree").is_none());
14414 assert!(body["modules"][0].get("subtree").is_none());
14415 }
14416 }
14417
14418 #[tokio::test(flavor = "current_thread")]
14419 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
14420 let handler = ControlHandler::new(Arc::new(Registry::default()));
14421 let capture = EventCapture::default();
14422 let _subscriber =
14423 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14424 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
14425 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14426
14427 let _ = handler
14428 .handle_client_control_request(
14429 &ctx,
14430 frame,
14431 ClientControlRequest::SupervisorSetEnabled {
14432 module_id: "broca".to_string(),
14433 enabled: false,
14434 },
14435 &mut None,
14436 )
14437 .await
14438 .unwrap();
14439
14440 let events = capture
14441 .events()
14442 .into_iter()
14443 .filter(|event| {
14444 event.target == "control"
14445 && event.fields.get("message").map(String::as_str)
14446 == Some("supervisor request received")
14447 })
14448 .collect::<Vec<_>>();
14449 assert_eq!(events.len(), 1, "one supervisor request log line");
14450 let event = &events[0];
14451 assert_eq!(event.level, tracing::Level::INFO);
14452 assert_eq!(
14453 event.fields.get("op"),
14454 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
14455 );
14456 assert_eq!(
14457 event.fields.get("module_id"),
14458 Some(&"\"broca\"".to_string())
14459 );
14460 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
14461 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
14462 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
14463 }
14464
14465 #[tokio::test(flavor = "current_thread")]
14466 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
14467 let registry = Arc::new(Registry::default());
14468 let module_connection = ConnectionId::new(303);
14469 registry
14470 .register_with_control_ops(
14471 manifest("aft", PROTOCOL_VERSION),
14472 PROTOCOL_VERSION,
14473 module_connection,
14474 module_baseline_control_ops(),
14475 )
14476 .unwrap();
14477 let handler = ControlHandler::new(registry);
14478 let capture = EventCapture::default();
14479 let _subscriber =
14480 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14481 let (ctx, _rx) = route_ctx(module_connection);
14482 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14483
14484 let _ = handler
14485 .handle_client_control_request(
14486 &ctx,
14487 frame,
14488 ClientControlRequest::SupervisorSetEnabled {
14489 module_id: "broca".to_string(),
14490 enabled: false,
14491 },
14492 &mut None,
14493 )
14494 .await
14495 .unwrap();
14496
14497 let callers = capture
14498 .events()
14499 .into_iter()
14500 .filter(|event| {
14501 event.fields.get("message").map(String::as_str)
14502 == Some("supervisor request received")
14503 })
14504 .map(|event| event.fields.get("caller").cloned())
14505 .collect::<Vec<_>>();
14506 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
14507 }
14508
14509 #[tokio::test(flavor = "current_thread")]
14510 async fn supervisor_list_does_not_log_request_received() {
14511 let handler = ControlHandler::new(Arc::new(Registry::default()));
14512 let capture = EventCapture::default();
14513 let _subscriber =
14514 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14515 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
14516 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14517
14518 handler
14519 .handle_client_control_request(
14520 &ctx,
14521 frame,
14522 ClientControlRequest::SupervisorList {},
14523 &mut None,
14524 )
14525 .await
14526 .unwrap();
14527
14528 assert!(capture.events().into_iter().all(|event| {
14529 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
14530 }));
14531 }
14532
14533 #[tokio::test(flavor = "current_thread")]
14534 async fn supervisor_rescan_preview_does_not_log_request_received() {
14535 let handler = ControlHandler::new(Arc::new(Registry::default()));
14536 let capture = EventCapture::default();
14537 let _subscriber =
14538 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14539 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
14540 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
14541
14542 handler
14543 .handle_client_control_request(
14544 &ctx,
14545 frame,
14546 ClientControlRequest::SupervisorRescan { preview: true },
14547 &mut None,
14548 )
14549 .await
14550 .unwrap();
14551
14552 assert!(capture.events().into_iter().all(|event| {
14553 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
14554 }));
14555 }
14556
14557 struct ApproveOperator;
14558 impl crate::operator_confirm::OperatorProvider for ApproveOperator {
14559 fn prompt(
14560 &self,
14561 _: &str,
14562 _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
14563 ) -> crate::operator_confirm::ProviderResult {
14564 crate::operator_confirm::ProviderResult::Approved
14565 }
14566 }
14567
14568 #[tokio::test]
14569 async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
14570 assert_eq!(
14571 ControlHandler::default().privileged_operator_prompt,
14572 PrivilegedOperatorPrompt::Log
14573 );
14574 for (initial, changed, running) in [
14575 ("log", "enforce", PrivilegedOperatorPrompt::Log),
14576 ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
14577 ] {
14578 let dir = ScratchDir::new("rescan-privileged-operator-prompt");
14579 let path = dir.join("subc.jsonc");
14580 std::fs::write(
14581 &path,
14582 json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
14583 )
14584 .unwrap();
14585 let config = crate::daemon_config::load(&path).unwrap().unwrap();
14586 let registry = Arc::new(Registry::default());
14587 let handle = SupervisorHandle::new();
14588 let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
14589 .with_handle(handle.clone());
14590 let handler = ControlHandler::new(registry)
14591 .with_supervisor(handle)
14592 .with_privileged_operator_prompt(config.privileged_operator_prompt)
14593 .with_supervisor_rescan(supervisor, &path, config.port);
14594 handler.forwarding.operator_confirms().configure(
14595 Arc::new(ApproveOperator),
14596 crate::operator_confirm::OperatorLimits::default(),
14597 );
14598 let (ctx, _rx) = route_ctx(ConnectionId::new(104));
14599
14600 for (disk_mode, needs_restart) in [
14601 (Some(initial), false),
14602 (Some(changed), true),
14603 (Some(changed), true),
14604 (None, running == PrivilegedOperatorPrompt::Enforce),
14605 (Some(initial), false),
14606 ] {
14607 let mut doc = json!({"version":1});
14608 if let Some(mode) = disk_mode {
14609 doc["privileged_operator_prompt"] = json!(mode);
14610 }
14611 std::fs::write(&path, doc.to_string()).unwrap();
14612 for preview in [true, false] {
14613 let request = ClientControlRequest::SupervisorRescan { preview };
14614 let frame = Frame::build(
14615 FrameType::Request,
14616 control_flags(),
14617 0,
14618 0,
14619 1,
14620 serde_json::to_vec(&request).unwrap(),
14621 )
14622 .unwrap();
14623 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
14624 assert_eq!(frames.len(), 1);
14625 let ClientControlResponse::SupervisorRescan { result } =
14626 serde_json::from_slice(&frames[0].body).unwrap()
14627 else {
14628 panic!("expected a rescan response");
14629 };
14630 let expected = if needs_restart {
14631 vec!["privileged_operator_prompt"]
14632 } else {
14633 Vec::new()
14634 };
14635 assert_eq!(result.restart_required, expected);
14636 assert_eq!(handler.privileged_operator_prompt, running);
14637 assert_eq!(
14638 handler.rescan.as_ref().unwrap().privileged_operator_prompt,
14639 running
14640 );
14641 }
14642 }
14643 }
14644 }
14645
14646 #[tokio::test]
14647 async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
14648 let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
14649 let path = dir.join("subc.jsonc");
14650 let registry = Arc::new(Registry::default());
14651 let handle = SupervisorHandle::new();
14652 let supervisor =
14653 Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
14654 let handler = ControlHandler::new(registry)
14655 .with_supervisor(handle)
14656 .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
14657 .with_supervisor_rescan(supervisor, &path, None);
14658 handler.forwarding.operator_confirms().configure(
14659 Arc::new(ApproveOperator),
14660 crate::operator_confirm::OperatorLimits::default(),
14661 );
14662 let (ctx, _rx) = route_ctx(ConnectionId::new(105));
14663 for preview in [true, false] {
14664 std::fs::write(
14665 &path,
14666 r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
14667 )
14668 .unwrap();
14669 let request = ClientControlRequest::SupervisorRescan { preview };
14670 let frame = Frame::build(
14671 FrameType::Request,
14672 control_flags(),
14673 0,
14674 0,
14675 1,
14676 serde_json::to_vec(&request).unwrap(),
14677 )
14678 .unwrap();
14679 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
14680 assert_eq!(frames.len(), 1);
14681 assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
14682 assert_eq!(
14683 handler.privileged_operator_prompt,
14684 PrivilegedOperatorPrompt::Enforce
14685 );
14686 }
14687 }
14688
14689 #[tokio::test]
14694 async fn route_open_refusal_names_the_check_that_refused() {
14695 let handler = ControlHandler::new(Arc::new(Registry::default()));
14696 let capture = EventCapture::default();
14697 let _subscriber =
14698 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14699 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
14700 let response = handler
14701 .handle_control_frame(
14702 &ctx,
14703 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
14704 )
14705 .await
14706 .unwrap();
14707
14708 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
14709 let event = capture
14710 .events()
14711 .into_iter()
14712 .find(|event| {
14713 event.target == "control"
14714 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
14715 })
14716 .expect("route.open refusal event");
14717 assert_eq!(
14718 event.fields.get("reason"),
14719 Some(&"\"not_registered\"".to_string())
14720 );
14721 }
14722
14723 #[tokio::test]
14724 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
14725 let registry = Arc::new(Registry::default());
14726 let supervisor_handle = SupervisorHandle::new();
14727 let supervisor =
14728 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
14729 .with_handle(supervisor_handle.clone())
14730 .with_connection_file_path(std::env::temp_dir().join(format!(
14731 "subc-route-open-refusal-info-{}",
14732 std::process::id()
14733 )));
14734 let module = supervisor
14735 .supervise_configured(
14736 ModuleSpec {
14737 carry_containment: false,
14738 module_id: "warming".to_string(),
14739 program: fake_aft_stub_path(),
14740 args: Vec::new(),
14741 env: Vec::new(),
14742 reserved: false,
14743 reserved_prefixes: Vec::new(),
14744 protocol: ModuleProtocol::Subc,
14745 overlap: Default::default(),
14746 },
14747 true,
14748 )
14749 .unwrap();
14750 assert_eq!(module.state().unwrap(), ModuleState::Running);
14751
14752 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
14753 assert!(handler
14754 .counters()
14755 .snapshot()
14756 .get("route_open_refused_by_code")
14757 .is_none());
14758 let capture = EventCapture::default();
14759 let _subscriber =
14760 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14761 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
14762 let response = handler
14763 .handle_control_frame(
14764 &ctx,
14765 route_open_frame(394, "warming", unique_project_root("refusal-info")),
14766 )
14767 .await
14768 .unwrap();
14769 module.stop().await.unwrap();
14770
14771 assert_eq!(parse_error(&response[0])["code"], "module_warming");
14772 let event = capture
14773 .events()
14774 .into_iter()
14775 .find(|event| {
14776 event.target == "control"
14777 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
14778 })
14779 .expect("route.open refusal event");
14780 assert_eq!(
14781 event.fields.get("module_id"),
14782 Some(&"\"warming\"".to_string())
14783 );
14784 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
14785 assert_eq!(
14786 event.fields.get("reason"),
14787 Some(&"\"supervised_not_registered\"".to_string())
14788 );
14789 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
14790 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
14791 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
14792 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
14793 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
14794 assert_eq!(event.fields.get("harness"), Some(&"\"unit\"".to_string()));
14795 assert_eq!(
14796 handler.counters().snapshot()["route_open_refused_by_code"],
14797 json!({ "module_warming": 1 })
14798 );
14799 }
14800
14801 const OUTAGE_START: &str = "route.open refusing module: not serving";
14802 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
14803
14804 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
14805 capture
14806 .events()
14807 .into_iter()
14808 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
14809 .collect()
14810 }
14811
14812 fn supervise_stub(
14813 registry: &Arc<Registry>,
14814 module_id: &str,
14815 enabled: bool,
14816 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
14817 let supervisor_handle = SupervisorHandle::new();
14818 let supervisor =
14819 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
14820 .with_handle(supervisor_handle.clone())
14821 .with_connection_file_path(std::env::temp_dir().join(format!(
14822 "subc-route-outage-{module_id}-{}",
14823 std::process::id()
14824 )));
14825 let module = supervisor
14826 .supervise_configured(
14827 ModuleSpec {
14828 carry_containment: false,
14829 module_id: module_id.to_string(),
14830 program: fake_aft_stub_path(),
14831 args: Vec::new(),
14832 env: Vec::new(),
14833 reserved: false,
14834 reserved_prefixes: Vec::new(),
14835 protocol: ModuleProtocol::Subc,
14836 overlap: Default::default(),
14837 },
14838 enabled,
14839 )
14840 .unwrap();
14841 (supervisor_handle, module)
14842 }
14843
14844 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
14845 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
14846 module_id: module_id.to_string(),
14847 drain_timeout_ms: Some(50),
14848 })
14849 .unwrap();
14850 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
14851 }
14852
14853 #[test]
14857 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
14858 let registry = Arc::new(Registry::default());
14859 let forwarding = Arc::new(ForwardingTable::default());
14860 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14861 let second = ControlHandler::with_forwarding(registry, forwarding);
14862 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
14863 }
14864
14865 #[tokio::test(flavor = "current_thread")]
14869 async fn route_open_unknown_module_refusals_add_no_outage_state() {
14870 let handler = ControlHandler::new(Arc::new(Registry::default()));
14871 let capture = EventCapture::default();
14872 let _subscriber =
14873 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14874 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
14875 for corr in 0..8 {
14876 let response = handler
14877 .handle_control_frame(
14878 &ctx,
14879 route_open_frame(
14880 380 + corr,
14881 &format!("nobody-{corr}"),
14882 unique_project_root("outage-unknown"),
14883 ),
14884 )
14885 .await
14886 .unwrap();
14887 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
14888 }
14889
14890 assert_eq!(handler.route_outages.tracked_module_count(), 0);
14891 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
14892 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
14893 }
14894
14895 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14900 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
14901 let registry = Arc::new(Registry::default());
14902 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
14903 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
14904 let capture = EventCapture::default();
14905 let _subscriber =
14906 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14907 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
14908 handler.route_outages.record_accepted("outage-restart");
14911
14912 let response = handler
14913 .handle_control_frame(
14914 &ctx,
14915 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
14916 )
14917 .await
14918 .unwrap();
14919 assert_eq!(response[0].header.ty, FrameType::Error);
14920 let starts = outage_lines(&capture, OUTAGE_START);
14921 assert_eq!(starts.len(), 1, "{starts:?}");
14922 assert_eq!(starts[0].level, tracing::Level::WARN);
14923 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
14924 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
14925 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
14926 handler.route_outages.record_accepted("outage-restart");
14927 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
14928
14929 let restart = handler
14930 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
14931 .await
14932 .unwrap();
14933 assert_eq!(
14934 restart[0].header.ty,
14935 FrameType::Response,
14936 "{:?}",
14937 parse_error(&restart[0])
14938 );
14939 handler
14940 .handle_control_frame(
14941 &ctx,
14942 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
14943 )
14944 .await
14945 .unwrap();
14946 module.stop().await.unwrap();
14947
14948 let starts = outage_lines(&capture, OUTAGE_START);
14949 assert_eq!(starts.len(), 2, "{starts:?}");
14950 assert_eq!(starts[1].level, tracing::Level::INFO);
14951 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
14952 }
14953
14954 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14958 async fn failed_operator_restart_leaves_no_operator_mark() {
14959 let registry = Arc::new(Registry::default());
14960 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
14961 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
14962 let capture = EventCapture::default();
14963 let _subscriber =
14964 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14965 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
14966 handler.route_outages.record_accepted("outage-disabled");
14967
14968 let restart = handler
14969 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
14970 .await
14971 .unwrap();
14972 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
14973 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
14974
14975 handler
14976 .handle_control_frame(
14977 &ctx,
14978 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
14979 )
14980 .await
14981 .unwrap();
14982 let starts = outage_lines(&capture, OUTAGE_START);
14983 assert_eq!(starts.len(), 1, "{starts:?}");
14984 assert_eq!(starts[0].level, tracing::Level::WARN);
14985 }
14986
14987 #[tokio::test(flavor = "current_thread")]
14988 async fn route_open_unknown_module_escapes_target_module_id() {
14989 let handler = ControlHandler::new(Arc::new(Registry::default()));
14990 let capture = EventCapture::default();
14991 let _subscriber =
14992 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14993 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
14994 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
14995 let response = handler
14996 .handle_control_frame(
14997 &ctx,
14998 route_open_frame(
14999 395,
15000 hostile_module_id,
15001 unique_project_root("hostile-target-module-id"),
15002 ),
15003 )
15004 .await
15005 .unwrap();
15006
15007 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
15008 let event = capture
15009 .events()
15010 .into_iter()
15011 .find(|event| {
15012 event.target == "control"
15013 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
15014 })
15015 .expect("route.open unknown-module refusal event");
15016 let logged = event.fields.get("module_id").expect("module_id field");
15017 assert!(!logged.bytes().any(|byte| byte < 0x20));
15018 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
15019 }
15020
15021 #[tokio::test(flavor = "current_thread")]
15024 async fn route_open_refusal_carries_daemon_origin() {
15025 let handler = ControlHandler::new(Arc::new(Registry::default()));
15026 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
15027 let response = handler
15028 .handle_control_frame(
15029 &ctx,
15030 route_open_frame(395, "nobody", unique_project_root("refusal-origin")),
15031 )
15032 .await
15033 .unwrap();
15034
15035 assert_eq!(response.len(), 1);
15036 assert_eq!(response[0].header.ty, FrameType::Error);
15037 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
15038 assert!(
15039 response[0].header.flags.is_daemon_origin(),
15040 "route.open refusal must carry daemon origin: {:?}",
15041 response[0].header.flags
15042 );
15043 }
15044
15045 #[tokio::test(flavor = "current_thread")]
15049 async fn route_open_to_draining_module_refuses_module_reloading_with_daemon_origin() {
15050 let registry = Arc::new(Registry::default());
15051 let forwarding = Arc::new(ForwardingTable::default());
15052 let handler =
15053 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15054 let module_connection = ConnectionId::new(95);
15055 let (module_ctx, mut module_rx) = route_ctx(module_connection);
15056 hello_via_sink(
15057 &handler,
15058 &module_ctx,
15059 &mut module_rx,
15060 hello_frame("aft", PROTOCOL_VERSION, 395),
15061 )
15062 .await;
15063 forwarding
15064 .begin_module_drain("aft", RouteCloseReason::Restart)
15065 .unwrap()
15066 .expect("the module endpoint drains");
15067
15068 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(96));
15069 let response = handler
15070 .handle_control_frame(
15071 &client_ctx,
15072 route_open_frame(396, "aft", unique_project_root("drain-origin")),
15073 )
15074 .await
15075 .unwrap();
15076
15077 assert_eq!(response.len(), 1);
15078 assert_eq!(response[0].header.ty, FrameType::Error);
15079 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
15080 assert!(
15081 response[0].header.flags.is_daemon_origin(),
15082 "drain module_reloading must carry daemon origin: {:?}",
15083 response[0].header.flags
15084 );
15085 }
15086
15087 #[tokio::test(flavor = "current_thread")]
15091 async fn route_open_module_rejection_with_daemon_code_carries_no_daemon_origin() {
15092 let registry = Arc::new(Registry::default());
15093 let forwarding = Arc::new(ForwardingTable::default());
15094 let handler =
15095 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15096 let module_connection = ConnectionId::new(95);
15097 let (module_ctx, mut module_rx) = route_ctx(module_connection);
15098 hello_via_sink(
15099 &handler,
15100 &module_ctx,
15101 &mut module_rx,
15102 hello_frame("aft", PROTOCOL_VERSION, 395),
15103 )
15104 .await;
15105
15106 let client_connection = ConnectionId::new(96);
15107 let (client_ctx, _client_rx) = route_ctx(client_connection);
15108 let (route_task, bind) = relay_route_open(
15109 &handler,
15110 client_connection,
15111 &client_ctx.egress,
15112 &mut module_rx,
15113 396,
15114 "aft",
15115 "spoofed-reloading",
15116 )
15117 .await;
15118 let rejection = Frame::build(
15121 FrameType::Error,
15122 crate::router::daemon_error_flags(),
15123 0,
15124 0,
15125 bind.header.corr,
15126 serde_json::to_vec(&ErrorBody::new(
15127 "module_reloading",
15128 "module refused route.bind",
15129 ))
15130 .unwrap(),
15131 )
15132 .unwrap();
15133 handler
15134 .handle_control_frame(&module_ctx, rejection)
15135 .await
15136 .unwrap();
15137
15138 let response = route_task.await.unwrap();
15139 assert_eq!(response.len(), 1);
15140 assert_eq!(response[0].header.ty, FrameType::Error);
15141 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
15142 assert!(
15143 !response[0].header.flags.is_daemon_origin(),
15144 "a module's own rejection must not carry daemon origin: {:?}",
15145 response[0].header.flags
15146 );
15147 }
15148
15149 #[tokio::test(flavor = "current_thread")]
15150 async fn route_open_module_rejection_uses_daemon_counter_key() {
15151 let registry = Arc::new(Registry::default());
15152 let forwarding = Arc::new(ForwardingTable::default());
15153 let handler =
15154 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15155 let module_connection = ConnectionId::new(95);
15156 let (module_ctx, mut module_rx) = route_ctx(module_connection);
15157 hello_via_sink(
15158 &handler,
15159 &module_ctx,
15160 &mut module_rx,
15161 hello_frame("aft", PROTOCOL_VERSION, 395),
15162 )
15163 .await;
15164
15165 let client_connection = ConnectionId::new(96);
15166 let (client_ctx, _client_rx) = route_ctx(client_connection);
15167 let capture = EventCapture::default();
15168 let _subscriber =
15169 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15170 let (route_task, bind) = relay_route_open(
15171 &handler,
15172 client_connection,
15173 &client_ctx.egress,
15174 &mut module_rx,
15175 396,
15176 "aft",
15177 "hostile-module-code",
15178 )
15179 .await;
15180 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
15181 let rejection = Frame::build(
15182 FrameType::Error,
15183 control_flags(),
15184 0,
15185 0,
15186 bind.header.corr,
15187 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
15188 )
15189 .unwrap();
15190 handler
15191 .handle_control_frame(&module_ctx, rejection)
15192 .await
15193 .unwrap();
15194
15195 let response = route_task.await.unwrap();
15196 assert_eq!(parse_error(&response[0])["code"], hostile_code);
15197 let counters = handler.counters().snapshot();
15198 assert_eq!(
15199 counters["route_open_refused_by_code"],
15200 json!({ "module_rejected": 1 })
15201 );
15202 assert!(counters["route_open_refused_by_code"]
15203 .get(hostile_code)
15204 .is_none());
15205
15206 let event = capture
15207 .events()
15208 .into_iter()
15209 .find(|event| {
15210 event.target == "control"
15211 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
15212 })
15213 .expect("route.open module-rejection refusal event");
15214 let logged = event.fields.get("module_code").expect("module_code field");
15215 assert!(!logged.bytes().any(|byte| byte < 0x20));
15216 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
15217 }
15218
15219 #[tokio::test]
15220 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
15221 let registry = Arc::new(Registry::default());
15222 let supervisor_handle = SupervisorHandle::new();
15223 let missing_program = std::env::temp_dir().join(format!(
15224 "subc-route-open-missing-program-{}",
15225 std::process::id()
15226 ));
15227 let supervisor =
15228 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
15229 .with_handle(supervisor_handle.clone());
15230 let module = supervisor
15231 .supervise_configured(
15232 ModuleSpec {
15233 carry_containment: false,
15234 module_id: "failed".to_string(),
15235 program: missing_program,
15236 args: Vec::new(),
15237 env: Vec::new(),
15238 reserved: false,
15239 reserved_prefixes: Vec::new(),
15240 protocol: ModuleProtocol::Subc,
15241 overlap: Default::default(),
15242 },
15243 true,
15244 )
15245 .unwrap();
15246 assert_eq!(module.state().unwrap(), ModuleState::Failed);
15247
15248 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
15249 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
15250 let response = handler
15251 .handle_control_frame(
15252 &ctx,
15253 route_open_frame(305, "failed", unique_project_root("failed")),
15254 )
15255 .await
15256 .unwrap();
15257
15258 assert_eq!(response[0].header.ty, FrameType::Error);
15259 let error = parse_error(&response[0]);
15260 assert_eq!(error["code"], "target_unavailable");
15261 assert!(error["message"]
15262 .as_str()
15263 .unwrap()
15264 .contains("state=failed, enabled=true, live=false"));
15265 }
15266
15267 #[tokio::test]
15268 async fn route_open_role_mismatch_remains_target_unavailable() {
15269 let registry = Arc::new(Registry::default());
15270 let handler = ControlHandler::new(Arc::clone(®istry));
15271 handler
15272 .handle_control(
15273 ConnectionId::new(41),
15274 non_routable_hello_frame_with_control_ops("health-only", 306, None),
15275 )
15276 .unwrap();
15277
15278 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
15279 let response = handler
15280 .handle_control_frame(
15281 &ctx,
15282 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
15283 )
15284 .await
15285 .unwrap();
15286
15287 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
15288 assert!(parse_error(&response[0])["message"]
15289 .as_str()
15290 .unwrap()
15291 .contains("does not provide the requested target"));
15292 }
15293
15294 #[tokio::test]
15295 async fn route_open_inactive_registration_remains_target_unavailable() {
15296 let registry = Arc::new(Registry::default());
15297 let handler = ControlHandler::new(Arc::clone(®istry));
15298 handler
15299 .handle_control(
15300 ConnectionId::new(43),
15301 hello_frame("inactive", PROTOCOL_VERSION, 308),
15302 )
15303 .unwrap();
15304 assert!(registry
15305 .set_module_state_for_test("inactive", ChannelState::Closed)
15306 .unwrap());
15307
15308 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
15309 let response = handler
15310 .handle_control_frame(
15311 &ctx,
15312 route_open_frame(309, "inactive", unique_project_root("inactive")),
15313 )
15314 .await
15315 .unwrap();
15316
15317 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
15318 assert!(parse_error(&response[0])["message"]
15319 .as_str()
15320 .unwrap()
15321 .contains("is not active"));
15322 }
15323
15324 #[tokio::test]
15325 async fn late_health_reply_is_recorded_through_the_module_response_path() {
15326 let registry = Arc::new(Registry::default());
15327 let forwarding = Arc::new(ForwardingTable::default());
15328 let supervisor_handle = SupervisorHandle::new();
15329 let supervisor =
15330 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
15331 .with_forwarding(Arc::clone(&forwarding))
15332 .with_handle(supervisor_handle.clone());
15333 let module = supervisor
15334 .supervise_configured(
15335 crate::ModuleSpec {
15336 carry_containment: false,
15337 module_id: "late-health-response".to_string(),
15338 program: PathBuf::from("disabled-module"),
15339 args: Vec::new(),
15340 env: Vec::new(),
15341 reserved: false,
15342 reserved_prefixes: Vec::new(),
15343 protocol: ModuleProtocol::Subc,
15344 overlap: Default::default(),
15345 },
15346 false,
15347 )
15348 .unwrap();
15349 let handler =
15350 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15351 .with_supervisor(supervisor_handle);
15352 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
15353 handler
15354 .handle_control_frame(
15355 &module_ctx,
15356 hello_frame_with_control_ops(
15357 "late-health-response",
15358 PROTOCOL_VERSION,
15359 7,
15360 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
15361 ),
15362 )
15363 .await
15364 .unwrap();
15365 let probe_started_at = Instant::now() - Duration::from_millis(80);
15366 let pending = forwarding
15367 .begin_health_probe_rpc_for(
15368 "late-health-response",
15369 MODULE_CONTROL_OP_HEALTH_CHECK,
15370 probe_started_at,
15371 Instant::now() - Duration::from_millis(1),
15372 )
15373 .unwrap();
15374 assert!(forwarding
15375 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
15376 .unwrap());
15377
15378 let responses = handler
15379 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
15380 .await
15381 .unwrap();
15382
15383 assert!(responses.is_empty());
15384 let health = module.status().unwrap().health;
15385 assert_eq!(health.late_answer_count, 1);
15386 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
15387 }
15388
15389 #[tokio::test]
15390 async fn health_probe_timeout_and_module_death_are_typed() {
15391 let registry = Arc::new(Registry::default());
15392 let forwarding = Arc::new(ForwardingTable::default());
15393 let handler =
15394 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15395 .with_health_probe_timeout(Duration::from_millis(50));
15396 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
15397 hello_via_sink(
15398 &handler,
15399 &module_ctx,
15400 &mut module_rx,
15401 hello_frame_with_control_ops(
15402 "aft",
15403 PROTOCOL_VERSION,
15404 7,
15405 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
15406 ),
15407 )
15408 .await;
15409
15410 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
15411 let responses = handler
15412 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
15413 .await
15414 .unwrap();
15415 assert_eq!(responses[0].header.ty, FrameType::Error);
15416 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
15417 let _ = module_rx.try_recv();
15418
15419 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
15420 let health_handler = handler.clone();
15421 let death_task = tokio::spawn(async move {
15422 health_handler
15423 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
15424 .await
15425 .unwrap()
15426 });
15427 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
15428 .await
15429 .unwrap()
15430 .unwrap();
15431 handler
15432 .cleanup_connection(module_ctx.connection_id)
15433 .unwrap();
15434 let responses = death_task.await.unwrap();
15435 assert_eq!(responses[0].header.ty, FrameType::Error);
15436 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
15437 }
15438
15439 #[test]
15440 fn hello_requires_exact_protocol_version() {
15441 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
15442 let registry = Arc::new(Registry::default());
15443 let handler = ControlHandler::new(Arc::clone(®istry));
15444 let responses = handler
15445 .handle_control(
15446 ConnectionId::new(connection),
15447 hello_frame("aft", offered, 9),
15448 )
15449 .unwrap();
15450
15451 assert_eq!(responses.len(), 1);
15452 assert_eq!(responses[0].header.ty, FrameType::Error);
15453 let error = parse_error(&responses[0]);
15454 assert_eq!(error["code"], "version_unsupported");
15455 assert!(registry.get_module("aft").unwrap().is_none());
15456 assert_eq!(registry.active_registration_count().unwrap(), 0);
15457 }
15458 }
15459
15460 #[test]
15461 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
15462 let registry = Arc::new(Registry::default());
15463 let forwarding = Arc::new(ForwardingTable::default());
15464 let handler =
15465 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15466 let module_connection = ConnectionId::new(301);
15467 let registration = registry
15468 .register_with_control_ops(
15469 manifest("aft-push", PROTOCOL_VERSION),
15470 PROTOCOL_VERSION,
15471 module_connection,
15472 module_baseline_control_ops(),
15473 )
15474 .unwrap();
15475 let (module_tx, _module_rx) = mpsc::channel(8);
15476 let endpoint = forwarding
15477 .register_module_connection(
15478 module_connection,
15479 "aft-push".to_string(),
15480 PROTOCOL_VERSION,
15481 manifest_concurrency(®istration.manifest),
15482 FrameSink::new(module_tx),
15483 )
15484 .unwrap();
15485
15486 let unknown = Frame::build(
15488 FrameType::Push,
15489 control_flags(),
15490 0,
15491 0,
15492 5,
15493 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
15494 )
15495 .unwrap();
15496 let out = handler.handle_status_update(endpoint, unknown).unwrap();
15497 assert!(
15498 out.is_empty(),
15499 "unknown push op must be ignored, got {out:?}"
15500 );
15501
15502 let malformed = Frame::build(
15504 FrameType::Push,
15505 control_flags(),
15506 0,
15507 0,
15508 6,
15509 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
15510 )
15511 .unwrap();
15512 let out = handler.handle_status_update(endpoint, malformed).unwrap();
15513 assert_eq!(out.len(), 1);
15514 assert_eq!(out[0].header.ty, FrameType::Error);
15515 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
15516 }
15517
15518 #[test]
15519 fn hello_rejected_when_connection_already_owns_client_routes() {
15520 let registry = Arc::new(Registry::default());
15521 let forwarding = Arc::new(ForwardingTable::default());
15522 let handler =
15523 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15524 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
15526 let client_connection = ConnectionId::new(202);
15527
15528 let responses = handler
15531 .handle_control(
15532 client_connection,
15533 hello_frame("aft-second", PROTOCOL_VERSION, 9),
15534 )
15535 .unwrap();
15536 assert_eq!(responses[0].header.ty, FrameType::Error);
15537 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
15538 assert!(registry.get_module("aft-second").unwrap().is_none());
15539 }
15540
15541 #[tokio::test]
15542 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
15543 let registry = Arc::new(Registry::default());
15544 let forwarding = Arc::new(ForwardingTable::default());
15545 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
15546 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
15547 hello_via_sink(
15548 &handler,
15549 &module_ctx,
15550 &mut module_rx,
15551 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
15552 )
15553 .await;
15554 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
15555 let pending = forwarding
15556 .begin_route_bind_relay_for_test(
15557 client_ctx.connection_id,
15558 client_ctx.egress.clone(),
15559 2,
15560 "alpha",
15561 )
15562 .unwrap();
15563 forwarding
15564 .complete_pending_relay(
15565 module_ctx.connection_id,
15566 pending.corr,
15567 RouteBindRelayOutcome::Accepted,
15568 )
15569 .unwrap();
15570 client_rx.try_recv().unwrap();
15571 for module_id in ["beta", "alpha"] {
15572 let replies = handler
15573 .handle_control_frame(
15574 &module_ctx,
15575 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
15576 )
15577 .await
15578 .unwrap();
15579 assert_eq!(replies.len(), 1, "second HELLO must be refused");
15580 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
15581 }
15582 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
15583 assert!(registry.get_module("beta").unwrap().is_none());
15584 assert!(matches!(
15585 forwarding
15586 .lookup_data_route(
15587 client_ctx.connection_id,
15588 pending.client_channel,
15589 pending.client_epoch,
15590 )
15591 .unwrap(),
15592 DataRoute::Client(DataRouteState::Bound(_))
15593 ));
15594 assert!(handler
15595 .hello_launch_nonces
15596 .lock()
15597 .unwrap()
15598 .presented(module_ctx.connection_id, Some("alpha-nonce")));
15599 assert!(module_rx.try_recv().is_err());
15600 }
15601
15602 #[test]
15603 fn reserved_module_hello_requires_matching_launch_nonce() {
15604 let registry = Arc::new(Registry::default());
15605 let supervisor = SupervisorHandle::new();
15606 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
15609 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
15610
15611 let no_nonce = handler
15613 .handle_control(
15614 ConnectionId::new(1),
15615 hello_frame("vault", PROTOCOL_VERSION, 1),
15616 )
15617 .unwrap();
15618 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
15619 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
15620 assert!(registry.get_module("vault").unwrap().is_none());
15621
15622 let wrong = handler
15624 .handle_control(
15625 ConnectionId::new(2),
15626 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
15627 )
15628 .unwrap();
15629 assert_eq!(wrong[0].header.ty, FrameType::Error);
15630 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
15631 assert!(registry.get_module("vault").unwrap().is_none());
15632
15633 let ok = handler
15635 .handle_control(
15636 ConnectionId::new(3),
15637 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
15638 )
15639 .unwrap();
15640 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
15641 assert!(registry.get_module("vault").unwrap().is_some());
15642 }
15643
15644 #[test]
15645 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
15646 let registry = Arc::new(Registry::default());
15647 let supervisor = SupervisorHandle::new();
15648 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
15649 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
15650 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
15651
15652 let squat = handler
15653 .handle_control(
15654 ConnectionId::new(1),
15655 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
15656 )
15657 .unwrap();
15658 assert_eq!(squat[0].header.ty, FrameType::Error);
15659 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
15660 assert!(parse_error(&squat[0])["message"]
15661 .as_str()
15662 .unwrap()
15663 .contains("fed:"));
15664
15665 let accepted_peer = handler
15666 .handle_control(
15667 ConnectionId::new(2),
15668 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
15669 )
15670 .unwrap();
15671 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
15672
15673 let accepted_short = handler
15674 .handle_control(
15675 ConnectionId::new(3),
15676 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
15677 )
15678 .unwrap();
15679 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
15680
15681 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
15682 let response = handler
15683 .handle_control(
15684 ConnectionId::new(conn),
15685 hello_frame(module_id, PROTOCOL_VERSION, conn),
15686 )
15687 .unwrap();
15688 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
15689 }
15690 }
15691
15692 #[test]
15693 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
15694 let registry = Arc::new(Registry::default());
15695 let supervisor = SupervisorHandle::new();
15696 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
15697 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
15698 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
15699 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
15700
15701 let owner_nonce = handler
15702 .handle_control(
15703 ConnectionId::new(1),
15704 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
15705 )
15706 .unwrap();
15707 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
15708 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
15709 assert!(registry.get_module("fed:special").unwrap().is_none());
15710
15711 let exact_nonce = handler
15712 .handle_control(
15713 ConnectionId::new(2),
15714 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
15715 )
15716 .unwrap();
15717 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
15718 assert!(registry.get_module("fed:special").unwrap().is_some());
15719 }
15720
15721 #[test]
15722 fn non_reserved_module_ignores_launch_nonce() {
15723 let registry = Arc::new(Registry::default());
15724 let handler = ControlHandler::new(Arc::clone(®istry));
15727 let no_nonce = handler
15728 .handle_control(
15729 ConnectionId::new(1),
15730 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
15731 )
15732 .unwrap();
15733 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
15734 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
15735
15736 let echoed_nonce = handler
15737 .handle_control(
15738 ConnectionId::new(2),
15739 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
15740 )
15741 .unwrap();
15742 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
15743 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
15744 }
15745
15746 #[test]
15747 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
15748 let handler = ControlHandler::default();
15749 let conn = ConnectionId::new(1);
15750 let malformed = Frame::build(
15751 FrameType::Hello,
15752 control_flags(),
15753 0,
15754 0,
15755 3,
15756 b"{not json".to_vec(),
15757 )
15758 .unwrap();
15759
15760 let error = handler.handle_control(conn, malformed).unwrap();
15761 assert_eq!(error[0].header.ty, FrameType::Error);
15762 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
15763
15764 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
15765 let pong = handler.handle_control(conn, ping).unwrap();
15766 assert_eq!(pong[0].header.ty, FrameType::Pong);
15767 assert_eq!(pong[0].header.corr, 4);
15768 }
15769
15770 #[test]
15771 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
15772 let registry = Arc::new(Registry::default());
15773 let handler = ControlHandler::new(Arc::clone(®istry));
15774
15775 handler
15776 .handle_control(
15777 ConnectionId::new(1),
15778 hello_frame("aft", PROTOCOL_VERSION, 1),
15779 )
15780 .unwrap();
15781 let duplicate = handler
15782 .handle_control(
15783 ConnectionId::new(2),
15784 hello_frame("aft", PROTOCOL_VERSION, 2),
15785 )
15786 .unwrap();
15787
15788 assert_eq!(duplicate[0].header.ty, FrameType::Error);
15789 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
15790 let registration = registry.get_module("aft").unwrap().unwrap();
15791 assert_eq!(registration.connection_id, ConnectionId::new(1));
15792 }
15793
15794 #[test]
15795 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
15796 let registry = Arc::new(Registry::default());
15797 let forwarding = Arc::new(ForwardingTable::default());
15798 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
15799 let handler =
15800 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15801 .with_process_liveness(process_liveness);
15802 let (ctx, route_channel, route_epoch) =
15803 bind_liveness_route(®istry, &forwarding, "aft-dead");
15804 let responses = handler
15805 .handle_route_poll(
15806 &ctx,
15807 route_poll_frame(41, PollKind::Liveness, route_channel),
15808 route_channel,
15809 route_epoch,
15810 PollKind::Liveness,
15811 )
15812 .unwrap();
15813
15814 assert_eq!(responses.len(), 1);
15815 assert_eq!(responses[0].header.ty, FrameType::Response);
15816 assert_route_poll_liveness(&responses[0], false);
15817 }
15818
15819 #[test]
15820 fn liveness_poll_without_process_source_uses_bound_route() {
15821 let registry = Arc::new(Registry::default());
15822 let forwarding = Arc::new(ForwardingTable::default());
15823 let handler =
15824 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15825 let (ctx, route_channel, route_epoch) =
15826 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
15827 let responses = handler
15828 .handle_route_poll(
15829 &ctx,
15830 route_poll_frame(42, PollKind::Liveness, route_channel),
15831 route_channel,
15832 route_epoch,
15833 PollKind::Liveness,
15834 )
15835 .unwrap();
15836
15837 assert_route_poll_liveness(&responses[0], true);
15838 }
15839
15840 #[test]
15841 fn liveness_poll_untracked_process_source_uses_bound_route() {
15842 let registry = Arc::new(Registry::default());
15843 let forwarding = Arc::new(ForwardingTable::default());
15844 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
15845 let handler =
15846 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15847 .with_process_liveness(process_liveness);
15848 let (ctx, route_channel, route_epoch) =
15849 bind_liveness_route(®istry, &forwarding, "aft-untracked");
15850 let responses = handler
15851 .handle_route_poll(
15852 &ctx,
15853 route_poll_frame(43, PollKind::Liveness, route_channel),
15854 route_channel,
15855 route_epoch,
15856 PollKind::Liveness,
15857 )
15858 .unwrap();
15859
15860 assert_route_poll_liveness(&responses[0], true);
15861 }
15862
15863 #[tokio::test]
15864 async fn unknown_op_returns_unknown_control_op() {
15865 let handler = ControlHandler::default();
15866 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
15867 let request = Frame::build(
15868 FrameType::Request,
15869 control_flags(),
15870 0,
15871 0,
15872 55,
15873 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
15874 )
15875 .unwrap();
15876
15877 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
15878
15879 assert_eq!(response.len(), 1);
15880 assert_eq!(response[0].header.ty, FrameType::Error);
15881 assert_eq!(response[0].header.corr, 55);
15882 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
15883 }
15884
15885 #[tokio::test]
15886 async fn supervisor_provenance_rejects_unknown_exact_module() {
15887 let handler = ControlHandler::default();
15888 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
15889 let request = Frame::build(
15890 FrameType::Request,
15891 control_flags(),
15892 0,
15893 0,
15894 57,
15895 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
15896 )
15897 .unwrap();
15898
15899 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
15900
15901 assert_eq!(response.len(), 1);
15902 assert_eq!(response[0].header.ty, FrameType::Error);
15903 assert_eq!(response[0].header.corr, 57);
15904 let error = parse_error(&response[0]);
15905 assert_eq!(error["code"], "unknown_module");
15906 assert_eq!(error["message"], "module_id 'missing' is not supervised");
15907 }
15908
15909 #[test]
15910 fn provenance_probe_override_keeps_handler_tests_deterministic() {
15911 let expected = subc_control::RunningImageAgreement::Unavailable {
15912 reason: subc_control::RunningImageUnavailableReason::HashFailed,
15913 };
15914 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
15915 assert_eq!(handler.provenance_probe_override, Some(expected));
15916 }
15917
15918 #[test]
15919 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
15920 let verdict = reload_verdict(
15921 std::path::Path::new("/bin/new"),
15922 Some(std::path::Path::new("/bin/old")),
15923 subc_control::RunningImageAgreement::Unavailable {
15924 reason: subc_control::RunningImageUnavailableReason::HashFailed,
15925 },
15926 );
15927 assert!(matches!(
15928 verdict.path,
15929 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
15930 if configured == std::path::Path::new("/bin/new")
15931 && spawned_from == std::path::Path::new("/bin/old")
15932 ));
15933 }
15934
15935 #[test]
15936 fn launch_spec_without_running_digest_is_unavailable_not_match() {
15937 use subc_control::{
15938 LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
15939 };
15940 let configured = crate::supervise::ModuleSpec {
15941 carry_containment: false,
15942 module_id: "legacy".into(),
15943 program: "/module".into(),
15944 args: Vec::new(),
15945 env: Vec::new(),
15946 reserved: false,
15947 reserved_prefixes: Vec::new(),
15948 protocol: subc_control::ModuleProtocol::Subc,
15949 overlap: Default::default(),
15950 };
15951 let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
15952 assert_eq!(
15953 launch_spec_agreement(&configured, false, Some(&digest)),
15954 Agreement::Unavailable {
15955 reason: Reason::NotRunning
15956 }
15957 );
15958 assert_eq!(
15959 launch_spec_agreement(&configured, true, None),
15960 Agreement::Unavailable {
15961 reason: Reason::SpawnedSpecUnavailable
15962 }
15963 );
15964 assert_eq!(
15965 launch_spec_agreement(&configured, true, Some(&digest)),
15966 Agreement::Match
15967 );
15968 }
15969
15970 #[cfg(windows)]
15971 #[test]
15972 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
15973 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
15974 let configured = dir.join("module.exe");
15975 std::fs::write(&configured, b"executable path fixture").unwrap();
15976 let spawned = std::fs::canonicalize(&configured).unwrap();
15977 assert_ne!(
15978 configured, spawned,
15979 "fixture must exercise different path spellings"
15980 );
15981 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
15982 let lower_case = configured.to_string_lossy().to_lowercase();
15983 for spelling in [
15984 &configured,
15985 Path::new(&forward_slashes),
15986 Path::new(&lower_case),
15987 ] {
15988 let verdict = reload_verdict(
15989 spelling,
15990 Some(&spawned),
15991 subc_control::RunningImageAgreement::Unavailable {
15992 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
15993 },
15994 );
15995 assert_eq!(
15996 verdict.path,
15997 ReloadPathAgreement::Match,
15998 "configured {spelling:?}, spawned {spawned:?}"
15999 );
16000 }
16001 }
16002
16003 #[cfg(windows)]
16004 #[test]
16005 fn windows_reload_path_keeps_different_existing_program_pending() {
16006 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
16007 let old = dir.join("old.exe");
16008 let new = dir.join("new.exe");
16009 std::fs::write(&old, b"same executable content").unwrap();
16010 std::fs::copy(&old, &new).unwrap();
16011 let spawned = std::fs::canonicalize(&old).unwrap();
16012 let image = subc_control::RunningImageAgreement::Unavailable {
16013 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
16014 };
16015 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
16016 assert!(
16017 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
16018 if configured == new && spawned_from == spawned)
16019 );
16020 let missing = dir.join("missing.exe");
16021 assert!(matches!(
16022 reload_verdict(&missing, Some(&spawned), image).path,
16023 ReloadPathAgreement::Mismatch { .. }
16024 ));
16025 }
16026
16027 #[test]
16028 fn reload_verdict_detects_replaced_image_at_same_path() {
16029 let image = subc_control::RunningImageAgreement::Mismatch {
16030 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
16031 digest: "old".into(),
16032 },
16033 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
16034 digest: "new".into(),
16035 },
16036 };
16037 let verdict = reload_verdict(
16038 std::path::Path::new("/bin/same"),
16039 Some(std::path::Path::new("/bin/same")),
16040 image.clone(),
16041 );
16042 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
16043 assert_eq!(verdict.image, image);
16044 }
16045
16046 #[test]
16047 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
16048 let image = subc_control::RunningImageAgreement::Unavailable {
16049 reason: subc_control::RunningImageUnavailableReason::NotRunning,
16050 };
16051 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
16052 assert_eq!(
16053 verdict.path,
16054 subc_control::ReloadPathAgreement::Unavailable {
16055 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
16056 }
16057 );
16058 assert_eq!(verdict.image, image);
16059
16060 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
16061 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
16062 };
16063 let verdict = reload_verdict(
16064 std::path::Path::new("/bin/same"),
16065 Some(std::path::Path::new("/bin/same")),
16066 unconfirmed.clone(),
16067 );
16068 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
16069 assert_eq!(verdict.image, unconfirmed);
16070 }
16071
16072 #[test]
16073 fn reload_verdict_preserves_each_image_unavailability_reason() {
16074 use subc_control::RunningImageUnavailableReason as Reason;
16075
16076 for reason in [
16077 Reason::NotRunning,
16078 Reason::UnsupportedPlatform,
16079 Reason::RunningExecutableUnreadable,
16080 Reason::SpawnedPathUnreadable,
16081 Reason::HashFailed,
16082 Reason::ProcessIdentityUnconfirmed,
16083 Reason::Unknown("future_probe_reason".to_string()),
16084 ] {
16085 let image = subc_control::RunningImageAgreement::Unavailable {
16086 reason: reason.clone(),
16087 };
16088 let verdict = reload_verdict(
16089 std::path::Path::new("/bin/same"),
16090 Some(std::path::Path::new("/bin/same")),
16091 image.clone(),
16092 );
16093 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
16094 assert_eq!(verdict.image, image, "{reason:?}");
16095 }
16096 }
16097
16098 #[tokio::test]
16099 async fn malformed_control_bodies_return_invalid_control_body() {
16100 let handler = ControlHandler::default();
16101 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
16102
16103 for (corr, body) in [
16104 (56, br#"{"route_channel":1}"#.as_slice()),
16105 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
16106 (
16107 58,
16108 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
16109 ),
16110 ] {
16111 let request = Frame::build(
16112 FrameType::Request,
16113 control_flags(),
16114 0,
16115 0,
16116 corr,
16117 body.to_vec(),
16118 )
16119 .unwrap();
16120 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
16121
16122 assert_eq!(response.len(), 1);
16123 assert_eq!(response[0].header.ty, FrameType::Error);
16124 assert_eq!(response[0].header.corr, corr);
16125 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
16126 }
16127 }
16128
16129 #[tokio::test]
16130 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
16131 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
16132 let registry = Arc::new(Registry::default());
16133 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
16134 let router = Router::with_control_handler(Arc::clone(&control));
16135 let connection = router.begin_connection();
16136 let (ctx, mut rx) = route_ctx(connection.id());
16137
16138 router
16139 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
16140 .await
16141 .unwrap();
16142 let response = rx.recv().await.unwrap();
16143 let ack = parse_ack(&response);
16144 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
16145 let channel = 1;
16146
16147 let goodbye =
16148 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
16149 router.route_for_connection(&ctx, goodbye).await.unwrap();
16150 assert!(rx.try_recv().is_err());
16151 assert!(registry.get_module("aft").unwrap().is_none());
16152
16153 router
16154 .route_for_connection(&ctx, channel_request(channel, 13))
16155 .await
16156 .unwrap();
16157 let error_frame = rx.recv().await.unwrap();
16158 assert_eq!(error_frame.header.ty, FrameType::Error);
16159 assert_eq!(error_frame.header.channel, channel);
16160 let captured = crate::router::test_log::captured_logs(&logs);
16161 assert_eq!(
16162 captured
16163 .lines()
16164 .filter(|line| {
16165 line.contains("module_id=aft")
16166 && line.contains("reason=explicit_goodbye")
16167 && line.contains("module registration ended")
16168 })
16169 .count(),
16170 1,
16171 "unexpected GOODBYE registry log: {captured}"
16172 );
16173 }
16174
16175 #[tokio::test]
16176 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
16177 let registry = Arc::new(Registry::default());
16178 let handler = ControlHandler::new(registry).with_capability_config(
16179 [("prov".to_string(), true), ("cons".to_string(), true)],
16180 BTreeMap::new(),
16181 );
16182 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
16183 register_capability_manifest(
16184 &handler,
16185 &provider_ctx,
16186 &mut provider_rx,
16187 capability_manifest("prov", &["thing/v1"], &[]),
16188 1,
16189 )
16190 .await;
16191 let mut consumer = capability_manifest("cons", &[], &[]);
16192 consumer.capabilities.as_mut().unwrap().requires.push(
16193 subc_protocol::manifest::CapabilityRequirement {
16194 capability: "thing/v1".to_string(),
16195 need: subc_protocol::manifest::CapabilityNeed::Required,
16196 },
16197 );
16198 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
16199 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
16200 assert_eq!(
16201 handler.capability_evaluator.verdict("cons", "thing/v1"),
16202 Some(CapabilityVerdict::Provided)
16203 );
16204 let (mut client_rx, _) = open_route_for_capability_test(
16205 &handler,
16206 &provider_ctx,
16207 &mut provider_rx,
16208 703,
16209 3,
16210 "prov",
16211 None,
16212 )
16213 .await;
16214 let goodbye =
16215 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
16216 handler
16217 .handle_control_frame(&provider_ctx, goodbye)
16218 .await
16219 .unwrap();
16220 assert_eq!(
16221 handler.capability_evaluator.verdict("cons", "thing/v1"),
16222 Some(CapabilityVerdict::NeverProvided)
16223 );
16224 let closed = client_rx
16225 .try_recv()
16226 .expect("GOODBYE pushes route.closed before route GOODBYE");
16227 assert!(
16228 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
16229 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
16230 );
16231 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
16232 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
16233 }
16234
16235 struct RouteClosedFixture {
16236 handler: ControlHandler,
16237 #[cfg(unix)]
16238 supervisor: Supervisor,
16239 module_ctx: RouteCtx,
16240 #[cfg(unix)]
16241 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16242 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16243 route: Arc<crate::forwarding::RouteBinding>,
16244 }
16245
16246 async fn route_closed_fixture() -> RouteClosedFixture {
16247 let registry = Arc::new(Registry::default());
16248 let handler = ControlHandler::new(Arc::clone(®istry));
16249 #[cfg(unix)]
16250 let supervisor = Supervisor::new(registry, RestartPolicy::default())
16251 .with_forwarding(Arc::clone(&handler.forwarding));
16252 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
16253 register_capability_manifest(
16254 &handler,
16255 &module_ctx,
16256 &mut module_rx,
16257 capability_manifest("prov", &[], &[]),
16258 1,
16259 )
16260 .await;
16261 let (client_rx, bind) = open_route_for_capability_test(
16262 &handler,
16263 &module_ctx,
16264 &mut module_rx,
16265 702,
16266 2,
16267 "prov",
16268 None,
16269 )
16270 .await;
16271 let ModuleControlRequest::RouteBind {
16272 route_channel: channel,
16273 epoch: route_epoch,
16274 ..
16275 } = bind
16276 else {
16277 panic!("expected route.bind");
16278 };
16279 let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
16280 handler
16281 .forwarding
16282 .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
16283 .unwrap()
16284 else {
16285 panic!("the acknowledged route must be bound");
16286 };
16287 RouteClosedFixture {
16288 handler,
16289 #[cfg(unix)]
16290 supervisor,
16291 module_ctx,
16292 #[cfg(unix)]
16293 module_rx,
16294 client_rx,
16295 route,
16296 }
16297 }
16298
16299 impl RouteClosedFixture {
16300 #[cfg(unix)]
16301 async fn cut(&mut self) {
16302 let notices = async {
16303 let mut draining = self.module_rx.recv().await.unwrap();
16304 assert!(matches!(
16305 serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
16306 &draining.body
16307 )
16308 .unwrap(),
16309 subc_protocol::session::ModuleControlCommand::Draining {
16310 reason: RouteCloseReason::Disable,
16311 ..
16312 }
16313 ));
16314 draining.flushed.take().unwrap().send(()).unwrap();
16315 let mut closing = self.client_rx.recv().await.unwrap();
16316 assert!(matches!(
16317 serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
16318 ClientControlPush::RouteClosing {
16319 reason: RouteCloseReason::Restart,
16320 ..
16321 }
16322 ));
16323 closing.flushed.take().unwrap().send(()).unwrap();
16324 };
16325 let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
16326 result.unwrap();
16327 }
16328
16329 fn disconnect(&mut self) -> ClientControlPush {
16330 self.handler
16331 .cleanup_connection(self.module_ctx.connection_id)
16332 .unwrap();
16333 let closed = self.client_rx.try_recv().unwrap();
16334 assert_eq!(closed.header.ty, FrameType::Push);
16335 let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
16336 assert!(matches!(
16337 &push,
16338 ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
16339 if channels == &[self.route.client_channel]
16340 ));
16341 assert_eq!(
16342 self.client_rx.try_recv().unwrap().header.ty,
16343 FrameType::Goodbye
16344 );
16345 assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
16346 push
16347 }
16348 }
16349
16350 #[cfg(unix)]
16351 #[tokio::test(start_paused = true)]
16352 async fn daemon_cut_quiet_routes_report_restart_and_drained() {
16353 let mut fixture = route_closed_fixture().await;
16354 fixture.cut().await;
16355 assert!(matches!(
16356 fixture.disconnect(),
16357 ClientControlPush::RouteClosed {
16358 reason: RouteCloseReason::Restart,
16359 drained: true,
16360 ..
16361 }
16362 ));
16363 }
16364
16365 #[cfg(unix)]
16366 #[tokio::test(start_paused = true)]
16367 async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
16368 let mut fixture = route_closed_fixture().await;
16369 fixture.route.flow.acquire_tagged(99, false).await.unwrap();
16370 let started = tokio::time::Instant::now();
16371 fixture.cut().await;
16372 assert!(started.elapsed() >= Duration::from_secs(2));
16373 assert_eq!(fixture.route.flow.in_flight(), 1);
16374 assert!(matches!(
16375 fixture.disconnect(),
16376 ClientControlPush::RouteClosed {
16377 reason: RouteCloseReason::Restart,
16378 drained: false,
16379 ..
16380 }
16381 ));
16382 }
16383
16384 #[tokio::test]
16385 async fn module_connection_crash_reports_crash_and_not_drained() {
16386 let mut fixture = route_closed_fixture().await;
16387 assert!(matches!(
16391 fixture.disconnect(),
16392 ClientControlPush::RouteClosed {
16393 reason: RouteCloseReason::Crash,
16394 drained: false,
16395 ..
16396 }
16397 ));
16398 }
16399
16400 #[tokio::test]
16401 async fn dropping_router_connection_releases_registration() {
16402 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
16403 let registry = Arc::new(Registry::default());
16404 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
16405 let router = Router::with_control_handler(Arc::clone(&control));
16406 let connection = router.begin_connection();
16407 let connection_id = connection.id();
16408 let (ctx, mut rx) = route_ctx(connection_id);
16409
16410 router
16411 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
16412 .await
16413 .unwrap();
16414 let response = rx.recv().await.unwrap();
16415 let ack = parse_ack(&response);
16416 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
16417 assert!(registry.get_module("aft").unwrap().is_some());
16418
16419 drop(connection);
16420
16421 assert!(registry.get_module("aft").unwrap().is_none());
16422 assert_eq!(registry.active_registration_count().unwrap(), 0);
16423
16424 control
16425 .cleanup_connection(ConnectionId::new(u64::MAX))
16426 .unwrap();
16427 let captured = crate::router::test_log::captured_logs(&logs);
16428 println!("captured registry lifecycle logs:\n{captured}");
16429 let events: Vec<_> = captured
16430 .lines()
16431 .filter(|line| {
16432 line.contains("module registered module_id=aft ")
16433 || line.contains("module registration ended")
16434 })
16435 .collect();
16436 assert_eq!(
16437 events.len(),
16438 2,
16439 "unexpected registry lifecycle logs: {captured}"
16440 );
16441 assert!(events[0].contains("module registered module_id=aft "));
16442 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
16443 assert!(events[1].contains(&format!(
16444 "module_id=aft connection_id={}",
16445 connection_id.get()
16446 )));
16447 assert!(events[1].contains("reason=connection_closed"));
16448 assert!(events[1].contains("module registration ended"));
16449 assert_eq!(
16450 captured
16451 .lines()
16452 .filter(|line| line.contains("module registered module_id=aft "))
16453 .count(),
16454 1,
16455 "legacy registration admission line must appear once: {captured}"
16456 );
16457 }
16458
16459 #[tokio::test]
16460 async fn hello_registration_keeps_legacy_module_registered_line_once() {
16461 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
16462 let registry = Arc::new(Registry::default());
16463 let control = ControlHandler::new(Arc::clone(®istry));
16464 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
16465 hello_via_sink(
16466 &control,
16467 &ctx,
16468 &mut rx,
16469 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
16470 )
16471 .await;
16472
16473 let captured = crate::router::test_log::captured_logs(&logs);
16474 let admissions: Vec<_> = captured
16475 .lines()
16476 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
16477 .collect();
16478 assert_eq!(
16479 admissions.len(),
16480 1,
16481 "legacy admission line must remain exactly once: {captured}"
16482 );
16483 assert!(admissions[0].contains("routable_provider=true"));
16484 assert!(admissions[0].contains("connection_id=777"));
16485 }
16486
16487 fn capability_manifest(
16488 module_id: &str,
16489 provides: &[&str],
16490 must_never_reach: &[&str],
16491 ) -> ModuleManifest {
16492 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
16493 manifest.capabilities = Some(CapabilityDeclarations {
16494 provides: provides
16495 .iter()
16496 .map(|capability| (*capability).to_string())
16497 .collect(),
16498 requires: Vec::new(),
16499 must_never_reach: must_never_reach
16500 .iter()
16501 .map(|capability| (*capability).to_string())
16502 .collect(),
16503 });
16504 manifest
16505 }
16506
16507 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
16508 Frame::build(
16509 FrameType::Hello,
16510 control_flags(),
16511 0,
16512 0,
16513 corr,
16514 serde_json::to_vec(&ModuleHelloBody {
16515 protocol_ver: manifest.protocol_ver,
16516 manifest,
16517 control_ops: None,
16518 launch_nonce: None,
16519 })
16520 .expect("capability test HELLO serializes"),
16521 )
16522 .expect("capability test HELLO frame builds")
16523 }
16524
16525 fn catalog_update_with_capabilities_frame(
16526 corr: u64,
16527 capabilities: CapabilityDeclarations,
16528 ) -> Frame {
16529 Frame::build(
16530 FrameType::Request,
16531 control_flags(),
16532 0,
16533 0,
16534 corr,
16535 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
16536 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
16537 capabilities: Some(capabilities),
16538 ready: None,
16539 })
16540 .expect("capability catalog.update serializes"),
16541 )
16542 .expect("capability catalog.update frame builds")
16543 }
16544
16545 async fn register_capability_manifest(
16546 handler: &ControlHandler,
16547 ctx: &RouteCtx,
16548 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
16549 manifest: ModuleManifest,
16550 corr: u64,
16551 ) {
16552 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
16553 }
16554
16555 async fn open_route_for_capability_test(
16556 handler: &ControlHandler,
16557 target_ctx: &RouteCtx,
16558 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
16559 client_connection_id: u64,
16560 corr: u64,
16561 target_module_id: &str,
16562 consumer_identity: Option<ConsumerIdentity>,
16563 ) -> (
16564 mpsc::Receiver<crate::router::OutboundFrame>,
16565 ModuleControlRequest,
16566 ) {
16567 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
16568 let route_handler = handler.clone();
16569 let target_module_id = target_module_id.to_string();
16570 let route_task = tokio::spawn(async move {
16571 route_handler
16572 .handle_control_frame(
16573 &client_ctx,
16574 route_open_frame_with_admission_facts(
16575 corr,
16576 &target_module_id,
16577 unique_project_root("admission-facts"),
16578 consumer_identity,
16579 None,
16580 ),
16581 )
16582 .await
16583 .expect("capability test route.open succeeds")
16584 });
16585 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
16586 .await
16587 .expect("capability test route.open must reach route.bind")
16588 .expect("target control receiver stays open");
16589 let bind_request: ModuleControlRequest =
16590 serde_json::from_slice(&bind.body).expect("route.bind decodes");
16591 handler
16592 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
16593 .await
16594 .expect("capability test route.bind ACK succeeds");
16595 assert!(route_task.await.expect("route.open task joins").is_empty());
16596 let opened = client_rx
16597 .recv()
16598 .await
16599 .expect("successful route.open publishes a response");
16600 assert!(matches!(
16601 serde_json::from_slice::<ClientControlResponse>(&opened.body),
16602 Ok(ClientControlResponse::RouteOpen { .. })
16603 ));
16604 (client_rx, bind_request)
16605 }
16606
16607 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) -> u16 {
16608 assert_eq!(frame.header.ty, FrameType::Push);
16609 assert_eq!(frame.header.channel, 0);
16610 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
16611 .expect("route.closed control push decodes");
16612 let ClientControlPush::RouteClosed { channels, .. } = &push else {
16613 panic!("expected route.closed");
16614 };
16615 assert_eq!(channels.len(), 1, "exactly one violating route closed");
16616 let closed_channel = channels[0];
16617 let channels = channels.clone();
16618 assert_eq!(
16619 push,
16620 ClientControlPush::RouteClosed {
16621 module_id: target_module_id.to_string(),
16622 channels,
16623 reason: RouteCloseReason::CapabilityDenied,
16624 drained: false,
16625 abandoned: 0,
16626 excluded_subscriptions: 0,
16627 terminal: Some(false),
16628 }
16629 );
16630 closed_channel
16631 }
16632
16633 #[tokio::test]
16634 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
16635 let registry = Arc::new(Registry::default());
16636 let forwarding = Arc::new(ForwardingTable::default());
16637 let supervisor = SupervisorHandle::new();
16638 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16639 let handler =
16640 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16641 .with_supervisor(supervisor);
16642 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
16643 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
16644 register_capability_manifest(
16645 &handler,
16646 &target_ctx,
16647 &mut target_rx,
16648 capability_manifest("target", &["credentials-provider/v1"], &[]),
16649 1,
16650 )
16651 .await;
16652 register_capability_manifest(
16653 &handler,
16654 &opener_ctx,
16655 &mut opener_rx,
16656 capability_manifest("opener", &[], &["credentials-provider/v1"]),
16657 2,
16658 )
16659 .await;
16660
16661 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
16662 let replies = handler
16663 .handle_control_frame(
16664 &client_ctx,
16665 route_open_frame_with_admission_facts(
16666 3,
16667 "target",
16668 unique_project_root("admission-facts"),
16669 Some(ConsumerIdentity {
16670 module_id: "opener".to_string(),
16671 launch_nonce: "opener-nonce".to_string(),
16672 }),
16673 None,
16674 ),
16675 )
16676 .await
16677 .expect("denied route.open returns a typed frame");
16678 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
16679 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
16680 assert!(
16681 target_rx.try_recv().is_err(),
16682 "forbidden route.open must not relay route.bind"
16683 );
16684 }
16685
16686 #[tokio::test]
16687 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
16688 let registry = Arc::new(Registry::default());
16689 let forwarding = Arc::new(ForwardingTable::default());
16690 let supervisor = SupervisorHandle::new();
16691 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16692 let handler =
16693 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16694 .with_supervisor(supervisor);
16695 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
16696 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
16697 register_capability_manifest(
16698 &handler,
16699 &target_ctx,
16700 &mut target_rx,
16701 capability_manifest("target", &["credentials-provider/v1"], &[]),
16702 1,
16703 )
16704 .await;
16705 register_capability_manifest(
16706 &handler,
16707 &old_opener_ctx,
16708 &mut old_opener_rx,
16709 capability_manifest("opener", &[], &[]),
16710 2,
16711 )
16712 .await;
16713 let (mut client_rx, _) = open_route_for_capability_test(
16714 &handler,
16715 &target_ctx,
16716 &mut target_rx,
16717 712,
16718 3,
16719 "target",
16720 Some(ConsumerIdentity {
16721 module_id: "opener".to_string(),
16722 launch_nonce: "opener-nonce".to_string(),
16723 }),
16724 )
16725 .await;
16726 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
16727
16728 handler
16729 .cleanup_connection(old_opener_ctx.connection_id)
16730 .expect("old opener registration cleans up");
16731 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
16732 register_capability_manifest(
16733 &handler,
16734 &new_opener_ctx,
16735 &mut new_opener_rx,
16736 capability_manifest("opener", &[], &["credentials-provider/v1"]),
16737 4,
16738 )
16739 .await;
16740
16741 let closed_channel = assert_capability_denied_push(
16742 client_rx
16743 .try_recv()
16744 .expect("HELLO deny addition must emit route.closed")
16745 .frame,
16746 "target",
16747 );
16748 let goodbye = client_rx
16749 .try_recv()
16750 .expect("HELLO deny addition must send the client a route GOODBYE")
16751 .frame;
16752 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16753 assert_eq!(goodbye.header.channel, closed_channel);
16754 assert_ne!(goodbye.header.channel, 0);
16755 assert_ne!(goodbye.header.epoch, 0);
16756 assert!(client_rx.try_recv().is_err());
16757 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
16758 assert!(matches!(
16759 target_rx.try_recv(),
16760 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
16761 ));
16762 }
16763
16764 #[tokio::test]
16765 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
16766 let registry = Arc::new(Registry::default());
16767 let forwarding = Arc::new(ForwardingTable::default());
16768 let supervisor = SupervisorHandle::new();
16769 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16770 let handler =
16771 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16772 .with_supervisor(supervisor);
16773 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
16774 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
16775 register_capability_manifest(
16776 &handler,
16777 &target_ctx,
16778 &mut target_rx,
16779 capability_manifest("target", &[], &[]),
16780 1,
16781 )
16782 .await;
16783 register_capability_manifest(
16784 &handler,
16785 &opener_ctx,
16786 &mut opener_rx,
16787 capability_manifest("opener", &[], &["credentials-provider/v1"]),
16788 2,
16789 )
16790 .await;
16791 let (mut client_rx, _) = open_route_for_capability_test(
16792 &handler,
16793 &target_ctx,
16794 &mut target_rx,
16795 722,
16796 3,
16797 "target",
16798 Some(ConsumerIdentity {
16799 module_id: "opener".to_string(),
16800 launch_nonce: "opener-nonce".to_string(),
16801 }),
16802 )
16803 .await;
16804 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
16805
16806 let replies = handler
16807 .handle_control_frame(
16808 &target_ctx,
16809 catalog_update_with_capabilities_frame(
16810 4,
16811 CapabilityDeclarations {
16812 provides: vec!["credentials-provider/v1".to_string()],
16813 requires: Vec::new(),
16814 must_never_reach: Vec::new(),
16815 },
16816 ),
16817 )
16818 .await
16819 .expect("claim catalog.update succeeds");
16820 assert!(matches!(
16821 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
16822 Ok(ModuleControlResponseToModule::CatalogUpdate {})
16823 ));
16824 let closed_channel = assert_capability_denied_push(
16825 client_rx
16826 .try_recv()
16827 .expect("claim addition must emit route.closed")
16828 .frame,
16829 "target",
16830 );
16831 let goodbye = client_rx
16832 .try_recv()
16833 .expect("claim addition must send the client a route GOODBYE")
16834 .frame;
16835 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16836 assert_eq!(goodbye.header.channel, closed_channel);
16837 assert_ne!(goodbye.header.channel, 0);
16838 assert_ne!(goodbye.header.epoch, 0);
16839 assert!(client_rx.try_recv().is_err());
16840 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
16841 assert!(matches!(
16842 target_rx.try_recv(),
16843 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
16844 ));
16845 }
16846
16847 #[tokio::test]
16848 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
16849 let registry = Arc::new(Registry::default());
16850 let forwarding = Arc::new(ForwardingTable::default());
16851 let supervisor = SupervisorHandle::new();
16852 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16853 let handler =
16854 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16855 .with_supervisor(supervisor);
16856 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
16857 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
16858 register_capability_manifest(
16859 &handler,
16860 &target_ctx,
16861 &mut target_rx,
16862 capability_manifest("target", &["credentials-provider/v1"], &[]),
16863 1,
16864 )
16865 .await;
16866 register_capability_manifest(
16867 &handler,
16868 &opener_ctx,
16869 &mut opener_rx,
16870 capability_manifest("opener", &[], &[]),
16871 2,
16872 )
16873 .await;
16874 let (mut client_rx, _) = open_route_for_capability_test(
16875 &handler,
16876 &target_ctx,
16877 &mut target_rx,
16878 732,
16879 3,
16880 "target",
16881 Some(ConsumerIdentity {
16882 module_id: "opener".to_string(),
16883 launch_nonce: "opener-nonce".to_string(),
16884 }),
16885 )
16886 .await;
16887 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
16888
16889 handler
16890 .handle_control_frame(
16891 &target_ctx,
16892 catalog_update_with_capabilities_frame(
16893 4,
16894 CapabilityDeclarations {
16895 provides: Vec::new(),
16896 requires: Vec::new(),
16897 must_never_reach: Vec::new(),
16898 },
16899 ),
16900 )
16901 .await
16902 .expect("claim removal catalog.update succeeds");
16903 assert_eq!(
16904 forwarding.active_binding_count().unwrap(),
16905 1,
16906 "removing an attested target claim must leave the route census unchanged"
16907 );
16908 assert!(
16909 client_rx.try_recv().is_err(),
16910 "claim removal must not emit route.closed or a client route GOODBYE"
16911 );
16912 assert!(
16913 target_rx.try_recv().is_err(),
16914 "claim removal must not send the target a route GOODBYE"
16915 );
16916 }
16917
16918 #[tokio::test]
16921 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
16922 let registry = Arc::new(Registry::default());
16923 let forwarding = Arc::new(ForwardingTable::default());
16924 let supervisor = SupervisorHandle::new();
16925 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16926 let handler =
16927 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16928 .with_supervisor(supervisor);
16929 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
16930 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
16931 register_capability_manifest(
16932 &handler,
16933 &target_ctx,
16934 &mut target_rx,
16935 capability_manifest("target", &["credentials-provider/v1"], &[]),
16936 1,
16937 )
16938 .await;
16939 register_capability_manifest(
16940 &handler,
16941 &opener_ctx,
16942 &mut opener_rx,
16943 capability_manifest("opener", &[], &["credentials-provider/v1"]),
16944 2,
16945 )
16946 .await;
16947
16948 let (_client_rx, bind) = open_route_for_capability_test(
16949 &handler,
16950 &target_ctx,
16951 &mut target_rx,
16952 742,
16953 3,
16954 "target",
16955 None,
16956 )
16957 .await;
16958 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
16959 panic!("direct scope-honesty route must bind");
16960 };
16961 assert_eq!(principal, Some(Principal::Direct));
16962 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
16963 }
16964
16965 #[tokio::test]
16968 async fn must_never_reach_self_route_is_capability_forbidden() {
16969 let registry = Arc::new(Registry::default());
16970 let forwarding = Arc::new(ForwardingTable::default());
16971 let supervisor = SupervisorHandle::new();
16972 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
16973 let handler =
16974 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16975 .with_supervisor(supervisor);
16976 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
16977 register_capability_manifest(
16978 &handler,
16979 &self_ctx,
16980 &mut self_rx,
16981 capability_manifest(
16982 "self-provider",
16983 &["credentials-provider/v1"],
16984 &["credentials-provider/v1"],
16985 ),
16986 1,
16987 )
16988 .await;
16989
16990 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
16991 let replies = handler
16992 .handle_control_frame(
16993 &client_ctx,
16994 route_open_frame_with_admission_facts(
16995 2,
16996 "self-provider",
16997 unique_project_root("admission-facts"),
16998 Some(ConsumerIdentity {
16999 module_id: "self-provider".to_string(),
17000 launch_nonce: "self-nonce".to_string(),
17001 }),
17002 None,
17003 ),
17004 )
17005 .await
17006 .expect("self-route refusal returns a typed frame");
17007 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
17008 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
17009 assert!(
17010 self_rx.try_recv().is_err(),
17011 "self denial must not relay route.bind"
17012 );
17013 }
17014
17015 #[test]
17016 fn unsupported_channel_zero_frame_returns_error() {
17017 let handler = ControlHandler::default();
17018 let request = Frame::build(
17019 FrameType::Request,
17020 control_flags(),
17021 0,
17022 0,
17023 21,
17024 b"opaque".to_vec(),
17025 )
17026 .unwrap();
17027
17028 let response = handler
17029 .handle_control(ConnectionId::new(1), request)
17030 .unwrap();
17031
17032 assert_eq!(response[0].header.ty, FrameType::Error);
17033 assert_eq!(
17034 parse_error(&response[0])["code"],
17035 "unsupported_control_frame"
17036 );
17037 }
17038
17039 mod swap {
17044 use super::*;
17045
17046 const INCUMBENT: ConnectionId = ConnectionId::new(30);
17047 const CANDIDATE: ConnectionId = ConnectionId::new(40);
17048
17049 struct Swap {
17050 registry: Arc<Registry>,
17051 forwarding: Arc<ForwardingTable>,
17052 handler: ControlHandler,
17053 incumbent_ctx: RouteCtx,
17054 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17055 candidate_ctx: RouteCtx,
17056 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17057 }
17058
17059 async fn swap_with_incumbent() -> Swap {
17060 let registry = Arc::new(Registry::default());
17061 let forwarding = Arc::new(ForwardingTable::default());
17062 let handler =
17063 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
17064 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
17065 hello_via_sink(
17066 &handler,
17067 &incumbent_ctx,
17068 &mut incumbent_rx,
17069 hello_frame("aft", PROTOCOL_VERSION, 7),
17070 )
17071 .await;
17072 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
17073 Swap {
17074 registry,
17075 forwarding,
17076 handler,
17077 incumbent_ctx,
17078 incumbent_rx,
17079 candidate_ctx,
17080 candidate_rx,
17081 }
17082 }
17083
17084 fn register_candidate(swap: &Swap, ready: Option<bool>) {
17085 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
17086 candidate_manifest.ready = ready;
17087 let registration = swap
17088 .registry
17089 .register_candidate_with_control_ops(
17090 candidate_manifest,
17091 PROTOCOL_VERSION,
17092 CANDIDATE,
17093 module_baseline_control_ops(),
17094 )
17095 .unwrap();
17096 swap.forwarding
17097 .register_candidate_module_connection(
17098 CANDIDATE,
17099 "aft".to_string(),
17100 PROTOCOL_VERSION,
17101 manifest_concurrency(®istration.manifest),
17102 swap.candidate_ctx.egress.clone(),
17103 )
17104 .unwrap();
17105 }
17106
17107 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
17108 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
17109 swap.registry.promote_candidate("aft").unwrap().unwrap();
17110 cutover.incumbent.unwrap()
17111 }
17112
17113 #[tokio::test]
17122 async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
17123 let mut swap = swap_with_incumbent().await;
17124 let hold = Arc::new(std::sync::Barrier::new(2));
17125 let handler = swap
17126 .handler
17127 .clone()
17128 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
17129 handler
17130 .supervisor
17131 .open_swap("aft", "candidate-nonce".to_string());
17132 let hello = {
17133 let handler = handler.clone();
17134 let candidate_ctx = swap.candidate_ctx.clone();
17135 std::thread::spawn(move || {
17136 handler
17137 .handle_hello(
17138 candidate_ctx.connection_id,
17139 Some(candidate_ctx.egress.clone()),
17140 hello_frame_with_nonce(
17141 "aft",
17142 PROTOCOL_VERSION,
17143 8,
17144 Some("candidate-nonce"),
17145 ),
17146 )
17147 .unwrap()
17148 })
17149 };
17150
17151 hold.wait();
17152 assert!(swap
17155 .registry
17156 .registration(crate::registry::RegistrationSlot::Candidate("aft"))
17157 .unwrap()
17158 .is_some());
17159 assert!(swap
17160 .registry
17161 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
17162 .unwrap()
17163 .is_none());
17164 assert!(swap
17165 .forwarding
17166 .module_endpoint_for_connection(CANDIDATE)
17167 .unwrap()
17168 .is_none());
17169
17170 hold.wait();
17171 assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
17172 assert_eq!(
17173 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
17174 FrameType::HelloAck
17175 );
17176 let published = swap
17177 .registry
17178 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
17179 .unwrap()
17180 .expect("candidate is published once its HELLO completes");
17181 assert_eq!(published.connection_id, CANDIDATE);
17182 assert!(swap
17183 .forwarding
17184 .module_endpoint_for_connection(CANDIDATE)
17185 .unwrap()
17186 .is_some());
17187 }
17188
17189 fn keyed_total(counters: &Value, key: &str) -> u64 {
17190 counters[key]
17191 .as_object()
17192 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
17193 .unwrap_or(0)
17194 }
17195
17196 #[tokio::test]
17197 async fn replacement_logs_old_end_and_new_admission_once() {
17198 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
17199 let mut swap = swap_with_incumbent().await;
17200 swap.handler
17201 .supervisor
17202 .open_swap("aft", "candidate-nonce".to_string());
17203 hello_via_sink(
17204 &swap.handler,
17205 &swap.candidate_ctx,
17206 &mut swap.candidate_rx,
17207 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
17208 )
17209 .await;
17210 cutover(&swap);
17211 swap.handler.cleanup_connection(INCUMBENT).unwrap();
17212
17213 let captured = crate::router::test_log::captured_logs(&logs);
17214 println!("captured replacement registry logs:\n{captured}");
17215 let new_admissions: Vec<_> = captured
17216 .lines()
17217 .filter(|line| {
17218 line.contains("connection_id=40")
17219 && line.contains("swap candidate registered; not routable until cutover")
17220 })
17221 .collect();
17222 assert_eq!(
17223 new_admissions.len(),
17224 1,
17225 "unexpected admission logs: {captured}"
17226 );
17227 assert!(new_admissions[0].contains("module_id=aft"));
17228 assert!(new_admissions[0].contains("connection_id=40"));
17229 assert!(
17230 new_admissions[0].contains("swap candidate registered; not routable until cutover")
17231 );
17232 assert!(new_admissions[0].contains("ready=true"));
17233
17234 let old_admissions: Vec<_> = captured
17235 .lines()
17236 .filter(|line| {
17237 line.contains("module registered module_id=aft ")
17238 && line.contains("connection_id=30")
17239 })
17240 .collect();
17241 assert_eq!(
17242 old_admissions.len(),
17243 1,
17244 "unexpected incumbent admission: {captured}"
17245 );
17246
17247 let old_ends: Vec<_> = captured
17248 .lines()
17249 .filter(|line| {
17250 line.contains("connection_id=30") && line.contains("module registration ended")
17251 })
17252 .collect();
17253 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
17254 assert!(old_ends[0].contains("module_id=aft"));
17255 assert!(old_ends[0].contains("connection_id=30"));
17256 assert!(old_ends[0].contains("reason=replaced"));
17257 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
17258 assert_eq!(
17259 captured
17260 .lines()
17261 .filter(|line| line.contains("module registration promoted"))
17262 .count(),
17263 1,
17264 "unexpected promotion logs: {captured}"
17265 );
17266 }
17267
17268 #[tokio::test]
17274 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
17275 let mut swap = swap_with_incumbent().await;
17276 let handler = swap.handler.clone();
17277
17278 let cotenant = ConnectionId::new(31);
17280 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
17281 let (cotenant_task, cotenant_bind) = relay_route_open(
17282 &handler,
17283 cotenant,
17284 &cotenant_ctx.egress,
17285 &mut swap.incumbent_rx,
17286 100,
17287 "aft",
17288 "swap-cotenant",
17289 )
17290 .await;
17291 handler
17292 .handle_control_frame(
17293 &swap.incumbent_ctx,
17294 route_bind_ack(cotenant_bind.header.corr),
17295 )
17296 .await
17297 .unwrap();
17298 assert!(cotenant_task.await.unwrap().is_empty());
17299 let (cotenant_channel, cotenant_epoch) =
17300 published_route(&cotenant_rx.recv().await.unwrap());
17301
17302 let caller = ConnectionId::new(32);
17304 let (caller_ctx, mut caller_rx) = route_ctx(caller);
17305 let (caller_task, caller_bind) = relay_route_open(
17306 &handler,
17307 caller,
17308 &caller_ctx.egress,
17309 &mut swap.incumbent_rx,
17310 101,
17311 "aft",
17312 "swap-caller",
17313 )
17314 .await;
17315 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
17316
17317 register_candidate(&swap, None);
17318 cutover(&swap);
17319
17320 let ack = handler
17322 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
17323 .await;
17324 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
17325 if module_loop_error.is_some() {
17326 handler.cleanup_connection(INCUMBENT).unwrap();
17329 }
17330
17331 assert!(
17333 cotenant_rx.try_recv().is_err(),
17334 "the co-tenant route on the incumbent was torn down by one late ack: \
17335 {module_loop_error:?}"
17336 );
17337 assert!(matches!(
17338 swap.forwarding
17339 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
17340 .unwrap(),
17341 DataRoute::Client(DataRouteState::Bound(_))
17342 ));
17343 assert_eq!(module_loop_error, None);
17344 assert!(swap
17345 .registry
17346 .get_module_by_connection(INCUMBENT)
17347 .unwrap()
17348 .is_some());
17349
17350 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
17352 .await
17353 .expect("the incumbent is told to drop the abandoned binding")
17354 .unwrap()
17355 .frame;
17356 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17357 assert_eq!(goodbye.header.channel, abandoned_channel);
17358 assert_eq!(goodbye.header.epoch, abandoned_epoch);
17359 assert!(swap.incumbent_rx.try_recv().is_err());
17360
17361 let response = caller_task.await.unwrap();
17363 assert_eq!(response.len(), 1);
17364 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
17365 assert!(caller_rx.try_recv().is_err());
17366
17367 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
17371 let counters = handler.counters().snapshot();
17372 assert_eq!(
17373 keyed_total(&counters, "route_open_accepted_by_principal"),
17374 1
17375 );
17376 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
17377 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
17378 }
17379
17380 #[tokio::test]
17384 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
17385 let mut swap = swap_with_incumbent().await;
17386 register_candidate(&swap, None);
17387 let incumbent = cutover(&swap);
17388 swap.forwarding
17389 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
17390 .unwrap()
17391 .expect("the incumbent is still registered");
17392
17393 let client = ConnectionId::new(33);
17394 let (client_ctx, mut client_rx) = route_ctx(client);
17395 let route_handler = swap.handler.clone();
17396 let open_ctx = RouteCtx {
17397 connection_id: client,
17398 egress: client_ctx.egress.clone(),
17399 };
17400 let mut route_task = tokio::spawn(async move {
17401 route_handler
17402 .handle_control_frame(
17403 &open_ctx,
17404 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
17405 )
17406 .await
17407 .unwrap()
17408 });
17409 let bind = tokio::select! {
17410 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
17411 response = &mut route_task => {
17412 let response = response.unwrap();
17413 panic!(
17414 "post-cutover route.open was refused instead of relayed to the candidate: {}",
17415 parse_error(&response[0])["code"]
17416 );
17417 }
17418 };
17419 swap.handler
17420 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
17421 .await
17422 .unwrap();
17423 assert!(route_task.await.unwrap().is_empty());
17424 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
17425 match swap
17426 .forwarding
17427 .lookup_data_route(client, channel, epoch)
17428 .unwrap()
17429 {
17430 DataRoute::Client(DataRouteState::Bound(route)) => {
17431 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
17432 }
17433 other => panic!("expected a bound route on the candidate, got {other:?}"),
17434 }
17435 assert!(swap.incumbent_rx.try_recv().is_err());
17436 }
17437
17438 #[tokio::test]
17443 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
17444 let swap = swap_with_incumbent().await;
17445 register_candidate(&swap, Some(false));
17446 let update = Frame::build(
17447 FrameType::Request,
17448 control_flags(),
17449 0,
17450 0,
17451 55,
17452 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
17453 provides: manifest("aft", PROTOCOL_VERSION).provides,
17454 capabilities: None,
17455 ready: Some(true),
17456 })
17457 .unwrap(),
17458 )
17459 .unwrap();
17460
17461 let replies = swap
17462 .handler
17463 .handle_control_frame(&swap.candidate_ctx, update)
17464 .await
17465 .unwrap();
17466
17467 assert_eq!(replies.len(), 1);
17468 assert_eq!(
17469 replies[0].header.ty,
17470 FrameType::Response,
17471 "candidate catalog.update was refused: {:?}",
17472 serde_json::from_slice::<Value>(&replies[0].body).ok()
17473 );
17474 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
17475 assert_eq!(
17476 swap.registry
17477 .get_module("aft")
17478 .unwrap()
17479 .unwrap()
17480 .connection_id,
17481 INCUMBENT
17482 );
17483 }
17484 }
17485
17486 mod swap_admission {
17490 use super::*;
17491
17492 const INCUMBENT_NONCE: &str = "incumbent-nonce";
17493 const CANDIDATE_NONCE: &str = "candidate-nonce";
17494
17495 fn handler_with_incumbent(
17496 module_id: &str,
17497 reserved: bool,
17498 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
17499 let registry = Arc::new(Registry::default());
17500 let supervisor = SupervisorHandle::new();
17501 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
17502 if reserved {
17503 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
17504 }
17505 let handler =
17506 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
17507 let incumbent = handler
17508 .handle_control(
17509 ConnectionId::new(1),
17510 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
17511 )
17512 .unwrap();
17513 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
17514 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
17515 (registry, supervisor, handler)
17516 }
17517
17518 #[test]
17524 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
17525 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
17526
17527 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
17528 let replies = handler
17529 .handle_control(
17530 ConnectionId::new(connection),
17531 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
17532 )
17533 .unwrap();
17534 assert_eq!(replies[0].header.ty, FrameType::Error);
17535 assert_eq!(
17536 parse_error(&replies[0])["code"],
17537 "swap_token_invalid",
17538 "nonce {nonce:?}"
17539 );
17540 }
17541 assert!(registry.get_candidate("aft").unwrap().is_none());
17542 assert_eq!(
17543 registry.get_module("aft").unwrap().unwrap().connection_id,
17544 ConnectionId::new(1)
17545 );
17546
17547 let admitted = handler
17550 .handle_control(
17551 ConnectionId::new(4),
17552 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
17553 )
17554 .unwrap();
17555 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
17556 assert_eq!(
17557 registry
17558 .get_candidate("aft")
17559 .unwrap()
17560 .unwrap()
17561 .connection_id,
17562 ConnectionId::new(4)
17563 );
17564 assert_eq!(
17565 registry.get_module("aft").unwrap().unwrap().connection_id,
17566 ConnectionId::new(1),
17567 "the candidate must not take the active slot"
17568 );
17569 let replayed = handler
17570 .handle_control(
17571 ConnectionId::new(5),
17572 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
17573 )
17574 .unwrap();
17575 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
17576
17577 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
17581 let squatter = handler
17582 .handle_control(
17583 ConnectionId::new(6),
17584 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
17585 )
17586 .unwrap();
17587 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
17588 assert!(
17589 registry.get_module("aft").unwrap().is_none(),
17590 "a squatter took the active slot of an id being swapped"
17591 );
17592 }
17593
17594 #[test]
17599 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
17600 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
17601
17602 let replies = handler
17603 .handle_control(
17604 ConnectionId::new(2),
17605 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
17606 )
17607 .unwrap();
17608
17609 assert_eq!(
17610 replies[0].header.ty,
17611 FrameType::HelloAck,
17612 "reserved candidate refused: {:?}",
17613 serde_json::from_slice::<Value>(&replies[0].body).ok()
17614 );
17615 assert_eq!(
17616 registry
17617 .get_candidate("vault")
17618 .unwrap()
17619 .unwrap()
17620 .connection_id,
17621 ConnectionId::new(2)
17622 );
17623 }
17624
17625 #[test]
17628 fn without_an_open_swap_the_ordinary_gates_decide() {
17629 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
17630 supervisor.close_swap("vault");
17631
17632 let candidate = handler
17633 .handle_control(
17634 ConnectionId::new(2),
17635 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
17636 )
17637 .unwrap();
17638 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
17639 let duplicate = handler
17640 .handle_control(
17641 ConnectionId::new(3),
17642 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
17643 )
17644 .unwrap();
17645 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
17646 assert!(registry.get_candidate("vault").unwrap().is_none());
17647 }
17648 }
17649
17650 mod scopes {
17654 mod idle_tests {
17655 use super::*;
17656 include!("scope_idle_tests.rs");
17657 }
17658 mod route_open_timings {
17659 use super::*;
17660 include!("route_open_timing_tests.rs");
17661 }
17662 mod route_open_refusal_log {
17663 use super::*;
17664 include!("route_open_refusal_log_tests.rs");
17665 }
17666 mod activity {
17667 include!("activity_scope_tests.rs");
17668 }
17669 mod delta {
17670 include!("scope_handler_delta_tests.rs");
17671 }
17672 mod call_modes {
17673 include!("scope_call_mode_tests.rs");
17674 }
17675 #[tokio::test]
17676 async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
17677 let rig = rig().await;
17678 let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
17679 "s".into(),
17680 1,
17681 "tool".into(),
17682 ))
17683 .unwrap();
17684 assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
17685 let (direct, _rx) = wide_ctx(999);
17686 for (ctx, expected) in [
17687 (&rig.owner, FrameType::Response),
17688 (&direct, FrameType::Error),
17689 ] {
17690 let frame =
17691 Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
17692 .unwrap();
17693 let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
17694 assert_eq!(replies[0].header.ty, expected);
17695 if expected == FrameType::Response {
17696 let reply: subc_control::ScopeServedByReply =
17697 serde_json::from_slice(&replies[0].body).unwrap();
17698 assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
17699 } else {
17700 assert_eq!(parse_error(&replies[0])["code"], "not_registered");
17701 }
17702 }
17703 let reply = call(
17704 &rig.handler,
17705 &rig.owner,
17706 &ModuleControlRequestFromModule::ScopeDescribe {
17707 owner: Principal::Reserved {
17708 module_id: OWNER.into(),
17709 },
17710 scope_ref: "s".into(),
17711 },
17712 )
17713 .await;
17714 assert!(matches!(
17715 serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
17716 ModuleControlResponseToModule::ScopeDescribe { .. }
17717 ));
17718 }
17719
17720 use subc_protocol::scope::{
17721 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
17722 ScopeStatus,
17723 };
17724
17725 use super::*;
17726
17727 const OWNER: &str = "prefrontal-core";
17728
17729 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
17730 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
17731 }
17732
17733 async fn call(
17734 handler: &ControlHandler,
17735 ctx: &RouteCtx,
17736 request: &ModuleControlRequestFromModule,
17737 ) -> Frame {
17738 let body = serde_json::to_vec(request).unwrap();
17739 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
17740 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
17741 assert_eq!(replies.len(), 1, "{replies:?}");
17742 replies.pop().unwrap()
17743 }
17744
17745 async fn sync(
17746 handler: &ControlHandler,
17747 ctx: &RouteCtx,
17748 generation: u64,
17749 scopes: Vec<ScopeRecord>,
17750 ) -> Result<ModuleControlResponseToModule, String> {
17751 let reply = call(
17752 handler,
17753 ctx,
17754 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
17755 )
17756 .await;
17757 match reply.header.ty {
17758 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
17759 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
17760 }
17761 }
17762
17763 async fn describe(
17764 handler: &ControlHandler,
17765 ctx: &RouteCtx,
17766 owner: &str,
17767 scope_ref: &str,
17768 ) -> ModuleControlResponseToModule {
17769 let reply = call(
17770 handler,
17771 ctx,
17772 &ModuleControlRequestFromModule::ScopeDescribe {
17773 owner: Principal::Reserved {
17774 module_id: owner.to_string(),
17775 },
17776 scope_ref: scope_ref.to_string(),
17777 },
17778 )
17779 .await;
17780 assert_eq!(
17781 reply.header.ty,
17782 FrameType::Response,
17783 "{:?}",
17784 parse_error(&reply)
17785 );
17786 serde_json::from_slice(&reply.body).unwrap()
17787 }
17788
17789 #[tokio::test]
17790 async fn scope_describe_never_echoes_bind_proof_or_hash() {
17791 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
17792 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17793 let mut rig = rig().await;
17794 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
17795 .await;
17796 let _bound = rig
17797 .bound(
17798 Some(AFT),
17799 PLEXUS,
17800 Some(
17801 rig.selector("s", Some(1))
17802 .with_bind_proof(Some(PROOF.into())),
17803 ),
17804 )
17805 .await;
17806 let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
17807 let output = serde_json::to_string(&reply).unwrap();
17808 assert!(output.contains("\"status\":\"live\""));
17809 assert!(!output.contains(PROOF));
17810 assert!(!output.contains(HASH));
17811 }
17812
17813 async fn module(
17815 handler: &ControlHandler,
17816 connection: u64,
17817 module_id: &str,
17818 nonce: Option<&str>,
17819 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
17820 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
17821 hello_via_sink(
17822 handler,
17823 &ctx,
17824 &mut rx,
17825 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
17826 )
17827 .await;
17828 (ctx, rx)
17829 }
17830
17831 #[tokio::test]
17834 async fn a_client_connection_cannot_sync_or_describe() {
17835 let handler = ControlHandler::new(Arc::new(Registry::default()));
17836 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
17837 for request in [
17838 ModuleControlRequestFromModule::ScopeSync {
17839 generation: 1,
17840 scopes: vec![head("s", 1)],
17841 },
17842 ModuleControlRequestFromModule::ScopeDescribe {
17843 owner: Principal::Direct,
17844 scope_ref: "s".to_string(),
17845 },
17846 ] {
17847 let reply = call(&handler, &ctx, &request).await;
17848 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
17849 }
17850 assert!(
17851 !handler
17852 .scopes
17853 .read()
17854 .unwrap()
17855 .describe(
17856 &Principal::Reserved {
17857 module_id: OWNER.to_string()
17858 },
17859 "s"
17860 )
17861 .owner_synced
17862 );
17863 }
17864
17865 #[tokio::test]
17868 async fn a_module_without_a_supervised_launch_cannot_sync() {
17869 let handler = ControlHandler::new(Arc::new(Registry::default()));
17870 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
17871 assert_eq!(
17872 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
17873 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
17874 );
17875 }
17876
17877 #[tokio::test]
17878 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
17879 let supervisor = SupervisorHandle::new();
17880 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
17881 let handler = ControlHandler::new(Arc::new(Registry::default()))
17882 .with_supervisor(supervisor.clone());
17883 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
17884 sync(&handler, &incumbent, 1, vec![head("s", 1)])
17885 .await
17886 .expect("the current launch syncs");
17887
17888 supervisor.open_swap(OWNER, "n2".to_string());
17891 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
17892 assert_eq!(
17893 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
17894 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
17895 );
17896 sync(&handler, &incumbent, 2, vec![head("s", 1)])
17897 .await
17898 .expect("the serving owner syncs during the swap");
17899
17900 supervisor.close_swap(OWNER);
17903 assert_eq!(
17904 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
17905 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
17906 );
17907 sync(&handler, &incumbent, 3, vec![head("s", 1)])
17908 .await
17909 .expect("the serving owner syncs after the rollback");
17910 handler.cleanup_connection(candidate.connection_id).unwrap();
17911
17912 supervisor.open_swap(OWNER, "n3".to_string());
17917 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
17918 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
17919 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
17920 .await
17921 .expect("the promoted launch takes authority");
17922 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
17923 panic!("unexpected reply {reply:?}");
17924 };
17925 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
17926 assert_eq!(
17927 sync(&handler, &incumbent, 4, Vec::new()).await,
17928 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
17929 );
17930 }
17931
17932 #[tokio::test]
17935 async fn closing_the_authority_connection_frees_sync_authority() {
17936 let supervisor = SupervisorHandle::new();
17937 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
17938 let handler = ControlHandler::new(Arc::new(Registry::default()))
17939 .with_supervisor(supervisor.clone());
17940 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
17941 sync(&handler, &first, 10, vec![head("s", 1)])
17942 .await
17943 .unwrap();
17944 handler.cleanup_connection(first.connection_id).unwrap();
17945
17946 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
17947 sync(&handler, &second, 1, vec![head("s", 1)])
17948 .await
17949 .expect("the next connection takes the released authority");
17950 }
17951
17952 #[tokio::test]
17953 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
17954 let supervisor = SupervisorHandle::new();
17955 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
17956 let handler =
17957 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
17958 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
17959 sync(&handler, &first, 10, vec![head("s", 1)])
17960 .await
17961 .unwrap();
17962 handler
17963 .handle_control_frame(
17964 &first,
17965 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
17966 )
17967 .await
17968 .unwrap();
17969 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
17970 sync(&handler, &second, 1, vec![head("s", 1)])
17971 .await
17972 .expect("GOODBYE releases authority even if the old socket remains open");
17973 }
17974
17975 #[tokio::test]
17976 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
17977 let registry = Arc::new(Registry::default());
17978 let supervisor_handle = SupervisorHandle::new();
17979 let supervisor =
17980 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
17981 .with_handle(supervisor_handle.clone())
17982 .with_daemon_incarnation("incarnation-7".to_string());
17983 supervisor
17986 .supervise_configured(
17987 ModuleSpec {
17988 carry_containment: false,
17989 module_id: OWNER.to_string(),
17990 program: PathBuf::from("/nonexistent/prefrontal-core"),
17991 args: Vec::new(),
17992 env: Vec::new(),
17993 reserved: false,
17994 reserved_prefixes: Vec::new(),
17995 protocol: ModuleProtocol::Subc,
17996 overlap: Default::default(),
17997 },
17998 false,
17999 )
18000 .unwrap();
18001 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
18002 let handler =
18003 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
18004 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
18005
18006 let ModuleControlResponseToModule::ScopeDescribe {
18008 status,
18009 daemon_incarnation,
18010 owner_synced,
18011 owner_configured,
18012 scope,
18013 ..
18014 } = describe(&handler, &reader, OWNER, "s").await
18015 else {
18016 panic!("not a describe reply");
18017 };
18018 assert_eq!(status, ScopeStatus::NotLive);
18019 assert_eq!(daemon_incarnation, "incarnation-7");
18020 assert!(!owner_synced);
18021 assert!(owner_configured);
18022 assert!(scope.is_none());
18023
18024 let ModuleControlResponseToModule::ScopeDescribe {
18027 status,
18028 owner_configured,
18029 ..
18030 } = describe(&handler, &reader, "ghost", "s").await
18031 else {
18032 panic!("not a describe reply");
18033 };
18034 assert_eq!(status, ScopeStatus::NotLive);
18035 assert!(!owner_configured);
18036
18037 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
18039 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
18040 let ModuleControlResponseToModule::ScopeDescribe {
18041 status,
18042 scope_epoch,
18043 owner_synced,
18044 scope,
18045 ..
18046 } = describe(&handler, &reader, OWNER, "s").await
18047 else {
18048 panic!("not a describe reply");
18049 };
18050 assert_eq!(status, ScopeStatus::Live);
18051 assert_eq!(scope_epoch, Some(4));
18052 assert!(owner_synced);
18053 let stamp = scope.expect("a live scope carries its stamp");
18054 assert!(
18055 stamp.owner_authorized,
18056 "prefrontal-core is the default authority"
18057 );
18058 assert_eq!(stamp.kind, ScopeKind::Head);
18059 }
18060
18061 #[tokio::test]
18062 async fn scope_authority_owners_decides_owner_authorized() {
18063 let supervisor = SupervisorHandle::new();
18064 supervisor.set_spawn_nonce("broca", "b1".to_string());
18065 let handler = ControlHandler::new(Arc::new(Registry::default()))
18066 .with_supervisor(supervisor)
18067 .with_scope_authority_owners(vec!["broca".to_string()]);
18068 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
18069 let mut gated = head("s", 1);
18070 gated.attributes.agent_id = Some("agent".to_string());
18071 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
18072 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
18073 describe(&handler, &broca, "broca", "s").await
18074 else {
18075 panic!("not a describe reply");
18076 };
18077 assert!(scope.unwrap().owner_authorized);
18078 }
18079
18080 #[tokio::test]
18084 async fn scope_ops_and_the_scopes_capability_are_advertised() {
18085 let handler = ControlHandler::new(Arc::new(Registry::default()));
18086 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
18087 let ack = hello_via_sink(
18088 &handler,
18089 &ctx,
18090 &mut rx,
18091 hello_frame("m", PROTOCOL_VERSION, 1),
18092 )
18093 .await;
18094 let ack = parse_ack(&ack);
18095 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
18096 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
18097 }
18098 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
18099
18100 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
18101 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
18102 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
18103 let reply = handler
18104 .handle_control_frame(&client, frame)
18105 .await
18106 .unwrap()
18107 .pop()
18108 .unwrap();
18109 let ClientControlResponse::ServerDescribe { capabilities, .. } =
18110 serde_json::from_slice(&reply.body).unwrap()
18111 else {
18112 panic!("not a server.describe reply");
18113 };
18114 assert!(
18115 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
18116 "{capabilities:?}"
18117 );
18118 }
18119
18120 const PLEXUS: &str = "plexus";
18123 const OTHER: &str = "other";
18124 const AFT: &str = "aft";
18125 const BROCA: &str = "broca";
18126 const MAGIC: &str = "magic-context";
18127
18128 fn nonce(module_id: &str) -> String {
18129 format!("nonce-{module_id}")
18130 }
18131
18132 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
18133 let (tx, rx) = mpsc::channel(64);
18134 (
18135 RouteCtx {
18136 connection_id: ConnectionId::new(connection),
18137 egress: FrameSink::new(tx),
18138 },
18139 rx,
18140 )
18141 }
18142
18143 struct Rig {
18147 handler: ControlHandler,
18148 forwarding: Arc<ForwardingTable>,
18149 owner: RouteCtx,
18150 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18151 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
18152 generation: u64,
18153 next_connection: u64,
18154 _supervisor: Supervisor,
18155 }
18156
18157 async fn rig() -> Rig {
18158 rig_with_flow_support(true).await
18159 }
18160
18161 async fn rig_with_flow_support(flow_support: bool) -> Rig {
18162 rig_with_scope_capabilities(flow_support, false).await
18163 }
18164
18165 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
18166 let mut provides = Vec::new();
18167 if flow_support {
18168 provides.push("flow-scopes/v1");
18169 }
18170 if run_support {
18171 provides.push("agent-run-scopes/v1");
18172 }
18173 rig_with_target_capabilities(&provides).await
18174 }
18175
18176 fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
18177 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
18178 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
18179 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
18182 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
18183 Frame::build(
18184 FrameType::Hello,
18185 control_flags(),
18186 0,
18187 0,
18188 connection,
18189 serde_json::to_vec(&body).unwrap(),
18190 )
18191 .unwrap()
18192 }
18193
18194 async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
18195 let registry = Arc::new(Registry::default());
18196 let forwarding = Arc::new(ForwardingTable::default());
18197 let supervisor_handle = SupervisorHandle::new();
18198 let supervisor =
18199 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
18200 .with_handle(supervisor_handle.clone());
18201 supervisor
18202 .supervise_configured(
18203 ModuleSpec {
18204 carry_containment: false,
18205 module_id: OWNER.to_string(),
18206 program: PathBuf::from("/nonexistent/prefrontal-core"),
18207 args: Vec::new(),
18208 env: Vec::new(),
18209 reserved: false,
18210 reserved_prefixes: Vec::new(),
18211 protocol: ModuleProtocol::Subc,
18212 overlap: Default::default(),
18213 },
18214 false,
18215 )
18216 .unwrap();
18217 for module_id in [OWNER, AFT, BROCA, MAGIC] {
18218 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
18219 }
18220 let handler =
18221 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
18222 .with_supervisor(supervisor_handle);
18223 let (owner, mut owner_rx) = wide_ctx(1);
18224 hello_via_sink(
18225 &handler,
18226 &owner,
18227 &mut owner_rx,
18228 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
18229 )
18230 .await;
18231 let mut modules = BTreeMap::new();
18232 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
18233 let (ctx, mut rx) = wide_ctx(connection);
18234 let hello = hello_with_capabilities(module_id, connection, provides);
18235 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
18236 modules.insert(module_id.to_string(), (ctx, rx));
18237 }
18238 Rig {
18239 handler,
18240 forwarding,
18241 owner,
18242 _owner_rx: owner_rx,
18243 modules,
18244 generation: 0,
18245 next_connection: 100,
18246 _supervisor: supervisor,
18247 }
18248 }
18249
18250 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
18251 ScopeCarrier::new(Principal::Reserved {
18252 module_id: module_id.to_string(),
18253 })
18254 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
18255 }
18256
18257 fn session(scope_epoch: u64) -> ScopeRecord {
18260 let mut record = head("s", scope_epoch);
18261 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
18262 record.attributes.agent_id = Some("agent-1".to_string());
18263 record.attributes.delegates = true;
18264 record
18265 }
18266
18267 impl Rig {
18268 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
18269 self.generation += 1;
18270 sync(&self.handler, &self.owner, self.generation, scopes)
18271 .await
18272 .expect("the owner's sync is accepted");
18273 }
18274
18275 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
18276 ScopeSelector::new(
18277 Principal::Reserved {
18278 module_id: OWNER.to_string(),
18279 },
18280 scope_ref.to_string(),
18281 scope_epoch,
18282 )
18283 }
18284
18285 fn open_frame(
18286 &mut self,
18287 opener: Option<&str>,
18288 target: &str,
18289 scope: Option<ScopeSelector>,
18290 ) -> (
18291 RouteCtx,
18292 mpsc::Receiver<crate::router::OutboundFrame>,
18293 Frame,
18294 ) {
18295 self.next_connection += 1;
18296 let (ctx, rx) = wide_ctx(self.next_connection);
18297 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
18298 busy_refusals: None,
18299 target: RouteTarget::ToolProvider {
18300 module_id: target.to_string(),
18301 },
18302 identity: BindIdentity::new("unit".to_string(), "session".to_string()),
18303 consumer_identity: opener.map(|module_id| ConsumerIdentity {
18304 module_id: module_id.to_string(),
18305 launch_nonce: nonce(module_id),
18306 }),
18307 consumer_capabilities: None,
18308 role_versions: None,
18309 admission_facts: None,
18310 scope,
18311 })
18312 .unwrap();
18313 let frame = Frame::build(
18314 FrameType::Request,
18315 control_flags(),
18316 0,
18317 0,
18318 self.next_connection,
18319 body,
18320 )
18321 .unwrap();
18322 (ctx, rx, frame)
18323 }
18324
18325 async fn refused(
18327 &mut self,
18328 opener: Option<&str>,
18329 target: &str,
18330 scope: Option<ScopeSelector>,
18331 ) -> String {
18332 self.refusal_body(opener, target, scope).await["code"]
18333 .as_str()
18334 .unwrap()
18335 .to_string()
18336 }
18337
18338 async fn refusal_body(
18339 &mut self,
18340 opener: Option<&str>,
18341 target: &str,
18342 scope: Option<ScopeSelector>,
18343 ) -> Value {
18344 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
18345 let replies = tokio::time::timeout(
18346 Duration::from_secs(2),
18347 self.handler.handle_control_frame(&ctx, frame),
18348 )
18349 .await
18350 .expect("the open must be refused before waiting for a bind ack")
18351 .unwrap();
18352 assert_eq!(replies.len(), 1, "{replies:?}");
18353 assert_eq!(replies[0].header.ty, FrameType::Error);
18354 let (_, module_rx) = self.modules.get_mut(target).unwrap();
18355 assert!(
18356 module_rx.try_recv().is_err(),
18357 "a refused open relays nothing"
18358 );
18359 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
18360 parse_error(&replies[0])
18361 }
18362
18363 async fn relayed(
18365 &mut self,
18366 opener: Option<&str>,
18367 target: &str,
18368 scope: Option<ScopeSelector>,
18369 ) -> Relayed {
18370 self.relayed_with_legacy_root(opener, target, scope, None)
18371 .await
18372 }
18373
18374 async fn relayed_with_legacy_root(
18375 &mut self,
18376 opener: Option<&str>,
18377 target: &str,
18378 scope: Option<ScopeSelector>,
18379 legacy_root: Option<&std::path::Path>,
18380 ) -> Relayed {
18381 let (ctx, rx, mut frame) = self.open_frame(opener, target, scope);
18382 if let Some(root) = legacy_root {
18383 let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
18384 body["identity"]["project_root"] = serde_json::json!(root);
18385 frame.body = serde_json::to_vec(&body).unwrap();
18386 }
18387 let handler = self.handler.clone();
18388 let task_ctx = ctx.clone();
18389 let task = tokio::spawn(async move {
18390 handler
18391 .handle_control_frame(&task_ctx, frame)
18392 .await
18393 .unwrap()
18394 });
18395 let (_, module_rx) = self.modules.get_mut(target).unwrap();
18396 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
18397 .await
18398 .expect("the target receives the relayed route.bind")
18399 .unwrap()
18400 .frame;
18401 Relayed {
18402 target: target.to_string(),
18403 client: ctx,
18404 client_rx: rx,
18405 task,
18406 bind,
18407 }
18408 }
18409
18410 async fn ack(&self, relayed: &Relayed) {
18411 let (module, _) = &self.modules[&relayed.target];
18412 self.handler
18413 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
18414 .await
18415 .unwrap();
18416 }
18417
18418 async fn bound(
18420 &mut self,
18421 opener: Option<&str>,
18422 target: &str,
18423 scope: Option<ScopeSelector>,
18424 ) -> Bound {
18425 let relayed = self.relayed(opener, target, scope).await;
18426 self.commit_relayed(relayed).await
18427 }
18428
18429 async fn bound_with_legacy_root(
18430 &mut self,
18431 opener: Option<&str>,
18432 target: &str,
18433 scope: Option<ScopeSelector>,
18434 root: &std::path::Path,
18435 ) -> Bound {
18436 let relayed = self
18437 .relayed_with_legacy_root(opener, target, scope, Some(root))
18438 .await;
18439 self.commit_relayed(relayed).await
18440 }
18441
18442 async fn commit_relayed(&self, relayed: Relayed) -> Bound {
18443 self.ack(&relayed).await;
18444 let Relayed {
18445 target,
18446 client,
18447 mut client_rx,
18448 task,
18449 bind,
18450 } = relayed;
18451 assert!(
18452 task.await.unwrap().is_empty(),
18453 "the open is answered by commit"
18454 );
18455 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
18456 Bound {
18457 target,
18458 client,
18459 client_rx,
18460 channel,
18461 epoch,
18462 bind,
18463 }
18464 }
18465
18466 fn live(&self, route: &Bound) -> bool {
18467 matches!(
18468 self.forwarding
18469 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
18470 .unwrap(),
18471 DataRoute::Client(DataRouteState::Bound(_))
18472 )
18473 }
18474 }
18475
18476 struct Relayed {
18477 target: String,
18478 client: RouteCtx,
18479 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18480 task: tokio::task::JoinHandle<Vec<Frame>>,
18481 bind: Frame,
18482 }
18483
18484 struct Bound {
18485 target: String,
18486 client: RouteCtx,
18487 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18488 channel: u16,
18489 epoch: u32,
18490 bind: Frame,
18491 }
18492
18493 impl Bound {
18494 fn closed_reason(&mut self) -> RouteCloseReason {
18497 let mut reason = None;
18498 let mut goodbye = false;
18499 while let Ok(outbound) = self.client_rx.try_recv() {
18500 let frame = outbound.frame;
18501 match frame.header.ty {
18502 FrameType::Goodbye => {
18503 assert_eq!(
18504 (frame.header.channel, frame.header.epoch),
18505 (self.channel, self.epoch)
18506 );
18507 goodbye = true;
18508 }
18509 FrameType::Push => {
18510 let ClientControlPush::RouteClosed {
18511 reason: r,
18512 module_id,
18513 ..
18514 } = serde_json::from_slice(&frame.body).unwrap()
18515 else {
18516 panic!("unexpected push");
18517 };
18518 assert_eq!(module_id, self.target);
18519 reason = Some(r);
18520 }
18521 other => panic!("unexpected frame {other:?}"),
18522 }
18523 }
18524 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
18525 reason.expect("the client is told why the route closed")
18526 }
18527
18528 fn untouched(&mut self) -> bool {
18529 self.client_rx.try_recv().is_err()
18530 }
18531
18532 fn stamp(&self) -> Option<ScopeStamp> {
18533 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
18534 ModuleControlRequest::RouteBind { scope, .. } => scope,
18535 other => panic!("expected a route.bind, got {other:?}"),
18536 }
18537 }
18538 }
18539
18540 #[tokio::test]
18541 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
18542 ) {
18543 let mut rig = rig().await;
18544 let mut record = session(1);
18545 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
18546 record.child_owners = vec![Principal::Reserved {
18547 module_id: MAGIC.to_string(),
18548 }];
18549 rig.sync(vec![record]).await;
18550 let scope = || Some(rig_selector("s", Some(1)));
18551
18552 rig.bound(Some(OWNER), PLEXUS, scope()).await;
18555 rig.bound(Some(AFT), OTHER, scope()).await;
18556 rig.bound(Some(BROCA), PLEXUS, scope()).await;
18557
18558 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
18562 assert_eq!(
18563 rig.refused(opener, target, scope()).await,
18564 error_codes::SCOPE_NOT_CARRIER,
18565 "{opener:?} -> {target}"
18566 );
18567 }
18568 }
18569
18570 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
18571 ScopeSelector::new(
18572 Principal::Reserved {
18573 module_id: OWNER.to_string(),
18574 },
18575 scope_ref.to_string(),
18576 scope_epoch,
18577 )
18578 }
18579
18580 #[tokio::test]
18581 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
18582 let mut rig = rig().await;
18583 rig.sync(vec![session(1)]).await;
18584 for opener in [OWNER, AFT] {
18585 assert_eq!(
18586 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
18587 .await,
18588 error_codes::SCOPE_EPOCH_REQUIRED,
18589 "{opener}"
18590 );
18591 }
18592 }
18593
18594 #[tokio::test]
18595 async fn admission_separates_not_synced_not_live_and_ended() {
18596 let mut rig = rig().await;
18597 let code = rig
18599 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18600 .await;
18601 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
18602 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
18603
18604 let ghost = ScopeSelector::new(
18606 Principal::Reserved {
18607 module_id: "ghost".to_string(),
18608 },
18609 "s".to_string(),
18610 Some(1),
18611 );
18612 assert_eq!(
18613 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
18614 error_codes::SCOPE_NOT_LIVE
18615 );
18616
18617 rig.sync(vec![session(2)]).await;
18618 assert_eq!(
18619 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
18620 .await,
18621 error_codes::SCOPE_NOT_LIVE
18622 );
18623 for epoch in [1, 3] {
18624 assert_eq!(
18625 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
18626 .await,
18627 error_codes::SCOPE_ENDED,
18628 "epoch {epoch}"
18629 );
18630 }
18631 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
18633 .await;
18634 }
18635
18636 #[tokio::test]
18637 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
18638 let mut rig = rig().await;
18639 rig.sync(vec![session(1)]).await;
18640 let route = rig
18641 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18642 .await;
18643 let stamp = route.stamp().expect("a scoped bind carries the stamp");
18644 assert_eq!(stamp.scope_ref, "s");
18645 assert_eq!(stamp.scope_epoch, 1);
18646 assert_eq!(stamp.kind, ScopeKind::Head);
18647 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
18648 assert!(stamp.attributes.delegates);
18649 assert!(stamp.owner_authorized);
18650 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
18651 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
18652
18653 let (broca, mut broca_rx) = wide_ctx(50);
18656 hello_via_sink(
18657 &rig.handler,
18658 &broca,
18659 &mut broca_rx,
18660 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
18661 )
18662 .await;
18663 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
18664 .await
18665 .unwrap();
18666 let own = ScopeSelector::new(
18667 Principal::Reserved {
18668 module_id: BROCA.to_string(),
18669 },
18670 "b".to_string(),
18671 Some(1),
18672 );
18673 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
18674 assert!(!route.stamp().unwrap().owner_authorized);
18675 }
18676
18677 #[tokio::test]
18678 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
18679 let mut rig = rig().await;
18680 let mut record = head("s", 1);
18681 record.carriers = vec![carrier(AFT, None)];
18682 let flow_id = "Flow:run-7/step_2!~";
18683 record.attributes.flow_id = Some(flow_id.to_string());
18684 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
18685 .await
18686 .unwrap();
18687 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
18688 panic!("not a sync reply");
18689 };
18690 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
18691 let route = rig
18692 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18693 .await;
18694 assert!(rig.live(&route), "the stamped bind committed");
18695 let stamp = route.stamp().expect("a flow scope carries a stamp");
18696 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
18697 assert_eq!(stamp.attributes.agent_id, None);
18698 assert!(!stamp.attributes.delegates);
18699 assert!(stamp.owner_authorized);
18700 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
18701 assert_eq!(unscoped.stamp(), None);
18702 }
18703
18704 #[tokio::test]
18705 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
18706 let mut rig = rig_with_flow_support(false).await;
18707 let mut record = session(1);
18708 record.attributes.flow_id = Some("flow:7".to_string());
18709 rig.sync(vec![record]).await;
18710 for opener in [OWNER, AFT] {
18711 let body = rig
18712 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
18713 .await;
18714 assert_eq!(body["code"], "target_flow_unsupported");
18715 let message = body["message"].as_str().unwrap();
18716 for required in [PLEXUS, "flow-scopes/v1"] {
18717 assert!(message.contains(required), "{message}");
18718 }
18719 }
18720 }
18721
18722 #[tokio::test]
18723 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
18724 let mut rig = rig_with_flow_support(true).await;
18725 let mut record = session(1);
18726 record.attributes.flow_id = Some("flow:7".to_string());
18727 rig.sync(vec![record]).await;
18728 for opener in [OWNER, AFT] {
18729 let route = rig
18730 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
18731 .await;
18732 assert!(rig.live(&route));
18733 assert_eq!(
18734 route.stamp().unwrap().attributes.flow_id.as_deref(),
18735 Some("flow:7")
18736 );
18737 }
18738 }
18739
18740 #[tokio::test]
18741 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
18742 use std::future::Future;
18743
18744 let mut rig = rig_with_flow_support(true).await;
18745 let mut record = session(1);
18746 record.attributes.flow_id = Some("flow:7".to_string());
18747 rig.sync(vec![record]).await;
18748 let (client, mut client_rx, frame) =
18749 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
18750 for _ in 0..64 {
18753 client.egress.try_send(route_bind_ack(1)).unwrap();
18754 }
18755 let handler = rig.handler.clone();
18756 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
18757 std::future::poll_fn(|cx| {
18758 assert!(open.as_mut().poll(cx).is_pending());
18759 std::task::Poll::Ready(())
18760 })
18761 .await;
18762 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18763
18764 let old_connection = rig.modules[PLEXUS].0.connection_id;
18765 rig.handler.cleanup_connection(old_connection).unwrap();
18766 let (replacement, mut replacement_rx) = wide_ctx(200);
18767 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
18768 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
18769 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
18770 let hello = Frame::build(
18771 FrameType::Hello,
18772 control_flags(),
18773 0,
18774 0,
18775 200,
18776 serde_json::to_vec(&body).unwrap(),
18777 )
18778 .unwrap();
18779 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
18780
18781 client_rx.try_recv().unwrap();
18782 let replies = tokio::time::timeout(Duration::from_secs(2), open)
18783 .await
18784 .expect("the replacement is refused without waiting for a bind ack")
18785 .unwrap();
18786 assert_eq!(replies.len(), 1);
18787 let body = parse_error(&replies[0]);
18788 assert_eq!(body["code"], "target_flow_unsupported");
18789 for required in [PLEXUS, "flow-scopes/v1"] {
18790 assert!(body["message"].as_str().unwrap().contains(required));
18791 }
18792 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
18793 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
18794 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18795 assert!(rig
18796 .handler
18797 .registry
18798 .get_module_by_connection(replacement.connection_id)
18799 .unwrap()
18800 .is_some());
18801 }
18802
18803 #[tokio::test]
18804 async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
18805 use std::future::Future;
18806 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
18807 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
18808 const NEXT_HASH: &str =
18809 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
18810 let mut rig = rig().await;
18811 let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
18812 record.carriers.push(ScopeCarrier::new(Principal::Direct));
18813 rig.sync(vec![record.clone()]).await;
18814 let (client, mut rx, frame) = rig.open_frame(
18815 None,
18816 PLEXUS,
18817 Some(
18818 rig.selector("s", Some(1))
18819 .with_bind_proof(Some(PROOF.into())),
18820 ),
18821 );
18822 for _ in 0..64 {
18825 client.egress.try_send(route_bind_ack(1)).unwrap();
18826 }
18827 let handler = rig.handler.clone();
18828 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
18829 std::future::poll_fn(|cx| {
18830 assert!(open.as_mut().poll(cx).is_pending());
18831 std::task::Poll::Ready(())
18832 })
18833 .await;
18834 record.bind_proof_sha256 = Some(NEXT_HASH.into());
18835 rig.sync(vec![record]).await;
18836 rx.try_recv().unwrap();
18837 let replies = tokio::time::timeout(Duration::from_secs(2), open)
18838 .await
18839 .unwrap()
18840 .unwrap();
18841 assert_eq!(
18842 parse_error(&replies[0])["code"],
18843 error_codes::SCOPE_BIND_PROOF_MISMATCH
18844 );
18845 assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
18846 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
18847 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18848 }
18849
18850 #[tokio::test]
18851 async fn scope_bind_proof_reserved_openers_ignore_proof() {
18852 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
18853 let mut rig = rig().await;
18854 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
18855 .await;
18856 for opener in [OWNER, AFT] {
18857 for proof in [None, Some("malformed+proof=")] {
18858 let route = rig
18859 .bound(
18860 Some(opener),
18861 PLEXUS,
18862 Some(
18863 rig.selector("s", Some(1))
18864 .with_bind_proof(proof.map(str::to_string)),
18865 ),
18866 )
18867 .await;
18868 assert!(rig.live(&route));
18869 }
18870 }
18871 }
18872
18873 #[tokio::test]
18874 async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
18875 use std::future::Future;
18876 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
18877 const NEXT_HASH: &str =
18878 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
18879 let mut rig = rig().await;
18880 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
18881 .await;
18882 let (client, mut rx, frame) =
18883 rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
18884 for _ in 0..64 {
18885 client.egress.try_send(route_bind_ack(1)).unwrap();
18886 }
18887 let handler = rig.handler.clone();
18888 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
18889 std::future::poll_fn(|cx| {
18890 assert!(open.as_mut().poll(cx).is_pending());
18891 std::task::Poll::Ready(())
18892 })
18893 .await;
18894 rig.sync(vec![
18895 session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
18896 ])
18897 .await;
18898 rx.try_recv().unwrap();
18899 let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
18900 let bind = std::future::poll_fn(|cx| {
18901 assert!(open.as_mut().poll(cx).is_pending());
18902 module_rx.poll_recv(cx)
18903 })
18904 .await
18905 .unwrap()
18906 .frame;
18907 handler
18908 .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
18909 .await
18910 .unwrap();
18911 let replies = tokio::time::timeout(Duration::from_secs(2), open)
18912 .await
18913 .unwrap()
18914 .unwrap();
18915 assert_eq!(replies.len(), 1);
18916 assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
18919 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18920 while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
18921 let route = rig
18922 .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
18923 .await;
18924 assert!(rig.live(&route));
18925 }
18926
18927 #[tokio::test]
18928 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
18929 {
18930 let mut rig = rig_with_flow_support(false).await;
18931 rig.sync(vec![session(1)]).await;
18932 let route = rig
18933 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18934 .await;
18935 assert!(rig.live(&route));
18936 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
18937 let mut record = session(1);
18938 record.attributes.flow_id = Some("flow:7".to_string());
18939 rig.sync(vec![record]).await;
18940 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
18941 assert!(rig.live(&unscoped));
18942 assert_eq!(unscoped.stamp(), None);
18943 }
18944
18945 #[tokio::test]
18946 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
18947 let mut rig = rig().await;
18948 let mut record = session(1);
18949 record.attributes.flow_id = Some("flow:7".to_string());
18950 rig.sync(vec![record.clone()]).await;
18951 let before = rig.forwarding.published_scope_tag(OWNER, "s");
18952 let mut owner_route = rig
18953 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
18954 .await;
18955 let mut carrier_route = rig
18956 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
18957 .await;
18958 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
18959 rig.sync(vec![record.clone()]).await;
18960 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
18961 assert!(rig.live(&owner_route) && owner_route.untouched());
18962 assert!(rig.live(&carrier_route) && carrier_route.untouched());
18963
18964 record.attributes.flow_id = Some("flow:8".to_string());
18965 rig.sync(vec![record]).await;
18966 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
18967 let before = before.unwrap();
18968 assert_eq!(after.scope_epoch, before.scope_epoch);
18969 assert!(after.version > before.version);
18970 for route in [&mut owner_route, &mut carrier_route] {
18971 assert!(!rig.live(route));
18972 assert_eq!(
18973 route.closed_reason(),
18974 RouteCloseReason::ScopeDelegationChanged
18975 );
18976 }
18977 assert!(rig.live(&unscoped) && unscoped.untouched());
18978 for target in [PLEXUS, OTHER] {
18981 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
18982 let goodbye = module_rx
18983 .try_recv()
18984 .expect("the provider sees the drain")
18985 .frame;
18986 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
18987 assert!(module_rx.try_recv().is_err());
18988 }
18989 let rebound = rig
18990 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18991 .await;
18992 assert_eq!(
18993 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
18994 Some("flow:8")
18995 );
18996 }
18997
18998 #[tokio::test]
19003 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
19004 let mut rig = rig().await;
19005 rig.sync(vec![session(1)]).await;
19006 let mut cotenant = rig
19007 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
19008 .await;
19009
19010 let mut changed = session(1);
19011 changed.child_owners.push(Principal::Reserved {
19012 module_id: MAGIC.to_string(),
19013 });
19014 let mut ended = None;
19015 for (code, next) in [
19016 (error_codes::SCOPE_CHANGED, vec![changed]),
19017 (error_codes::SCOPE_ENDED, Vec::new()),
19018 ] {
19019 let relayed = rig
19020 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19021 .await;
19022 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
19023 ended = Some(next.is_empty());
19024 rig.sync(next).await;
19025 rig.ack(&relayed).await;
19026 let replies = relayed.task.await.unwrap();
19027 assert_eq!(replies.len(), 1, "{replies:?}");
19028 assert_eq!(parse_error(&replies[0])["code"], code);
19029 assert_eq!(
19030 subc_protocol::error_codes::is_retryable_route_open(code),
19031 code == error_codes::SCOPE_CHANGED
19032 );
19033 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
19034 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
19036 let mut goodbyes = Vec::new();
19039 while let Ok(outbound) = plexus_rx.try_recv() {
19040 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
19041 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
19042 }
19043 assert!(
19044 goodbyes.contains(&(bind_channel, bind_epoch)),
19045 "{goodbyes:?}"
19046 );
19047 assert!(rig
19048 .handler
19049 .registry
19050 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
19051 .unwrap()
19052 .is_some());
19053 }
19054 assert_eq!(ended, Some(true));
19055 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
19058 }
19059
19060 #[tokio::test]
19063 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
19064 struct Case {
19065 name: &'static str,
19066 change: fn(&mut ScopeRecord),
19067 closed: [Option<RouteCloseReason>; 4],
19070 }
19071 use RouteCloseReason::*;
19072 let cases = [
19073 Case {
19074 name: "a carrier entry removed",
19075 change: |r| {
19076 r.carriers.retain(|c| {
19077 c.principal
19078 != Principal::Reserved {
19079 module_id: AFT.to_string(),
19080 }
19081 })
19082 },
19083 closed: [None, Some(ScopeCarrierRemoved), None, None],
19084 },
19085 Case {
19086 name: "a target removed from a carrier",
19087 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
19088 closed: [None, None, None, Some(ScopeCarrierRemoved)],
19089 },
19090 Case {
19091 name: "a bare carrier narrowed to targets",
19092 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
19093 closed: [None, Some(ScopeCarrierRemoved), None, None],
19094 },
19095 Case {
19096 name: "delegates turned off",
19097 change: |r| r.attributes.delegates = false,
19098 closed: [Some(ScopeDelegationChanged); 4],
19099 },
19100 Case {
19101 name: "agent_id changed",
19102 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
19103 closed: [Some(ScopeDelegationChanged); 4],
19104 },
19105 Case {
19106 name: "a carrier added, child owners changed, the record re-sent",
19107 change: |r| {
19108 r.carriers.push(carrier(MAGIC, None));
19109 r.child_owners.push(Principal::Reserved {
19110 module_id: MAGIC.to_string(),
19111 });
19112 },
19113 closed: [None; 4],
19114 },
19115 Case {
19116 name: "a target added",
19117 change: |r| {
19118 r.carriers[1]
19119 .targets
19120 .as_mut()
19121 .unwrap()
19122 .push("third".to_string())
19123 },
19124 closed: [None; 4],
19125 },
19126 Case {
19127 name: "delegates turned on",
19128 change: |r| r.attributes.delegates = true,
19129 closed: [None; 4],
19130 },
19131 ];
19132 for case in cases {
19133 let mut rig = rig().await;
19134 rig.sync(vec![session(1)]).await;
19135 let scope = || Some(rig_selector("s", Some(1)));
19136 let mut routes = [
19137 rig.bound(Some(OWNER), PLEXUS, scope()).await,
19138 rig.bound(Some(AFT), PLEXUS, scope()).await,
19139 rig.bound(Some(BROCA), PLEXUS, scope()).await,
19140 rig.bound(Some(BROCA), OTHER, scope()).await,
19141 ];
19142 let mut record = session(1);
19143 (case.change)(&mut record);
19144 rig.sync(vec![record]).await;
19145 for (index, expected) in case.closed.iter().enumerate() {
19146 let route = &mut routes[index];
19147 match expected {
19148 Some(reason) => {
19149 assert!(!rig.live(route), "{}: route {index} still live", case.name);
19150 assert_eq!(
19151 route.closed_reason(),
19152 *reason,
19153 "{}: route {index}",
19154 case.name
19155 );
19156 }
19157 None => {
19158 assert!(rig.live(route), "{}: route {index} closed", case.name);
19159 assert!(
19160 route.untouched(),
19161 "{}: route {index} was told something",
19162 case.name
19163 );
19164 }
19165 }
19166 }
19167 }
19168 }
19169
19170 #[tokio::test]
19171 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
19172 for next in [Vec::new(), vec![session(2)]] {
19174 let mut rig = rig().await;
19175 rig.sync(vec![session(1)]).await;
19176 let mut route = rig
19177 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19178 .await;
19179 rig.sync(next).await;
19180 assert!(!rig.live(&route));
19181 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
19182 }
19183
19184 let mut rig = rig().await;
19187 let mut child = session(1);
19188 child.scope_ref = "child".to_string();
19189 child.kind = ScopeKind::Worker;
19190 child.parent = Some(ScopeParent::new(
19191 Principal::Reserved {
19192 module_id: OWNER.to_string(),
19193 },
19194 "s".to_string(),
19195 1,
19196 ));
19197 rig.sync(vec![session(1), child.clone()]).await;
19198 let mut child_route = rig
19199 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
19200 .await;
19201 assert_eq!(
19202 child_route.stamp().unwrap().parent_state,
19203 Some(ParentState::Linked)
19204 );
19205 rig.sync(vec![child]).await;
19206 assert!(!rig.live(&child_route));
19207 assert_eq!(
19208 child_route.closed_reason(),
19209 RouteCloseReason::ScopeParentEnded
19210 );
19211 }
19212
19213 #[tokio::test]
19214 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
19215 ) {
19216 let mut rig = rig().await;
19217 rig.sync(vec![session(1)]).await;
19218 let mut route = rig
19219 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19220 .await;
19221 let before = rig.forwarding.published_scope_tag(OWNER, "s");
19222 rig.sync(vec![session(1)]).await;
19223 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
19224 assert!(rig.live(&route) && route.untouched());
19225
19226 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
19230 .forwarding
19231 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
19232 .unwrap()
19233 else {
19234 panic!("the route is bound");
19235 };
19236 binding.flow.acquire_tagged(9, false).await.unwrap();
19237 let mut widened = session(1);
19238 widened.carriers.push(carrier(MAGIC, None));
19239 rig.sync(vec![widened]).await;
19240 assert!(rig.live(&route) && route.untouched());
19241 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
19242 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
19243 assert_eq!(binding.flow.in_flight(), 1);
19246 binding
19247 .flow
19248 .acquire_tagged(10, false)
19249 .await
19250 .expect("the flow is still open");
19251 }
19252
19253 #[tokio::test]
19256 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
19257 let mut rig = rig().await;
19258 rig.sync(vec![session(1)]).await;
19259 let mut on_incumbent = rig
19260 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19261 .await;
19262
19263 let (candidate, _candidate_rx) = wide_ctx(9);
19266 let registration = rig
19267 .handler
19268 .registry
19269 .register_candidate_with_control_ops(
19270 manifest(PLEXUS, PROTOCOL_VERSION),
19271 PROTOCOL_VERSION,
19272 candidate.connection_id,
19273 module_baseline_control_ops(),
19274 )
19275 .unwrap();
19276 rig.forwarding
19277 .register_candidate_module_connection(
19278 candidate.connection_id,
19279 PLEXUS.to_string(),
19280 PROTOCOL_VERSION,
19281 manifest_concurrency(®istration.manifest),
19282 candidate.egress.clone(),
19283 )
19284 .unwrap();
19285 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
19286 rig.handler
19287 .registry
19288 .promote_candidate(PLEXUS)
19289 .unwrap()
19290 .unwrap();
19291 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
19292
19293 rig.sync(Vec::new()).await;
19294 assert!(!rig.live(&on_incumbent));
19295 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
19296 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
19297 let goodbye = incumbent_rx
19298 .try_recv()
19299 .expect("the superseded endpoint is told")
19300 .frame;
19301 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
19302 }
19303 }
19304}
19305
19306#[cfg(test)]
19307mod concurrency_default_exposure_tests {
19308 use super::*;
19309
19310 fn hello_body(role_json: &str) -> Vec<u8> {
19311 format!(
19312 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
19313 )
19314 .into_bytes()
19315 }
19316
19317 fn manifest_from(body: &[u8]) -> ModuleManifest {
19318 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
19319 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
19320 .expect("manifest parses")
19321 }
19322
19323 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
19324
19325 #[test]
19326 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
19327 let body = hello_body(&format!(
19328 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
19329 ));
19330 let manifest = manifest_from(&body);
19331 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
19334 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
19335 }
19336
19337 #[test]
19338 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
19339 let body = hello_body(&format!(
19340 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
19341 ));
19342 let manifest = manifest_from(&body);
19343 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
19344 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
19345 }
19346
19347 #[test]
19348 fn non_management_roles_are_never_reported() {
19349 let body = hello_body(
19350 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
19351 );
19352 let manifest = manifest_from(&body);
19353 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
19354 }
19355}