1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 path::{Path, PathBuf},
5 sync::{Arc, Mutex, RwLock},
6 time::{Duration, Instant as StdInstant},
7};
8
9use serde::{Deserialize, Serialize};
10use subc_control::{
11 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
12 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
13 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
14 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
15 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
16 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
17 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
18};
19use subc_protocol::{
20 error_codes,
21 manifest::{
22 validate_hello_capability_grammar, validate_hello_event_declarations,
23 validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
24 Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
25 },
26 scope::{
27 ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
28 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_DESCRIBE_OP, SCOPE_SYNC_OP,
29 },
30 session::{
31 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
32 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
33 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
34 ROLE_VERSIONS_FIELD,
35 },
36 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
37 Priority, RouteTarget, PROTOCOL_VERSION,
38};
39use tokio::time::{timeout_at, Instant};
40use tracing::{debug, info, warn};
41
42use crate::{
43 capability_requirements::{
44 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
45 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
46 RuntimeModule,
47 },
48 daemon_config::RestartRequiredSection,
49 forwarding::{
50 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
51 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
52 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
53 },
54 observability::{
55 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
56 },
57 provenance::{
58 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
59 },
60 registry::{ChannelState, ConnectionId, Registry, RegistryError},
61 router::{RouteCtx, RouterError},
62 scopes::{BoundScope, HelloLaunchNonces, ScopeTable},
63 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
64 stderr_tail::{CaptureState, TailEntry},
65 supervise::{
66 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
67 SupervisorHandle, SwapHelloAdmission,
68 },
69 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
70};
71
72pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
78
79const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
80const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
81const CAP_PING_PONG: &str = "ping_pong_v1";
82const CAP_SESSION_ATTACH: &str = "session_attach_v1";
83const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
84
85const SUBC_CONTROL_OPS: &[&str] = &[
86 ops::SERVER_DESCRIBE,
87 ops::CATALOG_LIST,
88 ops::ROUTE_OPEN,
89 ops::ROUTE_POLL,
90 ops::ROUTE_CLOSING,
91 ops::ROUTE_CLOSED,
92 ops::SUPERVISOR_LIST,
93 ops::SUPERVISOR_RESTART,
94 ops::SUPERVISOR_SWAP,
95 ops::SUPERVISOR_RELOAD,
96 ops::SUPERVISOR_RESCAN,
97 ops::SUPERVISOR_RELEASE_RESERVED,
98 ops::SUPERVISOR_SET_ENABLED,
99 ops::SUPERVISOR_HEALTH_PROBE,
100 ops::SUPERVISOR_HEALTH,
101 ops::SUPERVISOR_STDERR_TAIL,
102 ops::SUPERVISOR_TERMINALS,
103 ops::SUPERVISOR_ROUTES,
104 ops::SUPERVISOR_PROVENANCE,
105 ops::SUPERVISOR_SPAWN_SNAPSHOT,
106 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
107];
108
109const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
110 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
111 "supervisor.live_roots",
112 SCOPE_SYNC_OP,
113 SCOPE_DESCRIBE_OP,
114 "operator.confirm",
115];
116
117const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
121
122const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
123
124pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
132
133pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
148
149pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
167
168const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
169const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
170
171fn reload_verdict(
172 configured: &Path,
173 spawned_from: Option<&Path>,
174 image: subc_control::RunningImageAgreement,
175) -> PendingReloadVerdict {
176 let path = match spawned_from {
177 Some(spawned_from) if configured == spawned_from => ReloadPathAgreement::Match,
178 Some(spawned_from) => ReloadPathAgreement::Mismatch {
179 configured: configured.to_path_buf(),
180 spawned_from: spawned_from.to_path_buf(),
181 },
182 None => ReloadPathAgreement::Unavailable {
183 reason: if matches!(
184 image,
185 subc_control::RunningImageAgreement::Unavailable {
186 reason: subc_control::RunningImageUnavailableReason::NotRunning
187 }
188 ) {
189 ReloadPathUnavailableReason::NotRunning
190 } else {
191 ReloadPathUnavailableReason::SpawnedPathUnavailable
192 },
193 },
194 };
195 PendingReloadVerdict { path, image }
196}
197
198#[derive(Clone)]
199struct DaemonProvenanceFacts {
200 build: DaemonBuildProvenance,
201 pid: Option<u32>,
202 started_at_ms: Option<u64>,
203 start_clock: Option<crate::clock::StartClock>,
204 executable_path: Option<PathBuf>,
205 executable_identity: Option<SpawnedFileIdentity>,
206 process_start_time: Option<u64>,
207 probe: ExecutableIdentityProbe,
208}
209
210impl Default for DaemonProvenanceFacts {
211 fn default() -> Self {
212 Self {
213 build: DaemonBuildProvenance {
214 build_git_sha: None,
215 build_lock_digest: None,
216 },
217 pid: None,
218 started_at_ms: None,
219 start_clock: None,
220 executable_path: None,
221 executable_identity: None,
222 process_start_time: None,
223 probe: ExecutableIdentityProbe::default(),
224 }
225 }
226}
227
228#[derive(Debug, Clone)]
229struct SupervisorRescanContext {
230 supervisor: Supervisor,
231 config_path: PathBuf,
232 configured_port: Option<u16>,
233 storage_config: Option<crate::daemon_config::StorageConfig>,
234 admission_facts_carrier_module_id: Option<String>,
235 admission_facts_targets: Option<Vec<String>>,
236 scope_authority_owners: Vec<String>,
237}
238
239const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
260 "reloading",
261 "supervisor_not_live",
262 "registration_not_active",
263 "no_forwarding_connection",
264 "relay_send_failed",
265];
266
267#[derive(Clone)]
269pub struct ControlHandler {
270 registry: Arc<Registry>,
271 forwarding: Arc<ForwardingTable>,
272 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
273 supervisor: SupervisorHandle,
274 subc_capabilities: Arc<[String]>,
275 route_bind_relay_timeout: Duration,
279 route_bind_relay_timeouts: BTreeMap<String, Duration>,
283 route_bind_breakers: RouteBindBreakers,
286 route_bind_concurrency: RouteBindConcurrency,
289 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
294 route_bind_breaker_threshold: u32,
296 route_bind_breaker_cooldown: Duration,
298 health_probe_timeout: Duration,
299 storage_config: Option<crate::daemon_config::StorageConfig>,
302 machine_id: Option<crate::machine_id::MachineId>,
306 admission_facts_carrier_module_id: Option<String>,
307 admission_facts_targets: Option<Vec<String>>,
308 scopes: Arc<RwLock<ScopeTable>>,
312 scope_authority_owners: Vec<String>,
315 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
318 rescan: Option<SupervisorRescanContext>,
319 connected_clients: ConnectedClients,
320 counters: DaemonCounters,
321 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
322 daemon_provenance: DaemonProvenanceFacts,
323 #[cfg(test)]
324 control_dispatch_delay: Option<Duration>,
325 #[cfg(test)]
326 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
327}
328
329impl fmt::Debug for ControlHandler {
330 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
331 f.debug_struct("ControlHandler")
332 .field("registry", &self.registry)
333 .field("forwarding", &self.forwarding)
334 .field("process_liveness", &self.process_liveness.is_some())
335 .field("supervisor", &self.supervisor)
336 .field("subc_capabilities", &self.subc_capabilities)
337 .finish()
338 }
339}
340
341struct RouteOpenRequest {
342 target: RouteTarget,
343 identity: BindIdentity,
344 consumer_identity: Option<ConsumerIdentity>,
345 consumer_capabilities: Option<Vec<String>>,
346 role_versions: Option<BTreeMap<String, String>>,
347 admission_facts: Option<serde_json::Value>,
348 scope: Option<ScopeSelector>,
349}
350
351struct RouteBindReservationGuard {
352 forwarding: Arc<ForwardingTable>,
353 endpoint: ModuleEndpointId,
354 relay_corr: u64,
355 armed: bool,
356}
357
358struct ModuleControlRpcGuard {
359 forwarding: Arc<ForwardingTable>,
360 endpoint: ModuleEndpointId,
361 corr: u64,
362 armed: bool,
363}
364
365impl ModuleControlRpcGuard {
366 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
367 Self {
368 forwarding,
369 endpoint,
370 corr,
371 armed: true,
372 }
373 }
374
375 fn disarm(&mut self) {
376 self.armed = false;
377 }
378}
379
380impl Drop for ModuleControlRpcGuard {
381 fn drop(&mut self) {
382 if self.armed {
383 let _ = self
384 .forwarding
385 .cancel_module_control_rpc(self.endpoint, self.corr);
386 }
387 }
388}
389
390impl RouteBindReservationGuard {
391 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
392 Self {
393 forwarding,
394 endpoint,
395 relay_corr,
396 armed: true,
397 }
398 }
399
400 fn release_and_disarm(&mut self) {
401 if !self.armed {
402 return;
403 }
404 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
405 self.endpoint,
406 self.relay_corr,
407 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
408 ) {
409 send_goodbye_target_best_effort(
410 &self.forwarding.counters(),
411 &target,
412 "canceled route.bind",
413 );
414 }
415 self.armed = false;
416 }
417
418 fn disarm(&mut self) {
419 self.armed = false;
420 }
421}
422
423impl Drop for RouteBindReservationGuard {
424 fn drop(&mut self) {
425 self.release_and_disarm();
426 }
427}
428
429#[derive(Debug, Clone, Default)]
459pub(crate) struct RouteBindBreakers {
460 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
461}
462
463#[derive(Debug, Clone, Default)]
464pub(crate) struct RouteBindConcurrency {
465 modules: Arc<Mutex<HashMap<String, usize>>>,
466}
467
468struct RouteBindConcurrencyGuard {
469 concurrency: RouteBindConcurrency,
470 module_id: String,
471}
472
473impl RouteBindConcurrency {
474 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
477 let mut modules = self
478 .modules
479 .lock()
480 .expect("route.bind concurrency mutex poisoned");
481 let in_flight = modules.entry(module_id.to_string()).or_default();
482 if *in_flight >= limit {
483 return Err(*in_flight);
484 }
485 *in_flight += 1;
486 Ok(RouteBindConcurrencyGuard {
487 concurrency: self.clone(),
488 module_id: module_id.to_string(),
489 })
490 }
491}
492
493impl Drop for RouteBindConcurrencyGuard {
494 fn drop(&mut self) {
495 let mut modules = self
496 .concurrency
497 .modules
498 .lock()
499 .expect("route.bind concurrency mutex poisoned");
500 let remove = {
501 let in_flight = modules
502 .get_mut(&self.module_id)
503 .expect("admitted route.bind has a concurrency entry");
504 *in_flight -= 1;
505 *in_flight == 0
506 };
507 if remove {
508 modules.remove(&self.module_id);
509 }
510 }
511}
512
513#[derive(Debug, Default)]
514struct ModuleBreakerState {
515 consecutive_timeouts: u32,
517 cooldown_until: Option<Instant>,
520 probe_in_flight: Option<Arc<()>>,
525}
526
527enum RouteBindAdmission<'a> {
529 Admitted {
530 guard: RouteBindBreakerGuard<'a>,
531 probe: bool,
534 },
535 Refused {
536 consecutive_timeouts: u32,
537 retry_in: Duration,
541 probe_in_flight: bool,
542 },
543}
544
545struct RouteBindBreakerGuard<'a> {
552 breakers: RouteBindBreakers,
553 module_id: &'a str,
554 probe_token: Option<Arc<()>>,
555 settled: bool,
556}
557
558impl RouteBindBreakerGuard<'_> {
559 fn record_accepted(&mut self) -> bool {
563 self.settled = true;
564 self.breakers.record_accepted(self.module_id)
565 }
566
567 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
570 self.settled = true;
571 self.breakers.record_timeout(
572 self.module_id,
573 self.probe_token.as_ref(),
574 threshold,
575 cooldown,
576 )
577 }
578
579 fn record_inconclusive(&mut self) {
588 self.settled = true;
589 self.breakers
590 .record_inconclusive(self.module_id, self.probe_token.as_ref());
591 }
592}
593
594impl Drop for RouteBindBreakerGuard<'_> {
595 fn drop(&mut self) {
596 if !self.settled {
597 self.breakers
598 .record_inconclusive(self.module_id, self.probe_token.as_ref());
599 }
600 }
601}
602
603struct BreakerOpened {
607 consecutive_timeouts: u32,
608 reopened_after_probe: bool,
611}
612
613impl RouteBindBreakers {
614 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
615 self.modules
616 .lock()
617 .expect("route.bind breaker mutex poisoned")
618 }
619
620 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
623 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
624 probe: probe_token.is_some(),
625 guard: RouteBindBreakerGuard {
626 breakers: self.clone(),
627 module_id,
628 probe_token,
629 settled: false,
630 },
631 };
632
633 let mut modules = self.lock();
634 let Some(state) = modules.get_mut(module_id) else {
635 return admitted(None);
636 };
637 let Some(cooldown_until) = state.cooldown_until else {
638 return admitted(None);
639 };
640 if state.probe_in_flight.is_some() {
641 return RouteBindAdmission::Refused {
642 consecutive_timeouts: state.consecutive_timeouts,
643 retry_in: Duration::ZERO,
644 probe_in_flight: true,
645 };
646 }
647 let now = Instant::now();
648 if now < cooldown_until {
649 return RouteBindAdmission::Refused {
650 consecutive_timeouts: state.consecutive_timeouts,
651 retry_in: cooldown_until - now,
652 probe_in_flight: false,
653 };
654 }
655 let token = Arc::new(());
656 state.probe_in_flight = Some(Arc::clone(&token));
657 admitted(Some(token))
658 }
659
660 fn record_accepted(&self, module_id: &str) -> bool {
661 self.lock()
662 .remove(module_id)
663 .is_some_and(|state| state.cooldown_until.is_some())
664 }
665
666 fn record_timeout(
667 &self,
668 module_id: &str,
669 probe_token: Option<&Arc<()>>,
670 threshold: u32,
671 cooldown: Duration,
672 ) -> Option<BreakerOpened> {
673 let mut modules = self.lock();
674 let state = modules.entry(module_id.to_string()).or_default();
675 let was_open = state.cooldown_until.is_some();
676 let was_probe = Self::owns_probe(state, probe_token);
677 if was_probe {
678 state.probe_in_flight = None;
679 }
680 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
681 if state.consecutive_timeouts < threshold {
682 return None;
683 }
684 state.cooldown_until = Some(Instant::now() + cooldown);
685 Some(BreakerOpened {
686 consecutive_timeouts: state.consecutive_timeouts,
687 reopened_after_probe: was_open && was_probe,
688 })
689 }
690
691 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
692 state
696 .probe_in_flight
697 .as_ref()
698 .zip(token)
699 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
700 }
701
702 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
703 if let Some(state) = self.lock().get_mut(module_id) {
704 if Self::owns_probe(state, probe_token) {
705 state.probe_in_flight = None;
706 }
707 }
708 }
709
710 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
725 self.lock()
726 .remove(module_id)
727 .map(|state| state.consecutive_timeouts)
728 .filter(|discarded| *discarded > 0)
729 }
730
731 fn open_snapshot(&self) -> Option<serde_json::Value> {
738 let now = Instant::now();
739 let modules = self.lock();
740 let open = modules
741 .iter()
742 .filter_map(|(module_id, state)| {
743 let cooldown_until = state.cooldown_until?;
744 Some((
745 module_id.clone(),
746 serde_json::json!({
747 "consecutive_timeouts": state.consecutive_timeouts,
748 "cooldown_remaining_ms":
749 cooldown_until.saturating_duration_since(now).as_millis() as u64,
750 "probe_in_flight": state.probe_in_flight.is_some(),
751 }),
752 ))
753 })
754 .collect::<serde_json::Map<String, serde_json::Value>>();
755 (!open.is_empty()).then_some(serde_json::Value::Object(open))
756 }
757}
758
759impl ControlHandler {
760 pub fn new(registry: Arc<Registry>) -> Self {
761 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
762 }
763
764 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
765 let counters = forwarding.counters();
766 let route_bind_breakers = forwarding.route_bind_breakers();
770 let route_bind_concurrency = forwarding.route_bind_concurrency();
771 let route_outages = forwarding.route_outages();
772 Self {
773 registry,
774 forwarding,
775 process_liveness: None,
776 supervisor: SupervisorHandle::new(),
777 subc_capabilities: Arc::from([
778 CAP_MANIFEST_REGISTRATION.to_string(),
779 CAP_CHANNEL_LIFECYCLE.to_string(),
780 CAP_PING_PONG.to_string(),
781 CAP_SESSION_ATTACH.to_string(),
782 CAP_ADMISSION_FACTS_RELAY.to_string(),
783 CAP_SCOPES_V1.to_string(),
784 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
785 ]),
786 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
787 route_bind_relay_timeouts: BTreeMap::new(),
788 route_bind_breakers,
789 route_bind_concurrency,
790 route_outages,
791 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
792 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
793 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
794 storage_config: None,
795 machine_id: None,
796 admission_facts_carrier_module_id: None,
797 admission_facts_targets: None,
798 scopes: Arc::new(RwLock::new(ScopeTable::new(
799 crate::daemon_config::default_scope_authority_owners(),
800 ))),
801 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
802 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
803 rescan: None,
804 connected_clients: ConnectedClients::new(),
805 counters,
806 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
807 daemon_provenance: DaemonProvenanceFacts::default(),
808 #[cfg(test)]
809 control_dispatch_delay: None,
810 #[cfg(test)]
811 provenance_probe_override: None,
812 }
813 }
814
815 pub fn with_storage_config(
818 mut self,
819 storage_config: Option<crate::daemon_config::StorageConfig>,
820 ) -> Self {
821 self.storage_config = storage_config;
822 self
823 }
824
825 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
828 self.machine_id = machine_id;
829 self
830 }
831
832 pub fn with_admission_facts_config(
836 mut self,
837 carrier_module_id: Option<String>,
838 targets: Option<Vec<String>>,
839 ) -> Self {
840 self.admission_facts_carrier_module_id = carrier_module_id;
841 self.admission_facts_targets = targets;
842 self
843 }
844
845 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
849 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
850 self.scope_authority_owners = owners;
851 self
852 }
853
854 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
857 self.route_bind_relay_timeout = timeout;
858 self
859 }
860
861 pub fn with_route_bind_relay_timeouts(
867 mut self,
868 timeouts: impl IntoIterator<Item = (String, Duration)>,
869 ) -> Self {
870 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
871 self
872 }
873
874 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
880 self.route_bind_relay_timeouts
881 .get(module_id)
882 .copied()
883 .unwrap_or(self.route_bind_relay_timeout)
884 }
885
886 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
894 self.route_bind_breaker_threshold = threshold.max(1);
895 self.route_bind_breaker_cooldown = cooldown;
896 self
897 }
898
899 #[cfg(test)]
900 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
901 self.health_probe_timeout = timeout;
902 self
903 }
904
905 #[cfg(test)]
906 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
907 self.control_dispatch_delay = Some(delay);
908 self
909 }
910
911 pub fn with_process_liveness(
912 mut self,
913 process_liveness: Arc<dyn ModuleProcessLiveness>,
914 ) -> Self {
915 self.process_liveness = Some(process_liveness);
916 self
917 }
918
919 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
920 self.supervisor = supervisor;
921 self
922 }
923
924 pub fn with_daemon_provenance(
925 mut self,
926 pid: u32,
927 started_at_ms: u64,
928 executable_path: Option<PathBuf>,
929 build_git_sha: Option<String>,
930 build_lock_digest: Option<String>,
931 ) -> Self {
932 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
933 let process_start_time = process_start_time(pid);
934 self.daemon_provenance = DaemonProvenanceFacts {
935 build: DaemonBuildProvenance {
936 build_git_sha,
937 build_lock_digest,
938 },
939 pid: Some(pid),
940 started_at_ms: Some(started_at_ms),
941 start_clock: None,
942 executable_path,
943 executable_identity,
944 process_start_time,
945 probe: ExecutableIdentityProbe::default(),
946 };
947 self
948 }
949
950 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
951 self.daemon_provenance.start_clock = Some(clock);
952 self
953 }
954
955 #[cfg(test)]
956 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
957 self.provenance_probe_override = Some(result);
958 self
959 }
960
961 pub fn with_capability_config(
965 self,
966 modules: impl IntoIterator<Item = (String, bool)>,
967 reserved_capabilities: BTreeMap<String, String>,
968 ) -> Self {
969 self.capability_evaluator
970 .configure(modules, reserved_capabilities);
971 self
972 }
973
974 pub fn with_supervisor_rescan(
975 mut self,
976 supervisor: Supervisor,
977 config_path: impl Into<PathBuf>,
978 configured_port: Option<u16>,
979 ) -> Self {
980 self.rescan = Some(SupervisorRescanContext {
981 supervisor,
982 config_path: config_path.into(),
983 configured_port,
984 storage_config: self.storage_config.clone(),
985 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
986 admission_facts_targets: self.admission_facts_targets.clone(),
987 scope_authority_owners: self.scope_authority_owners.clone(),
988 });
989 self
990 }
991
992 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
993 self.connected_clients = connected_clients;
994 self
995 }
996
997 pub fn forwarding(&self) -> Arc<ForwardingTable> {
998 Arc::clone(&self.forwarding)
999 }
1000
1001 pub(crate) fn counters(&self) -> DaemonCounters {
1002 self.counters.clone()
1003 }
1004
1005 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1008 tokio::spawn(async move {
1009 loop {
1010 self.capability_evaluator
1011 .wait_for_change_or_deadline()
1012 .await;
1013 self.refresh_capability_requirements();
1014 }
1015 });
1016 }
1017
1018 fn runtime_capability_snapshot(
1019 &self,
1020 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1021 let runtime = self
1022 .supervisor
1023 .list()
1024 .into_iter()
1025 .map(|module| {
1026 let status = module.status().map_err(|err| {
1027 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1028 })?;
1029 Ok(RuntimeModule {
1030 module_id: status.module_id,
1031 state: status.state,
1032 enabled: status.enabled,
1033 })
1034 })
1035 .collect::<Result<Vec<_>, RouterError>>()?;
1036 let (_, registrations) = self.registry.list_modules().map_err(|err| {
1037 RouterError::backend(
1038 0,
1039 0,
1040 format!("failed to list capability registrations: {err}"),
1041 )
1042 })?;
1043 let registrations = registrations
1044 .into_iter()
1045 .map(|registration| RegisteredModule {
1046 module_id: registration.manifest.module_id,
1047 module_version: registration.manifest.module_version,
1048 capabilities: registration.manifest.capabilities,
1049 })
1050 .collect();
1051 Ok((runtime, registrations))
1052 }
1053
1054 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1061 let cached_registration = RegisteredModule {
1062 module_id: registration.manifest.module_id.clone(),
1063 module_version: registration.manifest.module_version.clone(),
1064 capabilities: registration.manifest.capabilities.clone(),
1065 };
1066 if self.capability_evaluator.record_hello(&cached_registration) {
1067 warn!(
1068 module_id = %cached_registration.module_id,
1069 "capability claims drifted from the cached manifest"
1070 );
1071 }
1072 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1073 self.enforce_capability_denies();
1074 }
1075 self.refresh_capability_requirements();
1076 }
1077
1078 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1082 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1083 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1084 self.supervisor.set_swap_promotion_observer(observer);
1085 }
1086
1087 pub fn refresh_capability_requirements(&self) {
1088 match self.runtime_capability_snapshot() {
1089 Ok((runtime, registrations)) => {
1090 log_requirement_events(
1091 self.capability_evaluator
1092 .evaluate_now(&runtime, ®istrations),
1093 );
1094 }
1095 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1096 }
1097 }
1098
1099 fn enforce_capability_denies(&self) {
1103 let (_, registrations) = match self.registry.list_modules() {
1104 Ok(snapshot) => snapshot,
1105 Err(err) => {
1106 warn!(error = %err, "failed to read registrations for capability deny census");
1107 return;
1108 }
1109 };
1110 let manifests = registrations
1111 .into_iter()
1112 .map(|registration| {
1113 (
1114 registration.manifest.module_id.clone(),
1115 registration.manifest,
1116 )
1117 })
1118 .collect::<BTreeMap<_, _>>();
1119 let census = match self.forwarding.route_census(None) {
1120 Ok(census) => census,
1121 Err(err) => {
1122 warn!(error = %err, "failed to read route census for capability deny enforcement");
1123 return;
1124 }
1125 };
1126
1127 for (target_module_id, routes) in census {
1128 let Some(target_manifest) = manifests.get(&target_module_id) else {
1129 continue;
1130 };
1131 let mut closed_routes = Vec::new();
1132 let mut module_goodbyes = Vec::new();
1133 for route in routes {
1134 let Principal::Reserved {
1135 module_id: opening_module_id,
1136 } = &route.principal
1137 else {
1138 continue;
1139 };
1140 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1141 continue;
1142 };
1143 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1144 continue;
1145 };
1146
1147 match self.forwarding.release_client_route(
1148 route.goodbye_target.connection_id,
1149 route.goodbye_target.channel,
1150 route.goodbye_target.epoch,
1151 ) {
1152 Ok(RouteRelease::Removed(module_goodbye)) => {
1153 warn!(
1154 opening_module_id,
1155 target_module_id,
1156 capability,
1157 "force-closing route because an attested capability deny edge now matches"
1158 );
1159 closed_routes.push(route);
1160 module_goodbyes.push(module_goodbye);
1161 }
1162 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1163 Err(err) => warn!(
1164 opening_module_id,
1165 target_module_id,
1166 capability,
1167 error = %err,
1168 "failed to force-close capability-denied route"
1169 ),
1170 }
1171 }
1172
1173 if closed_routes.is_empty() {
1174 continue;
1175 }
1176 send_route_control_pushes(
1177 &self.forwarding,
1178 closed_routes,
1179 ClientControlPush::RouteClosed {
1180 module_id: target_module_id,
1181 channels: Vec::new(),
1182 reason: RouteCloseReason::CapabilityDenied,
1183 drained: false,
1184 abandoned: 0,
1185 excluded_subscriptions: 0,
1186 terminal: Some(false),
1187 },
1188 );
1189 self.emit_route_goodbyes(module_goodbyes);
1190 }
1191 }
1192
1193 fn not_ready_reason(
1199 &self,
1200 registration: &crate::registry::ModuleRegistration,
1201 ) -> Option<NotReadyReason> {
1202 if !registration.ready {
1203 return Some(NotReadyReason {
1204 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1205 capability: None,
1206 });
1207 }
1208 self.first_unprovided_required_capability(registration)
1209 .map(|capability| NotReadyReason {
1210 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1211 capability: Some(capability),
1212 })
1213 }
1214
1215 fn first_unprovided_required_capability(
1233 &self,
1234 registration: &crate::registry::ModuleRegistration,
1235 ) -> Option<String> {
1236 let required = registration
1237 .manifest
1238 .capabilities
1239 .iter()
1240 .flat_map(|declarations| declarations.requires.iter())
1241 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1242 .map(|requirement| requirement.capability.as_str())
1243 .collect::<BTreeSet<_>>();
1244 if required.is_empty() {
1245 return None;
1246 }
1247 let module_id = registration.manifest.module_id.as_str();
1248 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1249 if required
1250 .iter()
1251 .any(|capability| verdict(capability).is_none())
1252 {
1253 self.refresh_capability_requirements();
1254 }
1255 required
1256 .into_iter()
1257 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1258 .map(str::to_string)
1259 }
1260
1261 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1262 self.capability_evaluator
1263 .statuses()
1264 .into_iter()
1265 .map(capability_requirement_status)
1266 .collect()
1267 }
1268
1269 fn deregister_connection(
1276 &self,
1277 connection_id: ConnectionId,
1278 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1279 self.registry.deregister_connection(connection_id)
1280 }
1281
1282 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1283 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1284 return None;
1285 }
1286 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1287 parse_client_control_request(&frame.body)
1288 else {
1289 return None;
1290 };
1291 Some(target_module_id(&target).to_string())
1292 }
1293
1294 pub(crate) fn route_open_capacity_refusal(
1295 &self,
1296 ctx: &RouteCtx,
1297 frame: &Frame,
1298 target_module_id: &str,
1299 in_flight: usize,
1300 limit: usize,
1301 ) -> Result<Frame, RouterError> {
1302 self.route_open_admission_refusal_frame(
1303 ctx,
1304 frame,
1305 target_module_id,
1306 "open_admission_full",
1307 (in_flight, limit),
1308 format!(
1309 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1310 ),
1311 )
1312 }
1313
1314 fn route_open_target_capacity_refusal(
1315 &self,
1316 ctx: &RouteCtx,
1317 frame: &Frame,
1318 target_module_id: &str,
1319 in_flight: usize,
1320 ) -> Result<Frame, RouterError> {
1321 self.route_open_admission_refusal_frame(
1322 ctx,
1323 frame,
1324 target_module_id,
1325 "target_binds_full",
1326 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1327 format!(
1328 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1329 ),
1330 )
1331 }
1332
1333 fn route_open_admission_refusal_frame(
1341 &self,
1342 ctx: &RouteCtx,
1343 frame: &Frame,
1344 target_module_id: &str,
1345 reason: &'static str,
1346 (in_flight, limit): (usize, usize),
1347 message: impl Into<String>,
1348 ) -> Result<Frame, RouterError> {
1349 let code = error_codes::TARGET_UNAVAILABLE;
1350 self.counters.increment_route_open_refused(code);
1351 info!(
1352 target: "control",
1353 code,
1354 reason,
1355 module_id = ?target_module_id,
1356 connection_id = ctx.connection_id.get(),
1357 in_flight,
1358 limit,
1359 "route.open refused"
1360 );
1361 control_error_frame(frame, code, message.into())
1362 }
1363
1364 #[cfg(test)]
1370 pub fn handle_control(
1371 &self,
1372 connection_id: ConnectionId,
1373 frame: Frame,
1374 ) -> Result<Vec<Frame>, RouterError> {
1375 match frame.header.ty {
1376 FrameType::Ping => Ok(vec![pong(&frame)?]),
1377 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1378 FrameType::Goodbye => self.handle_goodbye(connection_id),
1379 ty => Ok(vec![control_error_frame(
1380 &frame,
1381 "unsupported_control_frame",
1382 format!("unsupported channel-0 frame {ty:?}"),
1383 )?]),
1384 }
1385 }
1386
1387 pub async fn handle_control_frame(
1388 &self,
1389 ctx: &RouteCtx,
1390 frame: Frame,
1391 ) -> Result<Vec<Frame>, RouterError> {
1392 self.handle_control_frame_timed(ctx, frame, None).await
1393 }
1394
1395 pub(crate) async fn handle_control_frame_timed(
1396 &self,
1397 ctx: &RouteCtx,
1398 frame: Frame,
1399 dispatch_started_at: Option<StdInstant>,
1400 ) -> Result<Vec<Frame>, RouterError> {
1401 match frame.header.ty {
1402 FrameType::Ping => Ok(vec![pong(&frame)?]),
1403 FrameType::Hello => {
1404 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1405 }
1406 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1407 FrameType::Cancel => {
1408 if self
1411 .forwarding
1412 .operator_confirms()
1413 .cancel(ctx.connection_id, frame.header.corr)
1414 || self
1415 .supervisor
1416 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1417 {
1418 Ok(Vec::new())
1419 } else {
1420 Ok(vec![control_error_frame(
1421 &frame,
1422 "unknown_subscription",
1423 "no supervisor spawn subscription has this correlation id",
1424 )?])
1425 }
1426 }
1427 FrameType::Request => {
1428 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1431 if op
1432 .as_ref()
1433 .is_some_and(|probe| probe.op == "operator.confirm")
1434 {
1435 return self.handle_operator_confirm(ctx, frame);
1436 }
1437 if self
1438 .forwarding
1439 .module_endpoint_for_connection(ctx.connection_id)
1440 .map_err(RouterError::Forwarding)?
1441 .is_some()
1442 {
1443 if !is_known_module_request_op(&frame.body) {
1444 return Ok(vec![control_error_frame(
1445 &frame,
1446 "unsupported_control_frame",
1447 "module-originated channel-0 REQUEST is not supported",
1448 )?]);
1449 }
1450 let request = match parse_module_control_request_from_module(&frame.body) {
1451 Ok(request) => request,
1452 Err((err, ControlRequestBodyError::UnknownOp)) => {
1453 return Ok(vec![control_error_frame(
1454 &frame,
1455 "unsupported_control_frame",
1456 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1457 )?])
1458 }
1459 Err((err, ControlRequestBodyError::InvalidBody)) => {
1460 return Ok(vec![control_error_frame(
1461 &frame,
1462 "invalid_control_body",
1463 format!("malformed module control body: {err}"),
1464 )?])
1465 }
1466 };
1467 let op = module_control_request_op(&request);
1468 let corr = frame.header.corr;
1469 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1470 let result =
1471 self.handle_module_control_request(ctx.connection_id, frame, request);
1472 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1473 return result;
1474 }
1475
1476 if is_known_module_request_op(&frame.body) {
1477 return Ok(vec![control_error_frame(
1478 &frame,
1479 "not_registered",
1480 "catalog.update requires an active module registration owned by this connection",
1481 )?]);
1482 }
1483
1484 let request = match parse_client_control_request(&frame.body) {
1485 Ok(request) => request,
1486 Err((err, ControlRequestBodyError::UnknownOp)) => {
1487 return Ok(vec![control_error_frame(
1488 &frame,
1489 "unknown_control_op",
1490 format!("unknown client control op: {err}"),
1491 )?])
1492 }
1493 Err((err, ControlRequestBodyError::InvalidBody)) => {
1494 return Ok(vec![control_error_frame(
1495 &frame,
1496 "invalid_control_body",
1497 format!("malformed client control body: {err}"),
1498 )?])
1499 }
1500 };
1501 let op = client_control_request_op(&request);
1502 let corr = frame.header.corr;
1503 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1504 #[cfg(test)]
1505 if let Some(delay) = self.control_dispatch_delay {
1506 tokio::time::sleep(delay).await;
1507 }
1508 let result = self
1509 .handle_client_control_request(ctx, frame, request)
1510 .await;
1511 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1512 result
1513 }
1514 FrameType::Push => {
1515 let Some(endpoint) = self
1516 .forwarding
1517 .module_endpoint_for_connection(ctx.connection_id)
1518 .map_err(RouterError::Forwarding)?
1519 else {
1520 return Ok(vec![control_error_frame(
1521 &frame,
1522 "unsupported_control_frame",
1523 "client-originated channel-0 PUSH is not supported",
1524 )?]);
1525 };
1526 self.handle_status_update(endpoint, frame)
1527 }
1528 FrameType::Response | FrameType::Error
1529 if self
1530 .forwarding
1531 .module_endpoint_for_connection(ctx.connection_id)
1532 .map_err(RouterError::Forwarding)?
1533 .is_some() =>
1534 {
1535 self.handle_module_relay_response(ctx.connection_id, frame)
1536 }
1537 ty => Ok(vec![control_error_frame(
1538 &frame,
1539 "unsupported_control_frame",
1540 format!("unsupported channel-0 frame {ty:?}"),
1541 )?]),
1542 }
1543 }
1544
1545 pub fn cleanup_connection(
1546 &self,
1547 connection_id: ConnectionId,
1548 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1549 let crash_closed = self
1550 .registry
1551 .get_module_by_connection(connection_id)?
1552 .and_then(|registration| {
1553 self.forwarding
1554 .module_endpoint_for_connection(connection_id)
1555 .ok()
1556 .flatten()
1557 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1558 .map(|routes| (registration.manifest.module_id, routes))
1559 });
1560 let crash_closed = crash_closed.map(|(module_id, routes)| {
1561 let terminal = match self.supervisor.get(&module_id) {
1562 None => false,
1563 Some(module) => match module.will_recover_after_connection_loss() {
1564 Ok(will_recover) => !will_recover,
1565 Err(err) => {
1566 warn!(
1567 %module_id,
1568 error = %err,
1569 "failed to read crash recovery verdict; reporting non-terminal conservatively"
1570 );
1571 false
1572 }
1573 },
1574 };
1575 let reason = match self.forwarding.is_daemon_draining() {
1579 Ok(true) => RouteCloseReason::Restart,
1580 Ok(false) => RouteCloseReason::Crash,
1581 Err(err) => {
1582 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
1583 RouteCloseReason::Crash
1584 }
1585 };
1586 (module_id, routes, reason, terminal)
1587 });
1588 let registrations = self.deregister_connection(connection_id);
1589 let cleanup = if crash_closed.is_some() {
1590 self.forwarding.cleanup_connection_counted(connection_id)
1591 } else {
1592 self.forwarding
1598 .cleanup_connection(connection_id)
1599 .map(|released| crate::forwarding::ConnectionCleanup {
1600 released,
1601 abandoned_relays: 0,
1602 })
1603 };
1604 if let Some((module_id, routes, reason, terminal)) = crash_closed {
1609 let abandoned = cleanup
1610 .as_ref()
1611 .map_or(0, |cleanup| cleanup.abandoned_relays);
1612 send_route_control_pushes(
1613 &self.forwarding,
1614 routes,
1615 ClientControlPush::RouteClosed {
1616 module_id,
1617 channels: Vec::new(),
1618 reason,
1619 drained: false,
1620 abandoned,
1621 excluded_subscriptions: 0,
1622 terminal: Some(terminal),
1623 },
1624 );
1625 }
1626 if let Ok(cleanup) = cleanup {
1627 self.emit_route_goodbyes(cleanup.released);
1628 }
1629 if matches!(®istrations, Ok(r) if !r.is_empty()) {
1633 crate::supervise::notify_registration_release();
1634 self.capability_evaluator.wake_deadline_loop();
1635 self.refresh_capability_requirements();
1636 }
1637 self.supervisor.remove_spawn_subscribers(connection_id);
1638 self.hello_launch_nonces
1641 .lock()
1642 .unwrap_or_else(|poisoned| poisoned.into_inner())
1643 .forget(connection_id);
1644 self.scopes
1645 .write()
1646 .unwrap_or_else(|poisoned| poisoned.into_inner())
1647 .release_connection(connection_id);
1648 registrations
1649 }
1650
1651 pub(crate) fn handle_route_goodbye(
1652 &self,
1653 connection_id: ConnectionId,
1654 route_channel: u16,
1655 route_epoch: u32,
1656 ) -> Result<bool, RouterError> {
1657 debug!(
1658 connection_id = connection_id.get(),
1659 route_channel, route_epoch, "handling route GOODBYE"
1660 );
1661 let RouteRelease::Removed(released_route) = self
1662 .forwarding
1663 .release_client_route(connection_id, route_channel, route_epoch)
1664 .map_err(RouterError::Forwarding)?
1665 else {
1666 return Ok(false);
1667 };
1668 self.emit_route_goodbyes(vec![released_route]);
1669 Ok(true)
1670 }
1671
1672 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
1673 for released in released_routes {
1674 let frame = match Frame::build_with_version(
1675 released.negotiated_ver,
1676 FrameType::Goodbye,
1677 control_flags(),
1678 released.channel,
1679 released.epoch,
1680 0,
1681 Vec::new(),
1682 ) {
1683 Ok(frame) => frame,
1684 Err(err) => {
1685 warn!(
1686 route_channel = released.channel,
1687 error = %err,
1688 "failed to build route GOODBYE frame"
1689 );
1690 continue;
1691 }
1692 };
1693 if !released.close_on_delivery_failure() {
1694 crate::forwarding::send_module_route_goodbye(
1695 &self.counters,
1696 &released.sink,
1697 frame,
1698 released.module_id.as_deref(),
1699 "client route released",
1700 );
1701 continue;
1702 }
1703 if let Err(err) = released.sink.try_send(frame) {
1704 warn!(
1705 target_connection_id = released.connection_id.get(),
1706 route_channel = released.channel,
1707 error = %err,
1708 "route GOODBYE was not delivered to client; closing target connection"
1709 );
1710 if self
1711 .forwarding
1712 .escalate_client_delivery_failure(
1713 released.connection_id,
1714 released.channel,
1715 released.epoch,
1716 CloseReason::new(
1717 "route_goodbye_delivery_failed",
1718 format!(
1719 "failed to enqueue route GOODBYE for channel {}: {err}",
1720 released.channel
1721 ),
1722 ),
1723 crate::forwarding::UndeliveredFrame {
1724 module_id: released.module_id.as_deref(),
1725 sink: &released.sink,
1726 },
1727 )
1728 .unwrap_or(false)
1729 {
1730 self.counters.increment_goodbye_relay_client_failed();
1731 }
1732 }
1733 }
1734 }
1735
1736 fn send_abandoned_route_bind_goodbye(
1748 &self,
1749 module_sink: &crate::FrameSink,
1750 negotiated_ver: u8,
1751 module_channel: u16,
1752 module_epoch: u32,
1753 ) {
1754 let frame = match Frame::build_with_version(
1755 negotiated_ver,
1756 FrameType::Goodbye,
1757 control_flags(),
1758 module_channel,
1759 module_epoch,
1760 0,
1761 Vec::new(),
1762 ) {
1763 Ok(frame) => frame,
1764 Err(err) => {
1765 warn!(
1766 route_channel = module_channel,
1767 error = %err,
1768 "failed to build GOODBYE for abandoned route.bind"
1769 );
1770 return;
1771 }
1772 };
1773 crate::forwarding::send_module_route_goodbye(
1774 &self.counters,
1775 module_sink,
1776 frame,
1777 None,
1778 "abandoned route.bind",
1779 );
1780 }
1781
1782 fn handle_hello(
1783 &self,
1784 connection_id: ConnectionId,
1785 sink: Option<crate::FrameSink>,
1786 frame: Frame,
1787 ) -> Result<Vec<Frame>, RouterError> {
1788 debug!(
1789 connection_id = connection_id.get(),
1790 corr = frame.header.corr,
1791 "handling HELLO"
1792 );
1793 if self
1797 .registry
1798 .get_module_by_connection(connection_id)
1799 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
1800 .is_some()
1801 {
1802 return Ok(vec![control_error_frame(
1803 &frame,
1804 "invalid_hello",
1805 "connection is already registered as a module",
1806 )?]);
1807 }
1808 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
1809 Ok(value) => value,
1810 Err(err) => {
1811 return Ok(vec![control_error_frame(
1812 &frame,
1813 "invalid_hello",
1814 format!("malformed HELLO body: {err}"),
1815 )?])
1816 }
1817 };
1818 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
1819 return Ok(vec![control_error_frame(
1820 &frame,
1821 "invalid_capability_grammar",
1822 err.to_string(),
1823 )?]);
1824 }
1825 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
1826 return Ok(vec![control_error_frame(
1827 &frame,
1828 "invalid_manifest",
1829 err.to_string(),
1830 )?]);
1831 }
1832 if let Err(err) = validate_hello_event_declarations(&hello_value) {
1833 return Ok(vec![control_error_frame(
1834 &frame,
1835 "invalid_event_declaration",
1836 err.to_string(),
1837 )?]);
1838 }
1839 if let Some(provenance) = hello_value
1840 .get("manifest")
1841 .and_then(|manifest| manifest.get("provenance"))
1842 {
1843 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
1844 return Ok(vec![control_error_frame(
1845 &frame,
1846 "invalid_manifest",
1847 format!("malformed manifest provenance: {err}"),
1848 )?]);
1849 }
1850 }
1851 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
1852 Ok(hello) => hello,
1853 Err(err) => {
1854 return Ok(vec![control_error_frame(
1855 &frame,
1856 "invalid_hello",
1857 format!("malformed HELLO body: {err}"),
1858 )?])
1859 }
1860 };
1861
1862 if hello.protocol_ver != hello.manifest.protocol_ver {
1863 return Ok(vec![control_error_frame(
1864 &frame,
1865 "invalid_manifest",
1866 format!(
1867 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
1868 hello.protocol_ver, hello.manifest.protocol_ver
1869 ),
1870 )?]);
1871 }
1872
1873 if hello.manifest.module_id.trim().is_empty() {
1874 return Ok(vec![control_error_frame(
1875 &frame,
1876 "invalid_manifest",
1877 "manifest module_id must not be empty",
1878 )?]);
1879 }
1880
1881 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
1882 Ok(negotiated_ver) => negotiated_ver,
1883 Err(message) => {
1884 return Ok(vec![control_error_frame(
1885 &frame,
1886 "version_unsupported",
1887 message,
1888 )?])
1889 }
1890 };
1891
1892 let swap_admission = self
1900 .supervisor
1901 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
1902 if swap_admission == SwapHelloAdmission::Refused {
1903 warn!(
1904 module_id = %hello.manifest.module_id,
1905 connection_id = connection_id.get(),
1906 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
1907 );
1908 return Ok(vec![control_error_frame(
1909 &frame,
1910 "swap_token_invalid",
1911 format!(
1912 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
1913 hello.manifest.module_id
1914 ),
1915 )?]);
1916 }
1917 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
1918
1919 if let Some(rejection) = (!swap_candidate)
1927 .then(|| {
1928 self.supervisor.reserved_hello_rejection(
1929 &hello.manifest.module_id,
1930 hello.launch_nonce.as_deref(),
1931 )
1932 })
1933 .flatten()
1934 {
1935 let message = match rejection {
1936 ReservedHelloRejection::Exact { module_id } => format!(
1937 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
1938 ),
1939 ReservedHelloRejection::Prefix {
1940 prefix,
1941 owner_module_id,
1942 } => format!(
1943 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
1944 hello.manifest.module_id
1945 ),
1946 };
1947 return Ok(vec![control_error_frame(
1948 &frame,
1949 "reserved_module",
1950 message,
1951 )?]);
1952 }
1953
1954 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
1955 &hello.manifest.module_id,
1956 hello.manifest.capabilities.as_ref(),
1957 );
1958 if let Some(refusal) = reserved_capability_refusals.first() {
1959 let capability = refusal.capability.clone();
1960 let bound_module = refusal.claimants[0].clone();
1961 log_duplicate_claim_events(reserved_capability_refusals);
1962 return Ok(vec![control_error_frame(
1963 &frame,
1964 "reserved_capability",
1965 format!(
1966 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
1967 capability, bound_module, hello.manifest.module_id
1968 ),
1969 )?]);
1970 }
1971
1972 if self
1975 .forwarding
1976 .connection_has_client_routes(connection_id)
1977 .map_err(RouterError::Forwarding)?
1978 {
1979 return Ok(vec![control_error_frame(
1980 &frame,
1981 "invalid_hello",
1982 "connection has open client routes and cannot also register as a module",
1983 )?]);
1984 }
1985
1986 self.hello_launch_nonces
1991 .lock()
1992 .unwrap_or_else(|poisoned| poisoned.into_inner())
1993 .record(connection_id, hello.launch_nonce.as_deref());
1994 let control_ops = effective_module_control_ops(hello.control_ops);
1995 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
1998 if swap_candidate {
1999 return self.register_swap_candidate(
2000 connection_id,
2001 sink,
2002 &frame,
2003 hello.manifest,
2004 negotiated_ver,
2005 control_ops,
2006 hello_ack,
2007 );
2008 }
2009 let registration = match self.registry.register_with_control_ops(
2010 hello.manifest,
2011 negotiated_ver,
2012 connection_id,
2013 control_ops,
2014 ) {
2015 Ok(registration) => registration,
2016 Err(RegistryError::DuplicateModuleId { module_id }) => {
2017 return Ok(vec![control_error_frame(
2018 &frame,
2019 "duplicate_module_id",
2020 format!(
2021 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2022 ),
2023 )?])
2024 }
2025 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2026 return Ok(vec![control_error_frame(
2027 &frame,
2028 "invalid_module_id",
2029 err.to_string(),
2030 )?])
2031 }
2032 Err(err) => {
2033 return Ok(vec![control_error_frame(
2034 &frame,
2035 "registry_error",
2036 err.to_string(),
2037 )?])
2038 }
2039 };
2040
2041 let reply = if let Some(sink) = sink {
2042 let concurrency = manifest_concurrency(®istration.manifest);
2058 if let Err(err) = self.forwarding.register_module_connection_acked(
2059 connection_id,
2060 registration.manifest.module_id.clone(),
2061 negotiated_ver,
2062 concurrency,
2063 sink,
2064 hello_ack,
2065 ) {
2066 if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
2070 crate::supervise::notify_registration_release();
2071 }
2072 return Ok(vec![control_error_frame(
2073 &frame,
2074 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2075 "invalid_hello"
2076 } else {
2077 forwarding_error_code(&err)
2078 },
2079 err.to_string(),
2080 )?]);
2081 }
2082 Vec::new()
2083 } else {
2084 vec![hello_ack]
2087 };
2088
2089 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2098 info!(
2099 module_id = %registration.manifest.module_id,
2100 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2101 );
2102 }
2103
2104 self.apply_registration_capabilities(®istration);
2105
2106 info!(
2107 module_id = %registration.manifest.module_id,
2108 module_version = %registration.manifest.module_version,
2109 negotiated_ver,
2110 routable_provider = manifest_provides_routable_role(®istration.manifest),
2111 connection_id = connection_id.get(),
2112 "module registered"
2113 );
2114
2115 Ok(reply)
2116 }
2117
2118 #[allow(clippy::too_many_arguments)]
2128 fn register_swap_candidate(
2129 &self,
2130 connection_id: ConnectionId,
2131 sink: Option<crate::FrameSink>,
2132 frame: &Frame,
2133 manifest: ModuleManifest,
2134 negotiated_ver: u8,
2135 control_ops: Vec<String>,
2136 hello_ack: Frame,
2137 ) -> Result<Vec<Frame>, RouterError> {
2138 let module_id = manifest.module_id.clone();
2139 let registration = match self.registry.register_candidate_with_control_ops(
2140 manifest,
2141 negotiated_ver,
2142 connection_id,
2143 control_ops,
2144 ) {
2145 Ok(registration) => registration,
2146 Err(RegistryError::DuplicateModuleId { module_id }) => {
2147 return Ok(vec![control_error_frame(
2148 frame,
2149 "duplicate_module_id",
2150 format!(
2151 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2152 ),
2153 )?])
2154 }
2155 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2156 return Ok(vec![control_error_frame(
2157 frame,
2158 "invalid_module_id",
2159 err.to_string(),
2160 )?])
2161 }
2162 Err(err) => {
2163 return Ok(vec![control_error_frame(
2164 frame,
2165 "registry_error",
2166 err.to_string(),
2167 )?])
2168 }
2169 };
2170 let reply = if let Some(sink) = sink {
2171 let concurrency = manifest_concurrency(®istration.manifest);
2175 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2176 connection_id,
2177 module_id.clone(),
2178 negotiated_ver,
2179 concurrency,
2180 sink,
2181 hello_ack,
2182 ) {
2183 if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
2184 crate::supervise::notify_registration_release();
2185 }
2186 return Ok(vec![control_error_frame(
2187 frame,
2188 forwarding_error_code(&err),
2189 err.to_string(),
2190 )?]);
2191 }
2192 Vec::new()
2193 } else {
2194 vec![hello_ack]
2195 };
2196 self.supervisor.mark_swap_candidate_admitted(&module_id);
2197 info!(
2198 module_id = %module_id,
2199 module_version = %registration.manifest.module_version,
2200 negotiated_ver,
2201 ready = registration.ready,
2202 connection_id = connection_id.get(),
2203 "swap candidate registered; not routable until cutover"
2204 );
2205 Ok(reply)
2206 }
2207
2208 fn build_hello_ack(
2209 &self,
2210 frame: &Frame,
2211 negotiated_ver: u8,
2212 module_id: &str,
2213 ) -> Result<Frame, RouterError> {
2214 let ack = ModuleHelloAckBody {
2215 negotiated_ver,
2216 subc_ops: module_subc_ops(),
2217 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2218 storage: self
2219 .storage_config
2220 .as_ref()
2221 .map(|cfg| cfg.descriptor_for(module_id)),
2222 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2223 };
2224 let body = serde_json::to_vec(&ack).map_err(|err| {
2225 RouterError::backend(
2226 0,
2227 frame.header.corr,
2228 format!("failed to encode HELLO_ACK: {err}"),
2229 )
2230 })?;
2231
2232 Frame::build_with_version(
2233 negotiated_ver,
2234 FrameType::HelloAck,
2235 control_flags(),
2236 0,
2237 0,
2238 frame.header.corr,
2239 body,
2240 )
2241 .map_err(RouterError::FrameBuild)
2242 }
2243
2244 async fn handle_client_control_request(
2245 &self,
2246 ctx: &RouteCtx,
2247 frame: Frame,
2248 request: ClientControlRequest,
2249 ) -> Result<Vec<Frame>, RouterError> {
2250 match request {
2251 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2252 ClientControlRequest::CatalogList { module_id } => {
2253 self.handle_catalog_list(frame, module_id)
2254 }
2255 ClientControlRequest::RouteOpen {
2256 target,
2257 identity,
2258 consumer_identity,
2259 consumer_capabilities,
2260 role_versions,
2261 admission_facts,
2262 scope,
2263 } => {
2264 self.handle_route_open(
2265 ctx,
2266 frame,
2267 RouteOpenRequest {
2268 target,
2269 identity,
2270 consumer_identity,
2271 consumer_capabilities,
2272 role_versions,
2273 admission_facts,
2274 scope,
2275 },
2276 )
2277 .await
2278 }
2279 ClientControlRequest::RoutePoll {
2280 route_channel,
2281 route_epoch,
2282 kind,
2283 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2284 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2285 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2286 self.handle_supervisor_spawn_snapshot(frame)
2287 }
2288 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2289 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2290 }
2291 ClientControlRequest::SupervisorRestart {
2292 module_id,
2293 drain_timeout_ms,
2294 } => {
2295 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms)
2296 .await
2297 }
2298 ClientControlRequest::SupervisorSwap {
2299 module_id,
2300 ready_timeout_ms,
2301 } => {
2302 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms)
2303 .await
2304 }
2305 ClientControlRequest::SupervisorReload { module_id } => {
2306 self.handle_supervisor_reload(frame, module_id).await
2307 }
2308 ClientControlRequest::SupervisorRescan { preview } => {
2309 self.handle_supervisor_rescan(frame, preview).await
2310 }
2311 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2312 self.handle_supervisor_release_reserved(frame, module_id)
2313 .await
2314 }
2315 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2316 self.handle_supervisor_set_enabled(frame, module_id, enabled)
2317 .await
2318 }
2319 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2320 self.handle_supervisor_health_probe(frame, module_id).await
2321 }
2322 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2323 ClientControlRequest::SupervisorRoutes { module_id } => {
2324 self.handle_supervisor_routes(frame, module_id)
2325 }
2326 ClientControlRequest::SupervisorProvenance { module_id } => {
2327 self.handle_supervisor_provenance(frame, module_id).await
2328 }
2329 ClientControlRequest::SupervisorStderrTail {
2330 module_id,
2331 max_lines,
2332 max_bytes,
2333 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2334 ClientControlRequest::SupervisorTerminals { module_id } => {
2335 self.handle_supervisor_terminals(frame, module_id).await
2336 }
2337 }
2338 }
2339
2340 fn handle_module_control_request(
2341 &self,
2342 connection_id: ConnectionId,
2343 frame: Frame,
2344 request: ModuleControlRequestFromModule,
2345 ) -> Result<Vec<Frame>, RouterError> {
2346 match request {
2347 ModuleControlRequestFromModule::CatalogUpdate {
2348 provides,
2349 capabilities,
2350 ready,
2351 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2352 ModuleControlRequestFromModule::LiveRoots {} => {
2353 let registered = self
2354 .registry
2355 .get_module_by_connection(connection_id)
2356 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2357 let Some(registration) = registered else {
2358 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2359 };
2360 let response = self
2361 .forwarding
2362 .live_roots(®istration.manifest.module_id)
2363 .map_err(RouterError::Forwarding)?;
2364 Ok(vec![control_response_body_frame(
2365 &frame,
2366 &response,
2367 "ModuleControlResponseToModule::LiveRoots",
2368 )?])
2369 }
2370 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
2371 self.handle_scope_sync(connection_id, frame, generation, scopes)
2372 }
2373 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
2374 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
2375 }
2376 }
2377 }
2378
2379 fn handle_operator_confirm(
2380 &self,
2381 ctx: &RouteCtx,
2382 frame: Frame,
2383 ) -> Result<Vec<Frame>, RouterError> {
2384 use crate::operator_confirm::{audit, Outcome};
2385 let registration = self
2386 .registry
2387 .get_module_by_connection(ctx.connection_id)
2388 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2389 let Some(registration) = registration else {
2390 let outcome = Outcome::refusal("not_registered");
2391 audit("", "", "", outcome, Duration::ZERO, Duration::ZERO, false);
2392 return Ok(vec![outcome.frame(&frame)]);
2393 };
2394 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
2395 Ok(request) => request,
2396 Err(_) => {
2397 let outcome = Outcome::refusal("invalid_control_body");
2398 audit("", "", "", outcome, Duration::ZERO, Duration::ZERO, false);
2399 return Ok(vec![outcome.frame(&frame)]);
2400 }
2401 };
2402 let module_id = registration.manifest.module_id;
2403 let nonce = self
2407 .hello_launch_nonces
2408 .lock()
2409 .unwrap_or_else(|p| p.into_inner())
2410 .nonce(ctx.connection_id)
2411 .map(str::to_owned);
2412 let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
2413 self.supervisor
2414 .spawned_consumer_authorized(&module_id, nonce)
2415 });
2416 let confirms = self.forwarding.operator_confirms();
2417 self.forwarding
2418 .with_operator_route(
2419 ctx.connection_id,
2420 request.route_channel,
2421 request.route_epoch,
2422 |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
2423 )
2424 .map_err(RouterError::Forwarding)
2425 }
2426
2427 fn handle_scope_sync(
2431 &self,
2432 connection_id: ConnectionId,
2433 frame: Frame,
2434 generation: u64,
2435 scopes: Vec<ScopeRecord>,
2436 ) -> Result<Vec<Frame>, RouterError> {
2437 let Some(registration) = self
2438 .registry
2439 .get_module_by_connection(connection_id)
2440 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2441 else {
2442 return Ok(vec![control_error_frame(
2443 &frame,
2444 "not_registered",
2445 "scope.sync requires an active module registration owned by this connection",
2446 )?]);
2447 };
2448 let owner = registration.manifest.module_id;
2449 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
2450 let is_current_launch = |connection: ConnectionId| {
2451 self.hello_launch_nonces
2452 .lock()
2453 .unwrap_or_else(|poisoned| poisoned.into_inner())
2454 .presented(connection, current_nonce.as_deref())
2455 };
2456 let mut table = self
2461 .scopes
2462 .write()
2463 .unwrap_or_else(|poisoned| poisoned.into_inner());
2464 let outcome = table.sync(&owner, connection_id, is_current_launch, generation, scopes);
2465 let drained = match &outcome {
2466 Ok(applied) => self
2467 .forwarding
2468 .publish_scope_changes(&applied.tag_changes)
2469 .map_err(RouterError::Forwarding)?,
2470 Err(_) => Vec::new(),
2471 };
2472 drop(table);
2473 match outcome {
2474 Ok(applied) => {
2475 let counts = ScopeOutcomeCounts::of(&applied.results);
2476 info!(
2477 owner = %owner,
2478 generation,
2479 records = applied.results.len(),
2480 created = counts.created,
2481 replaced = counts.replaced,
2482 updated = counts.updated,
2483 unchanged = counts.unchanged,
2484 refused = counts.refused,
2485 ended = applied.ended.len(),
2486 tag_changes = applied.tag_changes.len(),
2487 routes_closed = drained.len(),
2488 "scope sync accepted"
2489 );
2490 for refused in applied
2496 .results
2497 .iter()
2498 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
2499 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
2500 {
2501 warn!(
2502 owner = %owner,
2503 generation,
2504 scope_ref = %refused.scope_ref,
2505 scope_epoch = refused.scope_epoch,
2506 code = refused.code.as_deref().unwrap_or(""),
2507 "scope record refused"
2508 );
2509 }
2510 self.close_scope_drained_routes(drained);
2511 let response = ModuleControlResponseToModule::ScopeSync {
2512 generation,
2513 results: applied.results,
2514 ended: applied.ended,
2515 };
2516 Ok(vec![control_response_body_frame(
2517 &frame,
2518 &response,
2519 "ModuleControlResponseToModule::ScopeSync",
2520 )?])
2521 }
2522 Err(refusal) => {
2523 info!(
2524 owner = %owner,
2525 generation,
2526 code = refusal.code,
2527 "scope sync refused"
2528 );
2529 Ok(vec![control_error_frame(
2530 &frame,
2531 refusal.code,
2532 refusal.message,
2533 )?])
2534 }
2535 }
2536 }
2537
2538 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
2544 if drained.is_empty() {
2545 return;
2546 }
2547 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
2548 BTreeMap::new();
2549 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
2550 for route in drained {
2551 warn!(
2552 module_id = %route.module_id,
2553 reason = ?route.reason,
2554 client_connection_id = route.client.connection_id.get(),
2555 route_channel = route.client.channel,
2556 "closing route because its scope changed"
2557 );
2558 pushes
2559 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
2560 .or_insert_with(|| (route.reason, Vec::new()))
2561 .1
2562 .push(EndpointRoute {
2563 goodbye_target: route.client.clone(),
2564 principal: Principal::Unverified,
2565 bound_at: Instant::now(),
2566 draining: false,
2567 drain_reason: None,
2568 });
2569 goodbyes.push(route.module);
2570 goodbyes.push(route.client);
2571 }
2572 for ((module_id, _), (reason, routes)) in pushes {
2573 send_route_control_pushes(
2574 &self.forwarding,
2575 routes,
2576 ClientControlPush::RouteClosed {
2577 module_id,
2578 channels: Vec::new(),
2579 reason,
2580 drained: false,
2581 abandoned: 0,
2582 excluded_subscriptions: 0,
2583 terminal: Some(false),
2584 },
2585 );
2586 }
2587 self.emit_route_goodbyes(goodbyes);
2588 }
2589
2590 fn handle_scope_describe(
2593 &self,
2594 connection_id: ConnectionId,
2595 frame: Frame,
2596 owner: Principal,
2597 scope_ref: String,
2598 ) -> Result<Vec<Frame>, RouterError> {
2599 let registered = self
2600 .registry
2601 .get_module_by_connection(connection_id)
2602 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2603 if registered.is_none() {
2604 return Ok(vec![control_error_frame(
2605 &frame,
2606 "not_registered",
2607 "scope.describe requires an active module registration owned by this connection",
2608 )?]);
2609 }
2610 let description = self
2611 .scopes
2612 .read()
2613 .unwrap_or_else(|poisoned| poisoned.into_inner())
2614 .describe(&owner, &scope_ref);
2615 let owner_configured = match &owner {
2616 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
2621 _ => false,
2622 };
2623 let response = ModuleControlResponseToModule::ScopeDescribe {
2624 status: description.status,
2625 scope_epoch: description.scope_epoch,
2626 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
2627 owner_synced: description.owner_synced,
2628 owner_configured,
2629 scope: description.stamp,
2630 };
2631 Ok(vec![control_response_body_frame(
2632 &frame,
2633 &response,
2634 "ModuleControlResponseToModule::ScopeDescribe",
2635 )?])
2636 }
2637
2638 fn handle_catalog_update(
2639 &self,
2640 connection_id: ConnectionId,
2641 frame: Frame,
2642 provides: Vec<ProviderRole>,
2643 capabilities: Option<CapabilityDeclarations>,
2644 ready: Option<bool>,
2645 ) -> Result<Vec<Frame>, RouterError> {
2646 self.refresh_capability_requirements();
2647 let Some(registration) = self
2648 .registry
2649 .get_module_by_connection(connection_id)
2650 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2651 else {
2652 return Ok(vec![control_error_frame(
2653 &frame,
2654 "not_registered",
2655 "catalog.update requires an active module registration owned by this connection",
2656 )?]);
2657 };
2658
2659 if let Some(message) =
2660 catalog_update_frozen_field_message(®istration.manifest, &provides)
2661 {
2662 return Ok(vec![control_error_frame(
2663 &frame,
2664 "catalog_update_frozen_field",
2665 message,
2666 )?]);
2667 }
2668
2669 let mut candidate = registration.manifest.clone();
2670 candidate.provides = provides.clone();
2671 candidate.capabilities = capabilities
2672 .clone()
2673 .or_else(|| registration.manifest.capabilities.clone());
2674 if let Err(err) = candidate.validate_capability_grammar() {
2675 return Ok(vec![control_error_frame(
2676 &frame,
2677 "invalid_capability_grammar",
2678 err.to_string(),
2679 )?]);
2680 }
2681
2682 let mut conflicts = self
2685 .capability_evaluator
2686 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
2687 if let Some(conflict) = conflicts.first() {
2688 let message = format!(
2689 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2690 conflict.capability, conflict.claimants[0], candidate.module_id
2691 );
2692 for conflict in &mut conflicts {
2693 conflict.source = DuplicateClaimSource::CatalogUpdate;
2694 }
2695 log_duplicate_claim_events(conflicts);
2696 return Ok(vec![control_error_frame(
2697 &frame,
2698 "reserved_capability",
2699 message,
2700 )?]);
2701 }
2702
2703 let updated = self
2704 .registry
2705 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
2706 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2707 if updated.is_none() {
2708 return Ok(vec![control_error_frame(
2709 &frame,
2710 "not_registered",
2711 "catalog.update requires an active module registration owned by this connection",
2712 )?]);
2713 }
2714 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
2715 log_duplicate_claim_events(
2716 self.capability_evaluator
2717 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
2718 );
2719 }
2720 if capability_census_trigger(
2721 registration.manifest.capabilities.as_ref(),
2722 updated
2723 .as_ref()
2724 .and_then(|entry| entry.manifest.capabilities.as_ref()),
2725 ) {
2726 self.enforce_capability_denies();
2727 }
2728 self.refresh_capability_requirements();
2729
2730 let response = ModuleControlResponseToModule::CatalogUpdate {};
2731 control_response_body_frame(
2732 &frame,
2733 &response,
2734 "ModuleControlResponseToModule::CatalogUpdate",
2735 )
2736 .map(|frame| vec![frame])
2737 }
2738
2739 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
2740 self.refresh_capability_requirements();
2741 let mut counters = self.counters.snapshot();
2747 if let (Ok((connections_with_routes, max)), Some(obj)) = (
2748 self.forwarding.client_route_concentration(),
2749 counters.as_object_mut(),
2750 ) {
2751 obj.insert(
2752 "client_connections_with_routes".into(),
2753 connections_with_routes.into(),
2754 );
2755 obj.insert("max_routes_on_one_connection".into(), max.into());
2756 }
2757 if let (Some(open_breakers), Some(obj)) = (
2763 self.route_bind_breakers.open_snapshot(),
2764 counters.as_object_mut(),
2765 ) {
2766 obj.insert("route_bind_breakers_open".into(), open_breakers);
2767 }
2768 let response = ClientControlResponse::ServerDescribe {
2769 protocol_ver: PROTOCOL_VERSION,
2770 subc_ops: subc_ops(),
2771 capabilities: self.subc_capabilities.as_ref().to_vec(),
2772 connected_clients: self.connected_clients.count(),
2773 counters: Some(counters),
2774 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
2775 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
2776 capability_requirements: self.capability_requirement_statuses(),
2777 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2778 };
2779 Ok(vec![control_response_body_frame(
2780 &frame,
2781 &response,
2782 "ClientControlResponse::ServerDescribe",
2783 )?])
2784 }
2785
2786 fn handle_catalog_list(
2787 &self,
2788 frame: Frame,
2789 module_id: Option<String>,
2790 ) -> Result<Vec<Frame>, RouterError> {
2791 let (generation, modules) = self.registry.list_modules().map_err(|err| {
2792 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
2793 })?;
2794 let entries = modules
2795 .into_iter()
2796 .filter(|registration| {
2797 module_id
2798 .as_deref()
2799 .map(|wanted| registration.manifest.module_id == wanted)
2800 .unwrap_or(true)
2801 })
2802 .map(|registration| {
2803 let not_ready = self.not_ready_reason(®istration);
2804 let roles = registration.manifest.provides;
2805 CatalogEntry {
2806 module_id: registration.manifest.module_id,
2807 ready: not_ready.is_none(),
2808 not_ready,
2809 module_version: Some(registration.manifest.module_version),
2810 roles,
2811 control_ops: registration.control_ops,
2812 capabilities: registration.manifest.capabilities,
2813 self_signals: registration.manifest.self_signals,
2814 }
2815 })
2816 .collect();
2817 let response = ClientControlResponse::CatalogList {
2818 generation,
2819 modules: entries,
2820 subc_ops: subc_ops(),
2821 };
2822 Ok(vec![control_response_body_frame(
2823 &frame,
2824 &response,
2825 "ClientControlResponse::CatalogList",
2826 )?])
2827 }
2828
2829 fn route_open_principal(
2830 &self,
2831 frame: &Frame,
2832 consumer_identity: Option<ConsumerIdentity>,
2833 ) -> Result<Result<Principal, Frame>, RouterError> {
2834 let Some(consumer_identity) = consumer_identity else {
2835 return Ok(Ok(Principal::Direct));
2836 };
2837
2838 if self.supervisor.spawned_consumer_authorized(
2839 &consumer_identity.module_id,
2840 &consumer_identity.launch_nonce,
2841 ) {
2842 return Ok(Ok(Principal::Reserved {
2843 module_id: consumer_identity.module_id,
2844 }));
2845 }
2846
2847 Ok(Err(control_error_frame(
2848 frame,
2849 "bad_consumer_identity",
2850 format!(
2851 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
2852 consumer_identity.module_id
2853 ),
2854 )?))
2855 }
2856
2857 fn route_open_refusal_frame(
2864 &self,
2865 ctx: &RouteCtx,
2866 frame: &Frame,
2867 module_id: &str,
2868 reason: &'static str,
2869 code: &'static str,
2870 message: impl Into<String>,
2871 ) -> Result<Frame, RouterError> {
2872 self.observe_route_open_refusal(ctx, module_id, reason, code);
2873 control_error_frame(frame, code, message.into())
2874 }
2875
2876 fn route_open_breaker_refusal_frame(
2894 &self,
2895 ctx: &RouteCtx,
2896 frame: &Frame,
2897 module_id: &str,
2898 consecutive_timeouts: u32,
2899 retry_in: Duration,
2900 probe_in_flight: bool,
2901 ) -> Result<Frame, RouterError> {
2902 self.counters
2903 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
2904 debug!(
2905 target: "control",
2906 code = "module_timeout",
2907 module_id = ?module_id,
2908 connection_id = ctx.connection_id.get(),
2909 consecutive_timeouts,
2910 retry_in_ms = retry_in.as_millis() as u64,
2911 probe_in_flight,
2912 "route.open refused by open bind-relay breaker"
2913 );
2914 let detail = if probe_in_flight {
2921 "one test route is already being tried; retry once it settles".to_string()
2922 } else {
2923 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
2924 };
2925 control_error_frame(
2926 frame,
2927 "module_timeout",
2928 format!(
2929 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
2930 timed out in a row); {detail}; its established routes are unaffected"
2931 ),
2932 )
2933 }
2934
2935 fn observe_route_open_refusal(
2948 &self,
2949 ctx: &RouteCtx,
2950 module_id: &str,
2951 reason: &'static str,
2952 code: &'static str,
2953 ) {
2954 self.counters.increment_route_open_refused(code);
2955 info!(
2956 target: "control",
2957 code,
2958 reason,
2959 module_id = ?module_id,
2960 connection_id = ctx.connection_id.get(),
2961 "route.open refused"
2962 );
2963 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
2964 self.route_outages.record_not_serving(module_id, reason);
2965 }
2966 }
2967
2968 fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
3013 self.route_outages.record_accepted(module_id);
3014 self.counters.increment_route_open_accepted(principal);
3015 info!(
3016 target: "control",
3017 principal,
3018 module_id,
3019 connection_id = ctx.connection_id.get(),
3020 "route.open accepted"
3021 );
3022 }
3023
3024 fn supervised_absent_route_open_refusal_frame(
3025 &self,
3026 ctx: &RouteCtx,
3027 frame: &Frame,
3028 module_id: &str,
3029 code: &'static str,
3030 status: &crate::supervise::ModuleStatus,
3031 ) -> Result<Frame, RouterError> {
3032 self.counters.increment_route_open_refused(code);
3033 info!(
3034 target: "control",
3035 code,
3036 reason = "supervised_not_registered",
3037 module_id = ?module_id,
3038 connection_id = ctx.connection_id.get(),
3039 state = %status.state,
3040 enabled = status.enabled,
3041 live = status.live,
3042 "route.open refused"
3043 );
3044 self.route_outages
3048 .record_not_serving(module_id, "supervised_not_registered");
3049 control_error_frame(
3050 frame,
3051 code,
3052 format!(
3053 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
3054 status.state, status.enabled, status.live
3055 ),
3056 )
3057 }
3058
3059 async fn handle_route_open(
3060 &self,
3061 ctx: &RouteCtx,
3062 frame: Frame,
3063 request: RouteOpenRequest,
3064 ) -> Result<Vec<Frame>, RouterError> {
3065 let RouteOpenRequest {
3066 target,
3067 mut identity,
3068 consumer_identity,
3069 consumer_capabilities,
3070 role_versions,
3071 admission_facts,
3072 scope,
3073 } = request;
3074 let target_module_id = target_module_id(&target).to_string();
3075 if self
3076 .registry
3077 .get_module_by_connection(ctx.connection_id)
3078 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3079 .is_some()
3080 {
3081 return Ok(vec![control_error_frame(
3082 &frame,
3083 "invalid_request",
3084 "module connections cannot open client routes",
3085 )?]);
3086 }
3087 debug!(
3088 connection_id = ctx.connection_id.get(),
3089 corr = frame.header.corr,
3090 module_id = %target_module_id,
3091 "handling route.open"
3092 );
3093
3094 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
3100 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
3101 self.observe_route_open_refusal(
3102 ctx,
3103 &target_module_id,
3104 "invalid_role_versions",
3105 error_codes::INVALID_REQUEST,
3106 );
3107 return Ok(vec![control_error_body_frame(
3108 &frame,
3109 ErrorBody {
3110 code: error_codes::INVALID_REQUEST.to_string(),
3111 message: error.to_string(),
3112 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
3113 },
3114 )?]);
3115 }
3116
3117 let Some(registration) = self
3136 .registry
3137 .get_module(&target_module_id)
3138 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3139 else {
3140 if let Some((status, warming)) =
3141 self.supervisor_status(&target_module_id, frame.header.corr)?
3142 {
3143 if status.protocol == ModuleProtocol::None {
3150 return Ok(vec![self.route_open_refusal_frame(
3151 ctx,
3152 &frame,
3153 &target_module_id,
3154 "protocol_none",
3155 error_codes::MODULE_NO_PROTOCOL,
3156 format!(
3157 "module_id '{target_module_id}' is declared protocol: none; \
3158 it speaks no subc wire and serves no routes"
3159 ),
3160 )?]);
3161 }
3162 let code = if warming {
3163 "module_warming"
3164 } else {
3165 "target_unavailable"
3166 };
3167 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
3168 ctx,
3169 &frame,
3170 &target_module_id,
3171 code,
3172 &status,
3173 )?]);
3174 }
3175 if let Some(removed_ago_ms) =
3176 self.supervisor.removal_tombstone_age_ms(&target_module_id)
3177 {
3178 return Ok(vec![self.route_open_refusal_frame(
3179 ctx,
3180 &frame,
3181 &target_module_id,
3182 "removed",
3183 error_codes::MODULE_REMOVED,
3184 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
3185 )?]);
3186 }
3187 return Ok(vec![self.route_open_refusal_frame(
3188 ctx,
3189 &frame,
3190 &target_module_id,
3191 "not_registered",
3192 error_codes::UNKNOWN_MODULE,
3193 format!("module_id '{target_module_id}' is not registered"),
3194 )?]);
3195 };
3196
3197 if !registration.ready {
3201 self.counters
3202 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
3203 info!(
3204 target: "control",
3205 code = error_codes::MODULE_WARMING,
3206 module_id = ?target_module_id,
3207 connection_id = ctx.connection_id.get(),
3208 reason = "declared_not_ready",
3209 "route.open refused"
3210 );
3211 self.route_outages
3214 .record_not_serving(&target_module_id, "declared_not_ready");
3215 return Ok(vec![control_error_body_frame(
3216 &frame,
3217 ErrorBody {
3218 code: error_codes::MODULE_WARMING.to_string(),
3219 message: format!(
3220 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
3221 ),
3222 detail: Some(serde_json::json!({
3223 "reason": "declared_not_ready"
3224 })),
3225 },
3226 )?]);
3227 }
3228
3229 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
3248 self.counters
3249 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
3250 info!(
3251 target: "control",
3252 code = error_codes::MODULE_WARMING,
3253 module_id = ?target_module_id,
3254 connection_id = ctx.connection_id.get(),
3255 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3256 capability = %capability,
3257 "route.open refused"
3258 );
3259 return Ok(vec![control_error_body_frame(
3260 &frame,
3261 ErrorBody {
3262 code: error_codes::MODULE_WARMING.to_string(),
3263 message: format!(
3264 "module_id '{target_module_id}' requires capability '{capability}', \
3265 which no registered module provides; retry"
3266 ),
3267 detail: Some(serde_json::json!({
3268 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3269 "capability": capability,
3270 })),
3271 },
3272 )?]);
3273 }
3274
3275 if !target_has_required_role(&target, ®istration.manifest.provides) {
3276 return Ok(vec![self.route_open_refusal_frame(
3277 ctx,
3278 &frame,
3279 &target_module_id,
3280 "role_not_provided",
3281 "target_unavailable",
3282 format!("module_id '{target_module_id}' does not provide the requested target"),
3283 )?]);
3284 }
3285
3286 if registration.state != ChannelState::Active {
3287 return Ok(vec![self.route_open_refusal_frame(
3288 ctx,
3289 &frame,
3290 &target_module_id,
3291 "registration_not_active",
3292 "target_unavailable",
3293 format!("module_id '{target_module_id}' is not active"),
3294 )?]);
3295 }
3296
3297 if self
3298 .forwarding
3299 .module_is_draining(&target_module_id)
3300 .map_err(RouterError::Forwarding)?
3301 {
3302 return Ok(vec![self.route_open_refusal_frame(
3303 ctx,
3304 &frame,
3305 &target_module_id,
3306 "reloading",
3307 "module_reloading",
3308 format!("module_id '{target_module_id}' is reloading"),
3309 )?]);
3310 }
3311
3312 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
3313 process_liveness.process_live(&target_module_id) == Some(false)
3314 }) {
3315 if process_liveness.process_replacing(&target_module_id) {
3322 return Ok(vec![self.route_open_refusal_frame(
3323 ctx,
3324 &frame,
3325 &target_module_id,
3326 "reloading",
3327 "module_reloading",
3328 format!("module_id '{target_module_id}' is reloading"),
3329 )?]);
3330 }
3331 return Ok(vec![self.route_open_refusal_frame(
3332 ctx,
3333 &frame,
3334 &target_module_id,
3335 "supervisor_not_live",
3336 "target_unavailable",
3337 format!("module_id '{target_module_id}' is not live"),
3338 )?]);
3339 }
3340
3341 if !self
3342 .forwarding
3343 .has_live_module_connection(&target_module_id)
3344 .map_err(RouterError::Forwarding)?
3345 {
3346 return Ok(vec![self.route_open_refusal_frame(
3347 ctx,
3348 &frame,
3349 &target_module_id,
3350 "no_forwarding_connection",
3351 "target_unavailable",
3352 format!("module_id '{target_module_id}' has no live forwarding connection"),
3353 )?]);
3354 }
3355
3356 if let Some(error) =
3357 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
3358 {
3359 self.observe_route_open_refusal(
3360 ctx,
3361 &target_module_id,
3362 "op_not_allowed",
3363 "op_not_allowed",
3364 );
3365 return Ok(vec![error]);
3366 }
3367
3368 let principal = match self.route_open_principal(&frame, consumer_identity)? {
3369 Ok(principal) => principal,
3370 Err(error) => {
3371 self.observe_route_open_refusal(
3372 ctx,
3373 &target_module_id,
3374 "bad_consumer_identity",
3375 "bad_consumer_identity",
3376 );
3377 return Ok(vec![error]);
3378 }
3379 };
3380
3381 if let Principal::Reserved {
3385 module_id: opening_module_id,
3386 } = &principal
3387 {
3388 if let Some(opening_registration) = self
3389 .registry
3390 .get_module(opening_module_id)
3391 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3392 {
3393 if let Some(capability) =
3394 denied_capability(&opening_registration.manifest, ®istration.manifest)
3395 {
3396 warn!(
3397 opening_module_id,
3398 target_module_id,
3399 capability,
3400 "refusing route.open because an attested capability deny edge matches"
3401 );
3402 return Ok(vec![self.route_open_refusal_frame(
3403 ctx,
3404 &frame,
3405 &target_module_id,
3406 "capability_deny_edge",
3407 "capability_forbidden",
3408 format!(
3409 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
3410 ),
3411 )?]);
3412 }
3413 }
3414 }
3415
3416 if admission_facts.is_some() {
3417 let carrier_matches = matches!(
3418 &principal,
3419 Principal::Reserved { module_id }
3420 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
3421 );
3422 if !carrier_matches {
3423 return Ok(vec![self.route_open_refusal_frame(
3424 ctx,
3425 &frame,
3426 &target_module_id,
3427 "admission_facts_carrier_not_permitted",
3428 "admission_facts_not_permitted",
3429 "admission facts may only be carried by the configured reserved module",
3430 )?]);
3431 }
3432
3433 let target_allowed = self
3434 .admission_facts_targets
3435 .as_ref()
3436 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
3437 if !target_allowed {
3438 return Ok(vec![self.route_open_refusal_frame(
3439 ctx,
3440 &frame,
3441 &target_module_id,
3442 "admission_facts_target_not_listed",
3443 "admission_facts_target_not_allowed",
3444 format!(
3445 "admission facts are not permitted for target module_id '{target_module_id}'"
3446 ),
3447 )?]);
3448 }
3449
3450 }
3454
3455 let (bound_scope, scope_stamp) = match scope {
3461 None => (None, None),
3462 Some(selector) => {
3463 let owner_configured = match &selector.owner {
3464 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3470 _ => false,
3471 };
3472 let admitted = self
3473 .scopes
3474 .read()
3475 .unwrap_or_else(|poisoned| poisoned.into_inner())
3476 .admit(&principal, &target_module_id, &selector, owner_configured)
3477 .and_then(|admission| {
3478 crate::scopes::check_target_flow_support(
3479 &admission.stamp,
3480 &target_module_id,
3481 registration.manifest.capabilities.as_ref(),
3482 )?;
3483 Ok(admission)
3484 });
3485 match admitted {
3486 Ok(admission) => (
3487 Some(BoundScope {
3488 owner: admission.owner,
3489 scope_ref: admission.stamp.scope_ref.clone(),
3490 tag: admission.tag,
3491 }),
3492 Some(admission.stamp),
3493 ),
3494 Err(refusal) => {
3495 return Ok(vec![self.route_open_refusal_frame(
3496 ctx,
3497 &frame,
3498 &target_module_id,
3499 refusal.code,
3500 refusal.code,
3501 refusal.message,
3502 )?]);
3503 }
3504 }
3505 }
3506 };
3507
3508 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
3530 Ok(project_root) => project_root,
3531 Err(err) => {
3532 return Ok(vec![control_error_frame(
3533 &frame,
3534 "invalid_project_root",
3535 err.to_string(),
3536 )?])
3537 }
3538 };
3539 identity.project_root = project_root.as_path().to_path_buf();
3540
3541 let _concurrency_guard = match self
3552 .route_bind_concurrency
3553 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
3554 {
3555 Ok(guard) => guard,
3556 Err(in_flight) => {
3557 return Ok(vec![self.route_open_target_capacity_refusal(
3558 ctx,
3559 &frame,
3560 &target_module_id,
3561 in_flight,
3562 )?]);
3563 }
3564 };
3565
3566 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
3569 RouteBindAdmission::Admitted { guard, probe } => {
3570 if probe {
3571 info!(
3572 module_id = %target_module_id,
3573 connection_id = ctx.connection_id.get(),
3574 "route.bind breaker half-open: admitting one probe"
3575 );
3576 }
3577 guard
3578 }
3579 RouteBindAdmission::Refused {
3580 consecutive_timeouts,
3581 retry_in,
3582 probe_in_flight,
3583 } => {
3584 return Ok(vec![self.route_open_breaker_refusal_frame(
3585 ctx,
3586 &frame,
3587 &target_module_id,
3588 consecutive_timeouts,
3589 retry_in,
3590 probe_in_flight,
3591 )?]);
3592 }
3593 };
3594
3595 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
3600 let relay_deadline = Instant::now() + route_bind_relay_timeout;
3601 let pending = match self
3602 .forwarding
3603 .begin_route_bind_relay_for(
3604 ctx.connection_id,
3605 ctx.egress.clone(),
3606 response_version(&frame),
3607 frame.header.corr,
3608 &target_module_id,
3609 principal.clone(),
3610 bound_scope,
3611 Some(project_root),
3612 relay_deadline,
3613 )
3614 .await
3615 {
3616 Ok(pending) => pending,
3617 Err(err) => {
3618 return Ok(vec![self.route_open_refusal_frame(
3619 ctx,
3620 &frame,
3621 &target_module_id,
3622 "relay_reservation_failed",
3623 forwarding_error_code(&err),
3624 err.to_string(),
3625 )?])
3626 }
3627 };
3628 let crate::forwarding::PendingRouteBindRelay {
3629 endpoint,
3630 module_sink,
3631 negotiated_ver,
3632 client_channel,
3633 client_epoch,
3634 module_channel,
3635 module_epoch,
3636 corr: relay_corr,
3637 receiver,
3638 } = pending;
3639 let mut reservation =
3640 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
3641
3642 if let Some(stamp) = scope_stamp
3647 .as_ref()
3648 .filter(|stamp| stamp.attributes.flow_id.is_some())
3649 {
3650 let relay_registration = self
3651 .registry
3652 .get_module_by_connection(endpoint.connection_id)
3653 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3654 if let Err(refusal) = crate::scopes::check_target_flow_support(
3655 stamp,
3656 &target_module_id,
3657 relay_registration
3658 .as_ref()
3659 .and_then(|registration| registration.manifest.capabilities.as_ref()),
3660 ) {
3661 reservation.release_and_disarm();
3662 return Ok(vec![self.route_open_refusal_frame(
3663 ctx,
3664 &frame,
3665 &target_module_id,
3666 refusal.code,
3667 refusal.code,
3668 refusal.message,
3669 )?]);
3670 }
3671 }
3672
3673 debug!(
3674 connection_id = ctx.connection_id.get(),
3675 client_channel,
3676 client_epoch,
3677 module_channel,
3678 module_epoch,
3679 "reserved route handle pair"
3680 );
3681 let principal_label = match &principal {
3684 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
3685 Principal::Direct => "direct".to_string(),
3686 other => format!("{other:?}"),
3687 };
3688 let relay = ModuleControlRequest::RouteBind {
3689 route_channel: module_channel,
3690 epoch: module_epoch,
3691 target,
3692 identity,
3693 principal: Some(principal),
3694 consumer_capabilities,
3695 role_versions,
3696 admission_facts,
3697 scope: scope_stamp,
3698 };
3699 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
3700 RouterError::backend(
3701 0,
3702 frame.header.corr,
3703 format!("failed to encode route.bind request: {err}"),
3704 )
3705 })?;
3706 let relay_frame = Frame::build_with_version(
3707 negotiated_ver,
3708 FrameType::Request,
3709 control_flags(),
3710 0,
3711 0,
3712 relay_corr,
3713 relay_body,
3714 )
3715 .map_err(RouterError::FrameBuild)?;
3716
3717 if let Err(err) = module_sink.send(relay_frame).await {
3718 reservation.release_and_disarm();
3719 return Ok(vec![self.route_open_refusal_frame(
3720 ctx,
3721 &frame,
3722 &target_module_id,
3723 "relay_send_failed",
3724 "target_unavailable",
3725 err.to_string(),
3726 )?]);
3727 }
3728
3729 if !self
3730 .forwarding
3731 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
3732 .map_err(RouterError::Forwarding)?
3733 {
3734 self.send_abandoned_route_bind_goodbye(
3735 &module_sink,
3736 negotiated_ver,
3737 module_channel,
3738 module_epoch,
3739 );
3740 }
3741
3742 match timeout_at(relay_deadline, receiver).await {
3743 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
3744 reservation.disarm();
3745 if breaker.record_accepted() {
3746 info!(
3747 module_id = %target_module_id,
3748 "route.bind breaker closed: the probe was accepted"
3749 );
3750 }
3751 self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
3752 Ok(Vec::new())
3753 }
3754 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
3755 reservation.release_and_disarm();
3756 breaker.record_inconclusive();
3760 let scope_code = match body.code.as_str() {
3764 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
3765 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
3766 _ => None,
3767 };
3768 if let Some(code) = scope_code {
3769 self.observe_route_open_refusal(
3770 ctx,
3771 &target_module_id,
3772 "scope_changed_before_commit",
3773 code,
3774 );
3775 return Ok(vec![control_error_body_frame(&frame, body)?]);
3776 }
3777 self.counters
3778 .increment_route_open_refused("module_rejected");
3779 info!(
3780 target: "control",
3781 code = "module_rejected",
3782 module_code = ?body.code,
3783 module_id = ?target_module_id,
3784 connection_id = ctx.connection_id.get(),
3785 "route.open refused"
3786 );
3787 Ok(vec![control_error_body_frame(&frame, body)?])
3788 }
3789 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
3790 reservation.release_and_disarm();
3791 breaker.record_inconclusive();
3792 tracing::warn!(
3799 module_id = %target_module_id,
3800 "route.bind relay abandoned: {message}"
3801 );
3802 Ok(vec![self.route_open_refusal_frame(
3803 ctx,
3804 &frame,
3805 &target_module_id,
3806 "relay_abandoned",
3807 "target_unavailable",
3808 message,
3809 )?])
3810 }
3811 Ok(Err(_)) => {
3812 reservation.release_and_disarm();
3813 breaker.record_inconclusive();
3814 Ok(vec![self.route_open_refusal_frame(
3815 ctx,
3816 &frame,
3817 &target_module_id,
3818 "relay_waiter_canceled",
3819 "target_unavailable",
3820 "route.bind relay waiter was canceled before the module responded",
3821 )?])
3822 }
3823 Err(_) => {
3824 reservation.release_and_disarm();
3825 if let Some(opened) = breaker.record_timeout(
3829 self.route_bind_breaker_threshold,
3830 self.route_bind_breaker_cooldown,
3831 ) {
3832 warn!(
3833 module_id = %target_module_id,
3834 consecutive_timeouts = opened.consecutive_timeouts,
3835 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
3836 reopened_after_probe = opened.reopened_after_probe,
3837 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
3838 );
3839 }
3840 tracing::warn!(
3848 module_id = %target_module_id,
3849 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
3850 "route.bind relay timed out: module did not ack within budget"
3851 );
3852 Ok(vec![self.route_open_refusal_frame(
3853 ctx,
3854 &frame,
3855 &target_module_id,
3856 "relay_timed_out",
3857 "module_timeout",
3858 format!(
3859 "module_id '{target_module_id}' did not answer route.bind within {:?}",
3860 route_bind_relay_timeout
3861 ),
3862 )?])
3863 }
3864 }
3865 }
3866
3867 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3868 let response = ClientControlResponse::SupervisorSpawnSnapshot {
3869 snapshot: self.supervisor.spawn_snapshot(),
3870 };
3871 Ok(vec![control_response_body_frame(
3872 &frame,
3873 &response,
3874 "ClientControlResponse::SupervisorSpawnSnapshot",
3875 )?])
3876 }
3877
3878 fn handle_supervisor_spawn_subscribe(
3879 &self,
3880 ctx: &RouteCtx,
3881 frame: Frame,
3882 since: Option<SpawnCursor>,
3883 ) -> Result<Vec<Frame>, RouterError> {
3884 match self.supervisor.subscribe_spawns(
3885 ctx.connection_id,
3886 frame.header.corr,
3887 response_version(&frame),
3888 since,
3889 ctx.egress.clone(),
3890 ) {
3891 Ok(()) => Ok(Vec::new()),
3892 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
3893 Ok(vec![control_error_body_frame(
3894 &frame,
3895 ErrorBody {
3896 code: "spawn_cursor_incarnation_mismatch".to_string(),
3897 message: "spawn cursor belongs to a different daemon incarnation"
3898 .to_string(),
3899 detail: Some(serde_json::json!({
3900 "current_daemon_incarnation": current
3901 })),
3902 },
3903 )?])
3904 }
3905 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
3906 &frame,
3907 ErrorBody {
3908 code: "spawn_cursor_too_old".to_string(),
3909 message: "spawn cursor predates the retained event ring".to_string(),
3910 detail: Some(serde_json::json!({
3911 "oldest_retained_cursor": oldest
3912 })),
3913 },
3914 )?]),
3915 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
3916 0,
3917 frame.header.corr,
3918 format!("failed to open supervisor spawn subscription: {error}"),
3919 )),
3920 }
3921 }
3922
3923 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3924 let generation = self
3925 .registry
3926 .generation()
3927 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3928 let mut modules = Vec::new();
3929 for module in self.supervisor.list() {
3930 let status = module.status_for_control("list").map_err(|err| {
3931 RouterError::backend(
3932 0,
3933 frame.header.corr,
3934 format!("failed to read supervisor status: {err}"),
3935 )
3936 })?;
3937 let (configured, _) = module.configuration().map_err(|err| {
3938 RouterError::backend(
3939 0,
3940 frame.header.corr,
3941 format!("failed to read module configuration: {err}"),
3942 )
3943 })?;
3944 let image = module.running_image_agreement().await;
3946 let resources = Some(module.child_resource_usage());
3949 let pending_reload = Some(reload_verdict(
3950 &configured.program,
3951 status.spawned_from.as_deref(),
3952 image,
3953 ));
3954 modules.push(SupervisorEntry {
3955 launch_nonce_env: Some(!cfg!(unix)),
3958 module_id: status.module_id,
3959 state: status.state.to_string(),
3960 enabled: status.enabled,
3961 live: status.live,
3962 protocol: status.protocol,
3963 health: status.health.status,
3964 pending_reload,
3965 last_probe_ms: status.health.last_probe_ms,
3966 last_exit_code: status.last_exit.as_ref().and_then(|e| e.code),
3967 last_exit_signal: status.last_exit.as_ref().and_then(|e| e.signal),
3968 last_exit_ms: status.last_exit.as_ref().map(|e| e.at_ms),
3969 last_exit_kind: status.last_exit.as_ref().map(|e| e.kind.into()),
3970 restart_count: Some(status.restart_count),
3971 max_restarts: Some(status.max_restarts),
3972 lifetime_restarts: Some(status.lifetime_restarts),
3973 spawn_generation: Some(status.spawn_generation),
3974 restart_window_secs: Some(status.restart_window.as_secs()),
3975 drain_timeout_ms: Some(status.drain_timeout.as_millis() as u64),
3976 restart_backoff_ms: Some(status.restart_backoff.as_millis() as u64),
3977 restart_max_backoff_ms: Some(status.restart_max_backoff.as_millis() as u64),
3978 resources,
3979 });
3980 }
3981 let response = ClientControlResponse::SupervisorList {
3982 generation,
3983 modules,
3984 };
3985 Ok(vec![control_response_body_frame(
3986 &frame,
3987 &response,
3988 "ClientControlResponse::SupervisorList",
3989 )?])
3990 }
3991
3992 fn handle_supervisor_stderr_tail(
3993 &self,
3994 frame: Frame,
3995 module_id: String,
3996 max_lines: Option<u32>,
3997 max_bytes: Option<u32>,
3998 ) -> Result<Vec<Frame>, RouterError> {
3999 let Some(module) = self.supervisor.get(&module_id) else {
4000 return Ok(vec![control_error_frame(
4001 &frame,
4002 "unknown_module",
4003 format!("module_id '{module_id}' is not supervised"),
4004 )?]);
4005 };
4006
4007 let snapshot = module.stderr_tail(
4008 max_lines.map(|value| value as usize),
4009 max_bytes.map(|value| value as usize),
4010 );
4011
4012 let response = ClientControlResponse::SupervisorStderrTail {
4013 module_id,
4014 tail: StderrTail {
4015 capture: match snapshot.capture {
4016 CaptureState::Captured => StderrCaptureState::Captured,
4017 CaptureState::Incomplete { reason } => {
4018 StderrCaptureState::Incomplete { reason }
4019 }
4020 CaptureState::NotCaptured { reason } => {
4021 StderrCaptureState::NotCaptured { reason }
4022 }
4023 },
4024 entries: snapshot
4025 .entries
4026 .into_iter()
4027 .map(|entry| match entry {
4028 TailEntry::Line {
4029 text,
4030 truncated,
4031 at_ms,
4032 } => StderrTailEntry::Line {
4033 text,
4034 truncated,
4035 at_ms,
4036 },
4037 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
4038 })
4039 .collect(),
4040 dropped_lines: snapshot.dropped_lines,
4041 },
4042 };
4043 Ok(vec![control_response_body_frame(
4044 &frame,
4045 &response,
4046 "ClientControlResponse::SupervisorStderrTail",
4047 )?])
4048 }
4049
4050 async fn handle_supervisor_terminals(
4051 &self,
4052 frame: Frame,
4053 module_id: String,
4054 ) -> Result<Vec<Frame>, RouterError> {
4055 let Some(module) = self.supervisor.get(&module_id) else {
4056 return Ok(vec![control_error_frame(
4057 &frame,
4058 "unknown_module",
4059 format!("module_id '{module_id}' is not supervised"),
4060 )?]);
4061 };
4062
4063 let terminals = module
4066 .read_durable_terminal_history()
4067 .await
4068 .map_err(|error| {
4069 RouterError::backend(
4070 0,
4071 frame.header.corr,
4072 format!("failed to read terminal history: {error}"),
4073 )
4074 })?;
4075 let response = ClientControlResponse::SupervisorTerminals {
4076 module_id,
4077 terminals,
4078 };
4079 Ok(vec![control_response_body_frame(
4080 &frame,
4081 &response,
4082 "ClientControlResponse::SupervisorTerminals",
4083 )?])
4084 }
4085
4086 fn handle_supervisor_routes(
4087 &self,
4088 frame: Frame,
4089 module_id: Option<String>,
4090 ) -> Result<Vec<Frame>, RouterError> {
4091 let modules = self
4092 .forwarding
4093 .route_census(module_id.as_deref())
4094 .map_err(RouterError::Forwarding)?
4095 .into_iter()
4096 .map(|(module_id, routes)| SupervisorRouteModule {
4097 module_id,
4098 routes: routes
4099 .into_iter()
4100 .map(|route| SupervisorRoute {
4101 consumer: match route.principal {
4102 Principal::Reserved { module_id } => {
4103 SupervisorRouteConsumer::Reserved { module_id }
4104 }
4105 Principal::Direct | Principal::Unverified => {
4106 SupervisorRouteConsumer::Direct {
4107 connection_id: route.goodbye_target.connection_id.get(),
4108 }
4109 }
4110 },
4111 age_ms: Instant::now()
4112 .saturating_duration_since(route.bound_at)
4113 .as_millis()
4114 .try_into()
4115 .unwrap_or(u64::MAX),
4116 draining: route.draining,
4117 drain_reason: route.drain_reason,
4118 })
4119 .collect(),
4120 })
4121 .collect();
4122 let response = ClientControlResponse::SupervisorRoutes { modules };
4123 Ok(vec![control_response_body_frame(
4124 &frame,
4125 &response,
4126 "ClientControlResponse::SupervisorRoutes",
4127 )?])
4128 }
4129
4130 async fn handle_supervisor_provenance(
4131 &self,
4132 frame: Frame,
4133 module_id: Option<String>,
4134 ) -> Result<Vec<Frame>, RouterError> {
4135 let mut selected = if let Some(module_id) = module_id {
4136 let Some(module) = self.supervisor.get(&module_id) else {
4137 return Ok(vec![control_error_frame(
4138 &frame,
4139 "unknown_module",
4140 format!("module_id '{module_id}' is not supervised"),
4141 )?]);
4142 };
4143 vec![module]
4144 } else {
4145 self.supervisor.list()
4146 };
4147
4148 let mut modules = Vec::with_capacity(selected.len());
4149 for module in selected.drain(..) {
4150 let (status, observed_image) = module
4151 .status_and_running_image_agreement()
4152 .await
4153 .map_err(|err| {
4154 RouterError::backend(
4155 0,
4156 frame.header.corr,
4157 format!("failed to read supervisor status: {err}"),
4158 )
4159 })?;
4160 let module_declared = self
4161 .registry
4162 .get_module(&status.module_id)
4163 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4164 .and_then(|registration| registration.manifest.provenance)
4165 .map(|build| ModuleDeclaredProvenance::Reported { build })
4166 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
4167 #[cfg(test)]
4168 let running_image = match &self.provenance_probe_override {
4169 Some(result) => result.clone(),
4170 None => observed_image,
4171 };
4172 #[cfg(not(test))]
4173 let running_image = observed_image;
4174 modules.push(SupervisorModuleProvenance {
4175 module_id: status.module_id,
4176 module_declared,
4177 daemon_observed: SupervisorObservedProcess {
4178 pid: status.pid,
4179 spawned_at_ms: status.spawned_at_ms,
4180 spawned_from: status.spawned_from,
4181 running_image,
4182 },
4183 });
4184 }
4185 let daemon = SupervisorDaemonProvenance {
4186 daemon_build: self.daemon_provenance.build.clone(),
4187 daemon_observed: DaemonObservedProcess {
4188 pid: self.daemon_provenance.pid,
4189 started_at_ms: self
4190 .daemon_provenance
4191 .start_clock
4192 .map(|clock| clock.started_at_ms())
4193 .or(self.daemon_provenance.started_at_ms),
4194 running_image: self
4195 .daemon_provenance
4196 .probe
4197 .observe(
4198 self.daemon_provenance.pid,
4199 self.daemon_provenance.executable_path.as_deref(),
4200 self.daemon_provenance.executable_identity,
4201 self.daemon_provenance.process_start_time,
4202 )
4203 .await,
4204 },
4205 };
4206 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
4207 Ok(vec![control_response_body_frame(
4208 &frame,
4209 &response,
4210 "ClientControlResponse::SupervisorProvenance",
4211 )?])
4212 }
4213
4214 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4215 self.refresh_capability_requirements();
4216 let generation = self
4217 .registry
4218 .generation()
4219 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4220 let modules = self
4221 .supervisor
4222 .list()
4223 .into_iter()
4224 .map(|module| {
4225 let status = module.status_for_control("health").map_err(|err| {
4226 RouterError::backend(
4227 0,
4228 frame.header.corr,
4229 format!("failed to read supervisor health: {err}"),
4230 )
4231 })?;
4232 let module_id = status.module_id;
4233 let capability_detail = self
4234 .capability_evaluator
4235 .required_problem_detail(&module_id);
4236 Ok(SupervisorHealthEntry {
4237 module_id,
4238 status: status.health.status,
4239 detail: append_capability_problem_detail(
4240 status.health.detail,
4241 capability_detail,
4242 ),
4243 metrics: status.health.metrics,
4244 consecutive_failures: status.health.consecutive_failures,
4245 late_answer_count: status.health.late_answer_count,
4246 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
4247 last_action: status.health.last_action,
4248 last_action_ms: status.health.last_action_ms,
4249 last_probe_ms: status.health.last_probe_ms,
4250 })
4251 })
4252 .collect::<Result<Vec<_>, RouterError>>()?;
4253 let response = ClientControlResponse::SupervisorHealth {
4254 generation,
4255 modules,
4256 };
4257 Ok(vec![control_response_body_frame(
4258 &frame,
4259 &response,
4260 "ClientControlResponse::SupervisorHealth",
4261 )?])
4262 }
4263
4264 async fn handle_supervisor_restart(
4265 &self,
4266 frame: Frame,
4267 module_id: String,
4268 drain_timeout_ms: Option<u64>,
4269 ) -> Result<Vec<Frame>, RouterError> {
4270 let operation_lock = self.supervisor.operation_lock();
4271 let _operation_guard = operation_lock.lock().await;
4272 let Some(module) = self.supervisor.get(&module_id) else {
4273 return Ok(vec![control_error_frame(
4274 &frame,
4275 "unknown_module",
4276 format!("module_id '{module_id}' is not supervised"),
4277 )?]);
4278 };
4279
4280 self.route_outages.mark_operator_action(&module_id);
4281 if let Err(err) = module.restart(drain_timeout_ms).await {
4282 self.route_outages
4283 .operator_action_ended_unrefused(&module_id);
4284 let (code, message) = match err {
4285 crate::supervise::SuperviseError::Disabled { .. } => {
4286 ("module_disabled", err.to_string())
4287 }
4288 crate::supervise::SuperviseError::SwapInProgress { .. } => {
4289 ("swap_in_progress", err.to_string())
4290 }
4291 _ => (
4292 "target_unavailable",
4293 format!("failed to restart module_id '{module_id}': {err}"),
4294 ),
4295 };
4296 return Ok(vec![control_error_frame(&frame, code, message)?]);
4297 }
4298
4299 let response = ClientControlResponse::SupervisorAck {
4300 module_id,
4301 applied: true,
4302 };
4303 Ok(vec![control_response_body_frame(
4304 &frame,
4305 &response,
4306 "ClientControlResponse::SupervisorAck",
4307 )?])
4308 }
4309
4310 async fn handle_supervisor_swap(
4314 &self,
4315 frame: Frame,
4316 module_id: String,
4317 ready_timeout_ms: Option<u64>,
4318 ) -> Result<Vec<Frame>, RouterError> {
4319 let module = {
4326 let operation_lock = self.supervisor.operation_lock();
4327 let _operation_guard = operation_lock.lock().await;
4328 self.supervisor.get(&module_id)
4329 };
4330 let Some(module) = module else {
4331 return Ok(vec![control_error_frame(
4332 &frame,
4333 "unknown_module",
4334 format!("module_id '{module_id}' is not supervised"),
4335 )?]);
4336 };
4337
4338 self.route_outages.mark_operator_action(&module_id);
4339 if let Err(err) = module
4340 .swap(ready_timeout_ms.map(Duration::from_millis))
4341 .await
4342 {
4343 self.route_outages
4344 .operator_action_ended_unrefused(&module_id);
4345 use crate::supervise::SuperviseError;
4346 let message = err.to_string();
4347 let error = match err {
4348 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
4349 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
4350 code: "swap_refused".to_string(),
4351 message,
4352 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
4353 },
4354 SuperviseError::SwapFailed {
4355 arm,
4356 candidate_exit,
4357 ..
4358 } => ErrorBody {
4359 code: "swap_failed".to_string(),
4360 message,
4361 detail: Some(serde_json::json!({
4362 "arm": arm.as_str(),
4363 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
4364 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
4365 })),
4366 },
4367 _ => ErrorBody::new(
4368 "target_unavailable",
4369 format!("failed to swap module_id '{module_id}': {message}"),
4370 ),
4371 };
4372 return Ok(vec![control_error_body_frame(&frame, error)?]);
4373 }
4374 self.route_outages
4378 .operator_action_ended_unrefused(&module_id);
4379
4380 let response = ClientControlResponse::SupervisorAck {
4381 module_id,
4382 applied: true,
4383 };
4384 Ok(vec![control_response_body_frame(
4385 &frame,
4386 &response,
4387 "ClientControlResponse::SupervisorAck",
4388 )?])
4389 }
4390
4391 async fn handle_supervisor_reload(
4392 &self,
4393 frame: Frame,
4394 module_id: String,
4395 ) -> Result<Vec<Frame>, RouterError> {
4396 let operation_lock = self.supervisor.operation_lock();
4397 let _operation_guard = operation_lock.lock().await;
4398 let Some(module) = self.supervisor.get(&module_id) else {
4399 return Ok(vec![control_error_frame(
4400 &frame,
4401 "unknown_module",
4402 format!("module_id '{module_id}' is not supervised"),
4403 )?]);
4404 };
4405
4406 self.route_outages.mark_operator_action(&module_id);
4407 if let Err(err) = module.reload().await {
4408 self.route_outages
4409 .operator_action_ended_unrefused(&module_id);
4410 let (code, message) = match err {
4411 crate::supervise::SuperviseError::Disabled { .. } => {
4412 ("module_disabled", err.to_string())
4413 }
4414 crate::supervise::SuperviseError::SwapInProgress { .. } => {
4415 ("swap_in_progress", err.to_string())
4416 }
4417 _ => (
4418 "reload_failed",
4419 format!("failed to reload module_id '{module_id}': {err}"),
4420 ),
4421 };
4422 return Ok(vec![control_error_frame(&frame, code, message)?]);
4423 }
4424
4425 let response = ClientControlResponse::SupervisorAck {
4426 module_id,
4427 applied: true,
4428 };
4429 Ok(vec![control_response_body_frame(
4430 &frame,
4431 &response,
4432 "ClientControlResponse::SupervisorAck",
4433 )?])
4434 }
4435
4436 async fn handle_supervisor_rescan(
4437 &self,
4438 frame: Frame,
4439 preview: bool,
4440 ) -> Result<Vec<Frame>, RouterError> {
4441 let Some(context) = self.rescan.clone() else {
4442 return Ok(vec![control_error_frame(
4443 &frame,
4444 "rescan_unavailable",
4445 "the daemon was not started with a reloadable config path".to_string(),
4446 )?]);
4447 };
4448
4449 let operation_lock = self.supervisor.operation_lock();
4450 let _operation_guard = operation_lock.lock().await;
4451 let loaded = match crate::daemon_config::load(&context.config_path) {
4452 Ok(config) => config,
4453 Err(err) => {
4454 return Ok(vec![control_error_frame(
4455 &frame,
4456 "invalid_daemon_config",
4457 format!("supervisor rescan rejected daemon config: {err}"),
4458 )?])
4459 }
4460 };
4461 let Some(config) = loaded else {
4469 return Ok(vec![control_error_frame(
4470 &frame,
4471 "invalid_daemon_config",
4472 format!(
4473 "daemon config not found at {}; refusing to rescan (an absent config would \
4474 retire every supervised module)",
4475 context.config_path.display()
4476 ),
4477 )?]);
4478 };
4479 let (
4480 configured_port,
4481 storage_config,
4482 admission_facts_carrier_module_id,
4483 admission_facts_targets,
4484 scope_authority_owners,
4485 modules,
4486 reserved_capabilities,
4487 ) = (
4488 config.port,
4489 config.storage,
4490 config.admission_facts_carrier_module_id,
4491 config.admission_facts_targets,
4492 config.scope_authority_owners,
4493 config.modules,
4494 config.reserved_capabilities,
4495 );
4496
4497 let mut restart_required = Vec::new();
4506 for section in RestartRequiredSection::ALL {
4507 let changed = match section {
4508 RestartRequiredSection::Port => configured_port != context.configured_port,
4509 RestartRequiredSection::Storage => storage_config != context.storage_config,
4510 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
4511 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
4512 }
4513 RestartRequiredSection::AdmissionFactsTargets => {
4514 admission_facts_targets != context.admission_facts_targets
4515 }
4516 RestartRequiredSection::ScopeAuthorityOwners => {
4517 scope_authority_owners != context.scope_authority_owners
4518 }
4519 };
4520 if changed {
4521 restart_required.push(section.label().to_string());
4522 }
4523 }
4524 if !restart_required.is_empty() {
4525 warn!(
4526 config_path = %context.config_path.display(),
4527 sections = %restart_required.join(", "),
4528 "daemon config changed outside the modules section; restart the daemon to apply those changes"
4529 );
4530 }
4531
4532 for configured in &modules {
4533 if let Err(err) = validate_spec(&configured.module_spec()) {
4534 return Ok(vec![control_error_frame(
4535 &frame,
4536 "invalid_daemon_config",
4537 format!("supervisor rescan rejected daemon config: {err}"),
4538 )?]);
4539 }
4540 }
4541
4542 let configured_capabilities = modules
4543 .iter()
4544 .map(|module| (module.module_id.clone(), module.enabled))
4545 .collect::<Vec<_>>();
4546 let preview_capability_warnings = if preview {
4547 let (_, registrations) = self.runtime_capability_snapshot()?;
4548 let current_modules = self
4549 .supervisor
4550 .list()
4551 .into_iter()
4552 .map(|module| module.module_id().to_string())
4553 .collect::<BTreeSet<_>>();
4554 let resulting_modules = configured_capabilities.clone();
4555 let removed = current_modules
4556 .into_iter()
4557 .filter(|module_id| {
4558 !resulting_modules
4559 .iter()
4560 .any(|(configured_id, _)| configured_id == module_id)
4561 })
4562 .collect::<Vec<_>>();
4563 self.capability_evaluator.preview_removal_warnings(
4564 resulting_modules,
4565 &removed,
4566 ®istrations,
4567 )
4568 } else {
4569 Vec::new()
4570 };
4571 let result = match self
4572 .reconcile_supervised_modules(&context.supervisor, modules, preview)
4573 .await
4574 {
4575 Ok(result) => result,
4576 Err(message) => {
4577 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
4578 }
4579 };
4580 if !preview {
4581 self.capability_evaluator
4582 .configure(configured_capabilities, reserved_capabilities);
4583 self.capability_evaluator.wake_deadline_loop();
4584 self.refresh_capability_requirements();
4585 }
4586 let mut result = result;
4587 result.restart_required = restart_required;
4588 result.capability_warnings = preview_capability_warnings;
4589 let response = ClientControlResponse::SupervisorRescan { result };
4590 Ok(vec![control_response_body_frame(
4591 &frame,
4592 &response,
4593 "ClientControlResponse::SupervisorRescan",
4594 )?])
4595 }
4596
4597 async fn handle_supervisor_release_reserved(
4598 &self,
4599 frame: Frame,
4600 module_id: String,
4601 ) -> Result<Vec<Frame>, RouterError> {
4602 let Some(context) = self.rescan.clone() else {
4603 return Ok(vec![control_error_frame(
4604 &frame,
4605 "release_unavailable",
4606 "reserved-id release requires a daemon started with a reloadable config path",
4607 )?]);
4608 };
4609 let operation_lock = self.supervisor.operation_lock();
4610 let _operation_guard = operation_lock.lock().await;
4611 let loaded = match crate::daemon_config::load(&context.config_path) {
4612 Ok(Some(config)) => config,
4613 Ok(None) => {
4614 return Ok(vec![control_error_frame(
4615 &frame,
4616 "invalid_daemon_config",
4617 format!(
4618 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
4619 context.config_path.display()
4620 ),
4621 )?])
4622 }
4623 Err(err) => {
4624 return Ok(vec![control_error_frame(
4625 &frame,
4626 "invalid_daemon_config",
4627 format!("unable to verify reserved-id release against daemon config: {err}"),
4628 )?])
4629 }
4630 };
4631 if loaded
4632 .modules
4633 .iter()
4634 .any(|configured| configured.module_id == module_id)
4635 {
4636 return Ok(vec![control_error_frame(
4637 &frame,
4638 "reserved_module_configured",
4639 format!(
4640 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
4641 ),
4642 )?]);
4643 }
4644 if !self.supervisor.release_retained_reserved_gate(&module_id) {
4645 return Ok(vec![control_error_frame(
4646 &frame,
4647 "reserved_gate_not_retained",
4648 format!(
4649 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
4650 ),
4651 )?]);
4652 }
4653
4654 let response = ClientControlResponse::SupervisorAck {
4655 module_id,
4656 applied: true,
4657 };
4658 Ok(vec![control_response_body_frame(
4659 &frame,
4660 &response,
4661 "ClientControlResponse::SupervisorAck",
4662 )?])
4663 }
4664
4665 async fn reconcile_supervised_modules(
4674 &self,
4675 supervisor: &Supervisor,
4676 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
4677 preview: bool,
4678 ) -> Result<SupervisorRescanResult, String> {
4679 let mut current = BTreeMap::new();
4680 for module in self.supervisor.list() {
4681 let (spec, health) = module.configuration().map_err(|err| {
4682 format!(
4683 "failed to read configuration for module_id '{}': {err}",
4684 module.module_id()
4685 )
4686 })?;
4687 let enabled = module
4688 .status()
4689 .map_err(|err| {
4690 format!(
4691 "failed to read status for module_id '{}': {err}",
4692 module.module_id()
4693 )
4694 })?
4695 .enabled;
4696 current.insert(
4697 module.module_id().to_string(),
4698 (module, spec, health, enabled),
4699 );
4700 }
4701 let configured = configured_modules
4702 .into_iter()
4703 .map(|module| (module.module_id.clone(), module))
4704 .collect::<BTreeMap<_, _>>();
4705
4706 let added = configured
4707 .keys()
4708 .filter(|module_id| !current.contains_key(*module_id))
4709 .cloned()
4710 .collect::<Vec<_>>();
4711 let removed = current
4712 .keys()
4713 .filter(|module_id| !configured.contains_key(*module_id))
4714 .cloned()
4715 .collect::<Vec<_>>();
4716 let mut changed_pending_reload = Vec::new();
4717 let mut configuration_changes = BTreeSet::new();
4718 let mut enabled_changes = BTreeSet::new();
4719 let mut unchanged = 0_u32;
4720
4721 for (module_id, configured_module) in &configured {
4722 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
4723 else {
4724 continue;
4725 };
4726 let launch_changed = *current_spec != configured_module.module_spec();
4730 let configuration_changed =
4731 launch_changed || *current_health != configured_module.health;
4732 let enabled_changed = *current_enabled != configured_module.enabled;
4733 if configuration_changed {
4734 configuration_changes.insert(module_id.clone());
4735 }
4736 if launch_changed {
4737 changed_pending_reload.push(module_id.clone());
4738 }
4739 if enabled_changed {
4740 enabled_changes.insert(module_id.clone());
4741 }
4742 if !configuration_changed && !enabled_changed {
4743 unchanged = unchanged.saturating_add(1);
4744 }
4745 }
4746
4747 if preview {
4753 return Ok(SupervisorRescanResult {
4754 added,
4755 removed,
4756 changed_pending_reload,
4757 enabled_changes: enabled_changes.iter().cloned().collect(),
4758 unchanged,
4759 preview: true,
4760 restart_required: Vec::new(),
4764 capability_warnings: Vec::new(),
4765 });
4766 }
4767
4768 for module_id in &removed {
4769 let module = ¤t
4770 .get(module_id)
4771 .expect("removed module came from current supervisor state")
4772 .0;
4773 module.retire().await.map_err(|err| {
4774 format!("failed to retire module_id '{module_id}' during rescan: {err}")
4775 })?;
4776 self.supervisor.record_rescan_removal(module_id);
4810 self.supervisor.retire(module_id);
4811 self.route_outages.forget(module_id);
4812 }
4813
4814 for module_id in configured.keys() {
4815 let Some((module, _, _, _)) = current.get(module_id) else {
4816 continue;
4817 };
4818 let configured_module = configured
4819 .get(module_id)
4820 .expect("configured module id came from configured map");
4821 if configuration_changes.contains(module_id) {
4822 module
4823 .update_configuration(
4824 configured_module.module_spec(),
4825 configured_module.health.clone(),
4826 configured_module.drain_timeout_ms,
4827 )
4828 .await
4829 .map_err(|err| {
4830 format!(
4831 "failed to update module_id '{module_id}' configuration during rescan: {err}"
4832 )
4833 })?;
4834 }
4835 if enabled_changes.contains(module_id) {
4836 self.route_outages.mark_operator_action(module_id);
4839 module
4840 .set_enabled(configured_module.enabled)
4841 .await
4842 .map_err(|err| {
4843 self.route_outages.operator_action_ended_unrefused(module_id);
4844 format!(
4845 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
4846 configured_module.enabled
4847 )
4848 })?;
4849 }
4850 }
4851
4852 for module_id in &added {
4853 let configured_module = configured
4854 .get(module_id)
4855 .expect("added module id came from configured map");
4856 supervisor
4857 .supervise_configured_with_health(
4858 configured_module.module_spec(),
4859 configured_module.enabled,
4860 configured_module.health.clone(),
4861 configured_module.drain_timeout_ms,
4862 configured_module.restart,
4863 )
4864 .map_err(|err| {
4865 format!("failed to add module_id '{module_id}' during rescan: {err}")
4866 })?;
4867 }
4868
4869 Ok(SupervisorRescanResult {
4870 added,
4871 removed,
4872 changed_pending_reload,
4873 enabled_changes: enabled_changes.iter().cloned().collect(),
4874 unchanged,
4875 preview: false,
4876 restart_required: Vec::new(),
4879 capability_warnings: Vec::new(),
4880 })
4881 }
4882
4883 async fn handle_supervisor_set_enabled(
4884 &self,
4885 frame: Frame,
4886 module_id: String,
4887 enabled: bool,
4888 ) -> Result<Vec<Frame>, RouterError> {
4889 let operation_lock = self.supervisor.operation_lock();
4890 let _operation_guard = operation_lock.lock().await;
4891 let Some(module) = self.supervisor.get(&module_id) else {
4892 return Ok(vec![control_error_frame(
4893 &frame,
4894 "unknown_module",
4895 format!("module_id '{module_id}' is not supervised"),
4896 )?]);
4897 };
4898
4899 self.route_outages.mark_operator_action(&module_id);
4902 let applied = match module.set_enabled(enabled).await {
4903 Ok(applied) => applied,
4904 Err(err) => {
4905 self.route_outages
4906 .operator_action_ended_unrefused(&module_id);
4907 return Ok(vec![control_error_frame(
4908 &frame,
4909 "target_unavailable",
4910 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
4911 )?]);
4912 }
4913 };
4914 if !applied {
4915 self.route_outages
4918 .operator_action_ended_unrefused(&module_id);
4919 }
4920
4921 self.capability_evaluator.wake_deadline_loop();
4922 self.refresh_capability_requirements();
4923 let response = ClientControlResponse::SupervisorAck { module_id, applied };
4924 Ok(vec![control_response_body_frame(
4925 &frame,
4926 &response,
4927 "ClientControlResponse::SupervisorAck",
4928 )?])
4929 }
4930
4931 async fn handle_supervisor_health_probe(
4932 &self,
4933 frame: Frame,
4934 module_id: String,
4935 ) -> Result<Vec<Frame>, RouterError> {
4936 self.refresh_capability_requirements();
4937 let Some(registration) = self
4938 .registry
4939 .get_module(&module_id)
4940 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4941 else {
4942 return Ok(vec![control_error_frame(
4943 &frame,
4944 "unknown_module",
4945 format!("module_id '{module_id}' is not registered"),
4946 )?]);
4947 };
4948
4949 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
4962 {
4963 return Ok(vec![control_error_frame(
4964 &frame,
4965 "health_not_advertised",
4966 format!("module_id '{module_id}' did not advertise health.check"),
4967 )?]);
4968 }
4969
4970 let deadline = Instant::now() + self.health_probe_timeout;
4971 let pending = match self.forwarding.begin_module_control_rpc_for(
4972 &module_id,
4973 MODULE_CONTROL_OP_HEALTH_CHECK,
4974 deadline,
4975 ) {
4976 Ok(pending) => pending,
4977 Err(err) => {
4978 return Ok(vec![control_error_frame(
4979 &frame,
4980 forwarding_error_code(&err),
4981 err.to_string(),
4982 )?])
4983 }
4984 };
4985
4986 let PendingModuleControlRpc {
4987 endpoint,
4988 module_sink,
4989 negotiated_ver,
4990 corr: probe_corr,
4991 receiver,
4992 } = pending;
4993 let mut guard =
4994 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
4995 let probe_body =
4996 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
4997 RouterError::backend(
4998 0,
4999 frame.header.corr,
5000 format!("failed to encode health.check request: {err}"),
5001 )
5002 })?;
5003 let probe_frame = Frame::build_with_version(
5004 negotiated_ver,
5005 FrameType::Request,
5006 control_flags(),
5007 0,
5008 0,
5009 probe_corr,
5010 probe_body,
5011 )
5012 .map_err(RouterError::FrameBuild)?;
5013
5014 if let Err(err) = module_sink.send(probe_frame).await {
5015 return Ok(vec![control_error_frame(
5016 &frame,
5017 "target_unavailable",
5018 err.to_string(),
5019 )?]);
5020 }
5021
5022 match timeout_at(deadline, receiver).await {
5023 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
5024 guard.disarm();
5025 let Some(report) = response.health_report() else {
5026 return Ok(vec![control_error_frame(
5027 &frame,
5028 "invalid_control_body",
5029 "health.check RPC returned a non-health response",
5030 )?]);
5031 };
5032 let HealthReport {
5037 status,
5038 detail,
5039 metrics,
5040 } = report;
5041 let capability_detail = self
5042 .capability_evaluator
5043 .required_problem_detail(&module_id);
5044 let response = ClientControlResponse::SupervisorHealthProbe {
5045 module_id,
5046 status,
5047 detail: append_capability_problem_detail(detail, capability_detail),
5048 metrics,
5049 };
5050 Ok(vec![control_response_body_frame(
5051 &frame,
5052 &response,
5053 "ClientControlResponse::SupervisorHealthProbe",
5054 )?])
5055 }
5056 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
5057 guard.disarm();
5058 Ok(vec![control_error_body_frame(&frame, body)?])
5059 }
5060 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5061 guard.disarm();
5062 Ok(vec![control_error_frame(
5063 &frame,
5064 "target_unavailable",
5065 message,
5066 )?])
5067 }
5068 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5069 guard.disarm();
5070 Ok(vec![control_error_frame(
5071 &frame,
5072 "invalid_control_body",
5073 message,
5074 )?])
5075 }
5076 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5077 guard.disarm();
5078 Ok(vec![control_error_frame(
5079 &frame,
5080 "invalid_control_body",
5081 format!("expected module-control op '{expected}', got '{actual}'"),
5082 )?])
5083 }
5084 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
5085 guard.disarm();
5086 Ok(vec![control_error_frame(
5087 &frame,
5088 "module_timeout",
5089 format!(
5090 "module_id '{module_id}' answered health.check after {:?}",
5091 self.health_probe_timeout
5092 ),
5093 )?])
5094 }
5095 Ok(Err(_)) => Ok(vec![control_error_frame(
5096 &frame,
5097 "target_unavailable",
5098 "health.check waiter was canceled before the module responded",
5099 )?]),
5100 Err(_) => Ok(vec![control_error_frame(
5101 &frame,
5102 "module_timeout",
5103 format!(
5104 "module_id '{module_id}' did not answer health.check within {:?}",
5105 self.health_probe_timeout
5106 ),
5107 )?]),
5108 }
5109 }
5110
5111 fn supervisor_status(
5112 &self,
5113 module_id: &str,
5114 corr: u64,
5115 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
5116 self.supervisor
5117 .get(module_id)
5118 .map(|module| {
5119 let warming = module.is_warming_for_control("status").map_err(|err| {
5120 RouterError::backend(
5121 0,
5122 corr,
5123 format!(
5124 "failed to read supervisor warming state for module_id '{module_id}': {err}"
5125 ),
5126 )
5127 })?;
5128 module.status_for_control("status").map_err(|err| {
5129 RouterError::backend(
5130 0,
5131 corr,
5132 format!(
5133 "failed to read supervisor status for module_id '{module_id}': {err}"
5134 ),
5135 )
5136 }).map(|status| (status, warming))
5137 })
5138 .transpose()
5139 }
5140
5141 fn guard_module_control_op(
5142 &self,
5143 frame: &Frame,
5144 module_id: &str,
5145 op: &str,
5146 ) -> Result<Option<Frame>, RouterError> {
5147 if self.module_grants_op(module_id, op, frame.header.corr)? {
5148 return Ok(None);
5149 }
5150
5151 Ok(Some(control_error_frame(
5152 frame,
5153 "op_not_allowed",
5154 format!("module_id '{module_id}' did not grant control op '{op}'"),
5155 )?))
5156 }
5157
5158 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
5159 let Some(registration) = self
5160 .registry
5161 .get_module(module_id)
5162 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
5163 else {
5164 return Ok(false);
5165 };
5166 Ok(module_registration_grants_op(®istration.control_ops, op))
5167 }
5168
5169 fn handle_status_update(
5170 &self,
5171 endpoint: ModuleEndpointId,
5172 frame: Frame,
5173 ) -> Result<Vec<Frame>, RouterError> {
5174 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
5175 Ok(update) => update,
5176 Err(err) => {
5177 if is_known_module_push_op(&frame.body) {
5182 return Ok(vec![control_error_frame(
5183 &frame,
5184 "invalid_control_body",
5185 format!("malformed module control push body: {err}"),
5186 )?]);
5187 }
5188 return Ok(Vec::new());
5189 }
5190 };
5191
5192 match update {
5193 ModuleControlPush::RouteStatus {
5194 route_channel,
5195 route_epoch,
5196 status,
5197 } => {
5198 self.forwarding
5199 .cache_status(endpoint, route_channel, route_epoch, status)
5200 .map_err(RouterError::Forwarding)?;
5201 }
5202 }
5203 Ok(Vec::new())
5204 }
5205
5206 fn handle_route_poll(
5207 &self,
5208 ctx: &RouteCtx,
5209 frame: Frame,
5210 route_channel: u16,
5211 route_epoch: u32,
5212 kind: PollKind,
5213 ) -> Result<Vec<Frame>, RouterError> {
5214 let snapshot = self
5215 .forwarding
5216 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
5217 .map_err(RouterError::Forwarding)?;
5218 let response = match (kind, snapshot) {
5219 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
5220 ClientControlResponse::RoutePoll {
5221 route_channel,
5222 route_epoch,
5223 status,
5224 live: None,
5225 }
5226 }
5227 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
5228 route_channel,
5229 route_epoch,
5230 status: None,
5231 live: None,
5232 },
5233 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
5234 let live = self
5260 .process_liveness
5261 .as_ref()
5262 .and_then(|source| source.process_live(&module_id))
5263 .unwrap_or(true);
5264 ClientControlResponse::RoutePoll {
5265 route_channel,
5266 route_epoch,
5267 status: None,
5268 live: Some(live),
5269 }
5270 }
5271 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
5272 route_channel,
5273 route_epoch,
5274 status: None,
5275 live: Some(false),
5276 },
5277 };
5278
5279 Ok(vec![control_response_body_frame(
5280 &frame,
5281 &response,
5282 "ClientControlResponse::RoutePoll",
5283 )?])
5284 }
5285
5286 pub(crate) fn observe_module_control_completion(
5287 &self,
5288 completion: ModuleControlRpcCompletion,
5289 ) -> bool {
5290 match completion {
5291 ModuleControlRpcCompletion::Unknown => false,
5292 ModuleControlRpcCompletion::Settled => true,
5293 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
5294 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
5295 info!(
5296 module_id = %module_id,
5297 latency_ms,
5298 "late health.check answer proves the module is alive"
5299 );
5300 match self
5301 .supervisor
5302 .record_late_health_answer(&module_id, latency_ms)
5303 {
5304 Ok(true) => {}
5305 Ok(false) => debug!(
5306 module_id = %module_id,
5307 latency_ms,
5308 "late health.check answer has no active supervisor snapshot"
5309 ),
5310 Err(err) => warn!(
5311 module_id = %module_id,
5312 latency_ms,
5313 error = %err,
5314 "failed to record late health.check answer"
5315 ),
5316 }
5317 true
5318 }
5319 }
5320 }
5321
5322 fn refuse_to_end_module_connection_for_a_client(
5343 &self,
5344 module_connection_id: ConnectionId,
5345 corr: u64,
5346 err: ForwardingError,
5347 ) -> Result<(), RouterError> {
5348 if let ForwardingError::ConnectionClosing { connection_id } = err {
5349 if connection_id != module_connection_id {
5350 warn!(
5351 module_connection_id = module_connection_id.get(),
5352 client_connection_id = connection_id.get(),
5353 corr,
5354 "dropping a route.bind response for a closing client; the module connection keeps serving"
5355 );
5356 return Ok(());
5357 }
5358 }
5359 Err(RouterError::Forwarding(err))
5360 }
5361
5362 fn handle_module_relay_response(
5363 &self,
5364 connection_id: ConnectionId,
5365 frame: Frame,
5366 ) -> Result<Vec<Frame>, RouterError> {
5367 let mut secondary_error = None;
5368 let outcome = match frame.header.ty {
5369 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
5370 Ok(probe) if probe.op == "route.bind" => {
5371 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
5372 Ok(ModuleControlResponse::RouteBindAck {}) => {
5373 RouteBindRelayOutcome::Accepted
5374 }
5375 Ok(other) => {
5376 let message =
5377 format!("route.bind response carried unexpected body: {other:?}");
5378 secondary_error = Some(control_error_frame(
5379 &frame,
5380 "invalid_control_body",
5381 message.clone(),
5382 )?);
5383 RouteBindRelayOutcome::ModuleGone(message)
5384 }
5385 Err(err) => {
5386 let message = format!("malformed route.bind response body: {err}");
5387 secondary_error = Some(control_error_frame(
5388 &frame,
5389 "invalid_control_body",
5390 message.clone(),
5391 )?);
5392 RouteBindRelayOutcome::ModuleGone(message)
5393 }
5394 }
5395 }
5396 Ok(probe) => {
5397 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
5398 {
5399 Ok(response) => ModuleControlRpcOutcome::Response(response),
5400 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
5401 "malformed {} response body: {err}",
5402 probe.op
5403 )),
5404 };
5405 let completion = self
5406 .forwarding
5407 .complete_module_control_rpc(
5408 connection_id,
5409 frame.header.corr,
5410 Some(&probe.op),
5411 outcome,
5412 )
5413 .map_err(RouterError::Forwarding)?;
5414 if !self.observe_module_control_completion(completion) {
5415 debug!(
5416 connection_id = connection_id.get(),
5417 corr = frame.header.corr,
5418 op = %probe.op,
5419 "dropping late or unknown module-control RPC response"
5420 );
5421 }
5422 return Ok(Vec::new());
5423 }
5424 Err(err) => {
5425 if let Some(expected_op) = self
5426 .forwarding
5427 .pending_module_control_op(connection_id, frame.header.corr)
5428 .map_err(RouterError::Forwarding)?
5429 {
5430 let completion = self
5431 .forwarding
5432 .complete_module_control_rpc(
5433 connection_id,
5434 frame.header.corr,
5435 None,
5436 ModuleControlRpcOutcome::MalformedResponse(format!(
5437 "malformed {expected_op} response body: {err}"
5438 )),
5439 )
5440 .map_err(RouterError::Forwarding)?;
5441 if !self.observe_module_control_completion(completion) {
5442 debug!(
5443 connection_id = connection_id.get(),
5444 corr = frame.header.corr,
5445 "dropping late malformed module-control RPC response"
5446 );
5447 }
5448 return Ok(Vec::new());
5449 }
5450 let message = format!("malformed route.bind response body: {err}");
5451 secondary_error = Some(control_error_frame(
5452 &frame,
5453 "invalid_control_body",
5454 message.clone(),
5455 )?);
5456 RouteBindRelayOutcome::ModuleGone(message)
5457 }
5458 },
5459 FrameType::Error => {
5460 if self
5461 .forwarding
5462 .pending_module_control_op(connection_id, frame.header.corr)
5463 .map_err(RouterError::Forwarding)?
5464 .is_some()
5465 {
5466 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
5467 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
5468 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
5469 "malformed module-control ERROR body: {err}"
5470 )),
5471 };
5472 let completion = self
5473 .forwarding
5474 .complete_module_control_rpc(
5475 connection_id,
5476 frame.header.corr,
5477 None,
5478 outcome,
5479 )
5480 .map_err(RouterError::Forwarding)?;
5481 if !self.observe_module_control_completion(completion) {
5482 debug!(
5483 connection_id = connection_id.get(),
5484 corr = frame.header.corr,
5485 "dropping late or unknown module-control RPC error"
5486 );
5487 }
5488 return Ok(Vec::new());
5489 }
5490 match serde_json::from_slice::<ErrorBody>(&frame.body) {
5491 Ok(body) => RouteBindRelayOutcome::Rejected(body),
5492 Err(err) => {
5493 let message = format!("malformed route.bind ERROR body: {err}");
5494 secondary_error = Some(control_error_frame(
5495 &frame,
5496 "invalid_control_body",
5497 message.clone(),
5498 )?);
5499 RouteBindRelayOutcome::ModuleGone(message)
5500 }
5501 }
5502 }
5503 ty => {
5504 return Ok(vec![control_error_frame(
5505 &frame,
5506 "unsupported_control_frame",
5507 format!("unsupported module channel-0 frame {ty:?}"),
5508 )?])
5509 }
5510 };
5511
5512 let settled =
5513 self.forwarding
5514 .complete_pending_relay(connection_id, frame.header.corr, outcome);
5515 let completion = match settled {
5516 Ok(completion) => completion,
5517 Err(err) => {
5518 self.refuse_to_end_module_connection_for_a_client(
5519 connection_id,
5520 frame.header.corr,
5521 err,
5522 )?;
5523 return Ok(secondary_error.into_iter().collect());
5524 }
5525 };
5526 if let Some(target) = completion.abandoned.as_ref() {
5527 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
5528 }
5529 if !completion.settled {
5530 debug!(
5531 connection_id = connection_id.get(),
5532 corr = frame.header.corr,
5533 frame_type = ?frame.header.ty,
5534 "dropping late or unknown route.bind relay response"
5535 );
5536 }
5537 Ok(secondary_error.into_iter().collect())
5538 }
5539
5540 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
5541 debug!(connection_id = connection_id.get(), "handling GOODBYE");
5542 self.cleanup_connection(connection_id)
5546 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
5547 Ok(Vec::new())
5548 }
5549}
5550
5551impl Default for ControlHandler {
5552 fn default() -> Self {
5553 Self::new(Arc::new(Registry::default()))
5554 }
5555}
5556
5557impl crate::supervise::SwapPromotionObserver for ControlHandler {
5558 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
5559 self.apply_registration_capabilities(registration);
5560 }
5561}
5562
5563fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
5564 CapabilityRequirementStatus {
5565 consumer: status.consumer,
5566 capability: status.capability,
5567 need: match status.need {
5568 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
5569 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
5570 },
5571 verdict: status.verdict.as_str().to_string(),
5572 episode_seq: status.episode_seq,
5573 config_satisfiable: status.config_satisfiable,
5574 runtime_available: status.runtime_available,
5575 detail: status.detail,
5576 }
5577}
5578
5579fn append_capability_problem_detail(
5580 detail: Option<String>,
5581 capability_detail: Option<String>,
5582) -> Option<String> {
5583 match (detail, capability_detail) {
5584 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
5585 (Some(detail), None) => Some(detail),
5586 (None, Some(capability_detail)) => Some(capability_detail),
5587 (None, None) => None,
5588 }
5589}
5590
5591fn subc_ops() -> Vec<String> {
5592 SUBC_CONTROL_OPS
5593 .iter()
5594 .map(|op| (*op).to_string())
5595 .collect()
5596}
5597
5598fn module_subc_ops() -> Vec<String> {
5599 SUBC_CONTROL_OPS
5600 .iter()
5601 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
5602 .map(|op| (*op).to_string())
5603 .collect()
5604}
5605
5606#[cfg(test)]
5607fn module_baseline_control_ops() -> Vec<String> {
5608 MODULE_BASELINE_CONTROL_OPS
5609 .iter()
5610 .map(|op| (*op).to_string())
5611 .collect()
5612}
5613
5614fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
5615 let mut seen = HashSet::new();
5616 let mut effective = Vec::new();
5617 for op in MODULE_BASELINE_CONTROL_OPS {
5618 if seen.insert((*op).to_string()) {
5619 effective.push((*op).to_string());
5620 }
5621 }
5622 for op in declared.unwrap_or_default() {
5623 if seen.insert(op.clone()) {
5624 effective.push(op);
5625 }
5626 }
5627 effective
5628}
5629
5630fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
5631 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
5632}
5633
5634fn target_module_id(target: &RouteTarget) -> &str {
5635 match target {
5636 RouteTarget::ToolProvider { module_id }
5637 | RouteTarget::ManagementSurface { module_id }
5638 | RouteTarget::InternalService { module_id, .. } => module_id,
5639 }
5640}
5641
5642fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
5643 roles.iter().any(|role| match (target, role) {
5644 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
5645 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
5646 (
5647 RouteTarget::InternalService { service_id, .. },
5648 ProviderRole::InternalService {
5649 service_id: provided,
5650 ..
5651 },
5652 ) => service_id == provided,
5653 _ => false,
5654 })
5655}
5656
5657fn is_routable_role(role: &ProviderRole) -> bool {
5658 matches!(
5659 role,
5660 ProviderRole::ToolProvider { .. }
5661 | ProviderRole::ManagementSurface { .. }
5662 | ProviderRole::InternalService { .. }
5663 )
5664}
5665
5666#[derive(Debug, Clone, Copy, PartialEq, Eq)]
5667enum ControlRequestBodyError {
5668 UnknownOp,
5669 InvalidBody,
5670}
5671
5672#[derive(Debug, Deserialize)]
5673struct ControlOpProbe {
5674 op: String,
5675}
5676
5677const MODULE_PUSH_OPS: &[&str] = &["route.status"];
5680
5681fn is_known_module_push_op(body: &[u8]) -> bool {
5682 serde_json::from_slice::<ControlOpProbe>(body)
5683 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
5684 .unwrap_or(false)
5685}
5686
5687fn is_known_module_request_op(body: &[u8]) -> bool {
5688 serde_json::from_slice::<ControlOpProbe>(body)
5689 .map(|probe| is_module_to_subc_op(&probe.op))
5690 .unwrap_or(false)
5691}
5692
5693fn is_module_to_subc_op(op: &str) -> bool {
5694 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
5695}
5696
5697fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
5698 debug!(
5699 op = %op,
5700 connection_id = connection_id.get(),
5701 corr,
5702 "control dispatch"
5703 );
5704}
5705
5706fn log_slow_control_dispatch(
5707 dispatch_started_at: Option<StdInstant>,
5708 op: &'static str,
5709 connection_id: ConnectionId,
5710 corr: u64,
5711) {
5712 let Some(dispatch_started_at) = dispatch_started_at else {
5713 return;
5714 };
5715 let elapsed = dispatch_started_at.elapsed();
5716 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
5717 warn!(
5718 op = %op,
5719 connection_id = connection_id.get(),
5720 corr,
5721 elapsed_ms = elapsed.as_millis() as u64,
5722 "slow control dispatch"
5723 );
5724 }
5725}
5726
5727fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
5728 match request {
5729 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
5730 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
5731 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
5732 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
5733 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
5734 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
5735 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
5736 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
5737 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
5738 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
5739 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
5740 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
5741 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
5742 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
5743 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
5744 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
5745 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
5746 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
5747 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
5748 }
5749}
5750
5751fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
5752 match request {
5753 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
5754 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
5755 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
5756 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
5757 }
5758}
5759
5760fn parse_client_control_request(
5761 body: &[u8],
5762) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
5763 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
5764 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5765 Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
5766 ControlRequestBodyError::InvalidBody
5767 }
5768 Ok(_) => ControlRequestBodyError::UnknownOp,
5769 Err(_) => ControlRequestBodyError::InvalidBody,
5770 };
5771 (err, classification)
5772 })
5773}
5774
5775fn parse_module_control_request_from_module(
5776 body: &[u8],
5777) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
5778 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
5779 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5780 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
5781 Ok(_) => ControlRequestBodyError::UnknownOp,
5782 Err(_) => ControlRequestBodyError::InvalidBody,
5783 };
5784 (err, classification)
5785 })
5786}
5787
5788#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
5789enum ProviderRoleKind {
5790 ToolProvider,
5791 PipelineStage,
5792 ManagementSurface,
5793 InternalService,
5794}
5795
5796fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
5797 match role {
5798 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
5799 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
5800 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
5801 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
5802 }
5803}
5804
5805fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
5806 roles.iter().map(provider_role_kind).collect()
5807}
5808
5809const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
5812
5813#[derive(Debug, Default, PartialEq, Eq)]
5815struct ScopeOutcomeCounts {
5816 created: usize,
5817 replaced: usize,
5818 updated: usize,
5819 unchanged: usize,
5820 refused: usize,
5821}
5822
5823impl ScopeOutcomeCounts {
5824 fn of(results: &[ScopeRecordResult]) -> Self {
5825 let mut counts = Self::default();
5826 for result in results {
5827 let slot = match result.outcome {
5828 ScopeRecordOutcome::Created => &mut counts.created,
5829 ScopeRecordOutcome::Replaced => &mut counts.replaced,
5830 ScopeRecordOutcome::Updated => &mut counts.updated,
5831 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
5832 ScopeRecordOutcome::Refused => &mut counts.refused,
5833 };
5834 *slot += 1;
5835 }
5836 counts
5837 }
5838}
5839
5840#[cfg(test)]
5841mod scope_outcome_count_tests {
5842 use super::*;
5843
5844 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
5845 ScopeRecordResult {
5846 scope_ref: "r".to_string(),
5847 scope_epoch: 1,
5848 outcome,
5849 code: None,
5850 message: None,
5851 version: None,
5852 parent_state: None,
5853 }
5854 }
5855
5856 #[test]
5859 fn every_outcome_is_counted_in_its_own_field() {
5860 let results = [
5861 result(ScopeRecordOutcome::Created),
5862 result(ScopeRecordOutcome::Created),
5863 result(ScopeRecordOutcome::Replaced),
5864 result(ScopeRecordOutcome::Updated),
5865 result(ScopeRecordOutcome::Unchanged),
5866 result(ScopeRecordOutcome::Refused),
5867 result(ScopeRecordOutcome::Refused),
5868 result(ScopeRecordOutcome::Refused),
5869 ];
5870 assert_eq!(
5871 ScopeOutcomeCounts::of(&results),
5872 ScopeOutcomeCounts {
5873 created: 2,
5874 replaced: 1,
5875 updated: 1,
5876 unchanged: 1,
5877 refused: 3,
5878 }
5879 );
5880 }
5881}
5882
5883fn capability_census_trigger(
5887 old: Option<&CapabilityDeclarations>,
5888 new: Option<&CapabilityDeclarations>,
5889) -> bool {
5890 let old_provides = old
5891 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
5892 .unwrap_or_default();
5893 let old_denies = old
5894 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
5895 .unwrap_or_default();
5896 let new = new.cloned().unwrap_or(CapabilityDeclarations {
5897 provides: Vec::new(),
5898 requires: Vec::new(),
5899 must_never_reach: Vec::new(),
5900 });
5901
5902 new.provides
5903 .iter()
5904 .any(|capability| !old_provides.contains(capability))
5905 || new
5906 .must_never_reach
5907 .iter()
5908 .any(|capability| !old_denies.contains(capability))
5909}
5910
5911fn denied_capability<'a>(
5914 opening_manifest: &'a ModuleManifest,
5915 target_manifest: &ModuleManifest,
5916) -> Option<&'a str> {
5917 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
5918 let target_capabilities = target_manifest.capabilities.as_ref()?;
5919 opening_capabilities
5920 .must_never_reach
5921 .iter()
5922 .find(|denied| {
5923 target_capabilities
5924 .provides
5925 .iter()
5926 .any(|provided| provided == *denied)
5927 })
5928 .map(String::as_str)
5929}
5930
5931fn catalog_update_frozen_field_message(
5932 registered: &ModuleManifest,
5933 provides: &[ProviderRole],
5934) -> Option<String> {
5935 let old_has_provides = !registered.provides.is_empty();
5936 let new_has_provides = !provides.is_empty();
5937 if old_has_provides != new_has_provides {
5938 return Some(format!(
5939 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
5940 registered.module_id
5941 ));
5942 }
5943
5944 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
5945 return Some(format!(
5946 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
5947 registered.module_id
5948 ));
5949 }
5950
5951 let registered_concurrency = manifest_concurrency(registered);
5952 let mut candidate = registered.clone();
5953 candidate.provides = provides.to_vec();
5954 let candidate_concurrency = manifest_concurrency(&candidate);
5955 if candidate_concurrency != registered_concurrency {
5956 return Some(format!(
5957 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
5958 registered.module_id, registered_concurrency, candidate_concurrency
5959 ));
5960 }
5961
5962 None
5965}
5966
5967fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
5968 manifest.provides.iter().any(is_routable_role)
5969}
5970
5971fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
5977 manifest
5978 .provides
5979 .iter()
5980 .find_map(|provider| match provider {
5981 ProviderRole::ToolProvider { concurrency, .. }
5982 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
5983 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
5984 })
5985 .unwrap_or(Concurrency::ModuleManaged)
5986}
5987
5988fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
5995 let has_management_surface = manifest
5996 .provides
5997 .iter()
5998 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
5999 if !has_management_surface {
6000 return false;
6001 }
6002 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
6003 return false;
6004 };
6005 let Some(provides) = raw
6006 .get("manifest")
6007 .and_then(|manifest| manifest.get("provides"))
6008 .and_then(serde_json::Value::as_array)
6009 else {
6010 return false;
6011 };
6012 provides.iter().any(|role| {
6017 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
6018 && role.get("concurrency").is_none()
6019 })
6020}
6021
6022fn negotiate_version(peer_version: u8) -> Result<u8, String> {
6023 if peer_version != PROTOCOL_VERSION {
6024 return Err(format!(
6025 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
6026 ));
6027 }
6028 Ok(PROTOCOL_VERSION)
6029}
6030
6031fn pong(frame: &Frame) -> Result<Frame, RouterError> {
6032 Frame::build_with_version(
6033 response_version(frame),
6034 FrameType::Pong,
6035 frame.header.flags,
6036 0,
6037 0,
6038 frame.header.corr,
6039 Vec::new(),
6040 )
6041 .map_err(RouterError::FrameBuild)
6042}
6043
6044fn control_error_frame(
6045 frame: &Frame,
6046 code: &'static str,
6047 message: impl Into<String>,
6048) -> Result<Frame, RouterError> {
6049 control_error_body_frame(
6050 frame,
6051 ErrorBody {
6052 code: code.to_string(),
6053 message: message.into(),
6054 detail: None,
6055 },
6056 )
6057}
6058
6059fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
6060 let body = serde_json::to_vec(&error).map_err(|err| {
6061 RouterError::backend(
6062 0,
6063 frame.header.corr,
6064 format!("failed to encode control ERROR: {err}"),
6065 )
6066 })?;
6067
6068 Frame::build_with_version(
6069 response_version(frame),
6070 FrameType::Error,
6071 control_flags(),
6072 0,
6073 0,
6074 frame.header.corr,
6075 body,
6076 )
6077 .map_err(RouterError::FrameBuild)
6078}
6079
6080fn control_response_body_frame<T: Serialize>(
6081 frame: &Frame,
6082 reply: &T,
6083 label: &'static str,
6084) -> Result<Frame, RouterError> {
6085 let body = serde_json::to_vec(reply).map_err(|err| {
6086 RouterError::backend(
6087 0,
6088 frame.header.corr,
6089 format!("failed to encode {label}: {err}"),
6090 )
6091 })?;
6092
6093 Frame::build_with_version(
6094 response_version(frame),
6095 FrameType::Response,
6096 control_flags(),
6097 0,
6098 0,
6099 frame.header.corr,
6100 body,
6101 )
6102 .map_err(RouterError::FrameBuild)
6103}
6104
6105fn forwarding_error_code(err: &ForwardingError) -> &'static str {
6135 match err {
6136 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
6137 ForwardingError::NoModuleConnection => "target_unavailable",
6138 ForwardingError::ModuleReloading { .. } => "module_reloading",
6139 ForwardingError::ClientRouteChannelExhausted { .. }
6140 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
6141 ForwardingError::StaleModuleEndpoint
6142 | ForwardingError::UnknownReservation { .. }
6143 | ForwardingError::ConnectionClosing { .. }
6144 | ForwardingError::ClientEgressClosed { .. }
6145 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
6146 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
6149 ForwardingError::RelayCorrelationExhausted
6150 | ForwardingError::RouteOpenBuild(_)
6151 | ForwardingError::Poisoned => "forwarding_error",
6152 }
6153}
6154
6155fn response_version(frame: &Frame) -> u8 {
6156 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
6157 frame.header.ver
6158 } else {
6159 PROTOCOL_VERSION
6160 }
6161}
6162
6163fn control_flags() -> Flags {
6164 Flags::new(false, Priority::Passive, false)
6165}
6166
6167fn send_goodbye_target_best_effort(
6172 counters: &DaemonCounters,
6173 target: &GoodbyeTarget,
6174 context: &'static str,
6175) {
6176 let Ok(frame) = Frame::build_with_version(
6177 target.negotiated_ver,
6178 FrameType::Goodbye,
6179 control_flags(),
6180 target.channel,
6181 target.epoch,
6182 0,
6183 Vec::new(),
6184 ) else {
6185 return;
6186 };
6187 crate::forwarding::send_module_route_goodbye(
6188 counters,
6189 &target.sink,
6190 frame,
6191 target.module_id.as_deref(),
6192 context,
6193 );
6194}
6195
6196pub(crate) fn send_route_control_pushes(
6197 forwarding: &ForwardingTable,
6198 routes: Vec<EndpointRoute>,
6199 push: ClientControlPush,
6200) {
6201 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
6202 for route in routes {
6203 let target = route.goodbye_target;
6204 if let Some((existing, channels)) = targets
6205 .iter_mut()
6206 .find(|(existing, _)| existing.connection_id == target.connection_id)
6207 {
6208 debug_assert_eq!(
6209 existing.negotiated_ver, target.negotiated_ver,
6210 "one connection cannot negotiate multiple frame versions"
6211 );
6212 if !channels.contains(&target.channel) {
6213 channels.push(target.channel);
6214 }
6215 continue;
6216 }
6217 let channel = target.channel;
6218 targets.push((target, vec![channel]));
6219 }
6220 for (target, mut channels) in targets {
6221 channels.sort_unstable();
6222 let mut push = push.clone();
6223 match &mut push {
6224 ClientControlPush::RouteClosing {
6225 channels: covered, ..
6226 }
6227 | ClientControlPush::RouteClosed {
6228 channels: covered, ..
6229 } => *covered = channels,
6230 }
6231 let body = match serde_json::to_vec(&push) {
6232 Ok(body) => body,
6233 Err(err) => {
6234 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
6235 continue;
6236 }
6237 };
6238 let frame = match Frame::build_with_version(
6239 target.negotiated_ver,
6240 FrameType::Push,
6241 control_flags(),
6242 0,
6243 0,
6244 0,
6245 body.clone(),
6246 ) {
6247 Ok(frame) => frame,
6248 Err(err) => {
6249 warn!(
6250 route_channel = target.channel,
6251 error = %err,
6252 "failed to build route lifecycle control PUSH frame"
6253 );
6254 continue;
6255 }
6256 };
6257 if let Err(err) = target.sink.try_send(frame) {
6258 if target.close_on_delivery_failure() {
6259 warn!(
6260 target_connection_id = target.connection_id.get(),
6261 route_channel = target.channel,
6262 error = %err,
6263 "route lifecycle control PUSH was not delivered to client; closing target connection"
6264 );
6265 let _ = forwarding.escalate_client_delivery_failure(
6266 target.connection_id,
6267 target.channel,
6268 target.epoch,
6269 CloseReason::new(
6270 "route_lifecycle_push_delivery_failed",
6271 format!(
6272 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
6273 target.channel
6274 ),
6275 ),
6276 crate::forwarding::UndeliveredFrame {
6277 module_id: target.module_id.as_deref(),
6278 sink: &target.sink,
6279 },
6280 );
6281 }
6282 }
6283 }
6284}
6285
6286#[cfg(test)]
6287mod tests {
6288 #[cfg(unix)]
6289 #[tokio::test]
6290 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
6291 let dir = subc_test_support::TestTempDir::new("rescan-live-health");
6292 let path = dir.join("subc.jsonc");
6293 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
6294 "program":"/bin/sleep","args":["60"],"protocol":"none",
6295 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
6296 }}}).to_string()).unwrap();
6297 let mut configured = crate::daemon_config::load(&path)
6298 .unwrap()
6299 .unwrap()
6300 .modules
6301 .pop()
6302 .unwrap();
6303 let registry = std::sync::Arc::new(crate::Registry::default());
6304 let handle = crate::SupervisorHandle::new();
6305 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
6306 .with_handle(handle.clone());
6307 let module = supervisor
6308 .supervise_configured_with_health(
6309 configured.module_spec(),
6310 true,
6311 configured.health.clone(),
6312 None,
6313 configured.restart,
6314 )
6315 .unwrap();
6316 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
6317 let before = module.status().unwrap().pid;
6318 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
6319 configured.health.cadence = std::time::Duration::from_secs(3600);
6320 let health_only = handler
6321 .reconcile_supervised_modules(&supervisor, vec![configured.clone()], false)
6322 .await
6323 .unwrap();
6324 assert!(
6325 health_only.changed_pending_reload.is_empty(),
6326 "health policy is already applied live"
6327 );
6328 assert_eq!(module.status().unwrap().pid, before);
6329 assert_eq!(
6330 module.configuration().unwrap().1.http,
6331 configured.health.http
6332 );
6333 configured.args = vec!["61".into()];
6334 let launch = handler
6335 .reconcile_supervised_modules(&supervisor, vec![configured], false)
6336 .await
6337 .unwrap();
6338 assert_eq!(launch.changed_pending_reload, ["stock"]);
6339 assert_eq!(
6340 module.status().unwrap().pid,
6341 before,
6342 "a launch edit is stored until reload"
6343 );
6344 module.drain().await.unwrap();
6345 }
6346 use std::{
6347 collections::BTreeMap,
6348 fmt,
6349 path::PathBuf,
6350 sync::{Arc, Mutex},
6351 time::Duration,
6352 };
6353 use subc_test_support::TestTempDir;
6354
6355 use serde_json::{json, Value};
6356 use subc_protocol::{
6357 manifest::{
6358 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
6359 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
6360 },
6361 session::HealthStatus,
6362 FrameType,
6363 };
6364
6365 use super::*;
6366 use crate::{
6367 forwarding::{DataRoute, DataRouteState},
6368 registry::ChannelState,
6369 router::FrameSink,
6370 stderr_tail::DEFAULT_MAX_LINE_BYTES,
6371 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
6372 RouteCtx, Router,
6373 };
6374 use tokio::{
6375 sync::mpsc,
6376 time::{sleep, Instant},
6377 };
6378 use tracing::{
6379 field::{Field, Visit},
6380 Event, Subscriber,
6381 };
6382 use tracing_subscriber::{layer::Context, prelude::*, Layer};
6383
6384 fn fake_aft_stub_path() -> PathBuf {
6403 let mut path = std::env::current_exe().expect("current_exe available in tests");
6404 path.pop(); path.pop(); path.push(if cfg!(windows) {
6407 "fake-aft-stub.exe"
6408 } else {
6409 "fake-aft-stub"
6410 });
6411 assert!(
6412 path.exists(),
6413 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
6414 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
6415 path.display()
6416 );
6417 path
6418 }
6419
6420 fn client_retries(code: &str) -> bool {
6425 subc_protocol::error_codes::is_retryable_route_open(code)
6426 }
6427
6428 #[test]
6439 fn retryability_of_forwarding_codes_matches_the_failure() {
6440 let transient = [
6443 ForwardingError::NoModuleConnection,
6444 ForwardingError::ModuleReloading {
6445 module_id: "m".into(),
6446 },
6447 ForwardingError::StaleModuleEndpoint,
6448 ForwardingError::UnknownReservation {
6449 client_channel: 1,
6450 module_channel: 1,
6451 },
6452 ForwardingError::ConnectionClosing {
6453 connection_id: ConnectionId::new(1),
6454 },
6455 ForwardingError::ClientEgressClosed {
6456 connection_id: ConnectionId::new(1),
6457 },
6458 ForwardingError::ModuleEgressUnavailable {
6459 connection_id: ConnectionId::new(1),
6460 },
6461 ];
6462 for err in transient {
6463 let code = forwarding_error_code(&err);
6464 assert!(
6465 client_retries(code),
6466 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
6467 );
6468 }
6469
6470 let permanent = [
6475 ForwardingError::ConnectionRoleConflict {
6476 connection_id: ConnectionId::new(1),
6477 },
6478 ForwardingError::ClientRouteChannelExhausted {
6479 connection_id: ConnectionId::new(1),
6480 },
6481 ForwardingError::ModuleRouteChannelExhausted {
6482 endpoint: ModuleEndpointId {
6483 connection_id: ConnectionId::new(1),
6484 generation: 1,
6485 },
6486 },
6487 ForwardingError::RelayCorrelationExhausted,
6488 ForwardingError::RouteOpenBuild("x".into()),
6489 ForwardingError::Poisoned,
6490 ];
6491 for err in permanent {
6492 let code = forwarding_error_code(&err);
6493 assert!(
6494 !client_retries(code),
6495 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
6496 );
6497 }
6498 }
6499
6500 #[tokio::test]
6528 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
6529 let handler = ControlHandler::default();
6530 let frame =
6531 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
6532
6533 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
6537 assert_eq!(
6538 stamped,
6539 Principal::Direct,
6540 "a caller that proved nothing must not be stamped as a supervised module"
6541 );
6542
6543 let forged = handler
6548 .route_open_principal(
6549 &frame,
6550 Some(ConsumerIdentity {
6551 module_id: "aft".to_string(),
6552 launch_nonce: "not-a-real-nonce".to_string(),
6553 }),
6554 )
6555 .unwrap();
6556 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
6557 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
6558 }
6559
6560 #[test]
6577 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
6578 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
6579 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
6580 let ClientControlRequest::RouteOpen {
6581 consumer_identity, ..
6582 } = parsed
6583 else {
6584 panic!("route.open body must parse as RouteOpen");
6585 };
6586 assert_eq!(
6587 consumer_identity,
6588 Some(ConsumerIdentity {
6589 module_id: "aft".to_string(),
6590 launch_nonce: "n".to_string(),
6591 }),
6592 "the wire field name must reach the value route_open_principal reads"
6593 );
6594 }
6595
6596 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
6597 ModuleManifest::builder(module_id, "0.1.0")
6598 .protocol_ver(protocol_ver)
6599 .provides(vec![ProviderRole::ToolProvider {
6600 tools: vec![Tool {
6601 name: "read".to_string(),
6602 description: None,
6603 execution_mode: ExecutionMode::Pure,
6604 schema: json!({"type": "object"}),
6605 }],
6606 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
6607 concurrency: Concurrency::ModuleManaged,
6608 emits_push: true,
6609 sub_supervises: true,
6610 }])
6611 .build()
6612 }
6613
6614 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
6615 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
6616 }
6617
6618 fn hello_frame_with_control_ops(
6619 module_id: &str,
6620 protocol_ver: u8,
6621 corr: u64,
6622 control_ops: Option<Vec<String>>,
6623 ) -> Frame {
6624 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
6625 }
6626
6627 fn hello_frame_with_nonce(
6628 module_id: &str,
6629 protocol_ver: u8,
6630 corr: u64,
6631 launch_nonce: Option<&str>,
6632 ) -> Frame {
6633 hello_frame_full(
6634 module_id,
6635 protocol_ver,
6636 corr,
6637 None,
6638 launch_nonce.map(ToOwned::to_owned),
6639 )
6640 }
6641
6642 fn hello_frame_full(
6643 module_id: &str,
6644 protocol_ver: u8,
6645 corr: u64,
6646 control_ops: Option<Vec<String>>,
6647 launch_nonce: Option<String>,
6648 ) -> Frame {
6649 let body = serde_json::to_vec(&ModuleHelloBody {
6650 manifest: manifest(module_id, protocol_ver),
6651 protocol_ver,
6652 control_ops,
6653 launch_nonce,
6654 })
6655 .unwrap();
6656 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
6657 }
6658
6659 fn non_routable_hello_frame_with_control_ops(
6660 module_id: &str,
6661 corr: u64,
6662 control_ops: Option<Vec<String>>,
6663 ) -> Frame {
6664 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
6665 manifest.provides.clear();
6666 let body = serde_json::to_vec(&ModuleHelloBody {
6667 manifest,
6668 protocol_ver: PROTOCOL_VERSION,
6669 control_ops,
6670 launch_nonce: None,
6671 })
6672 .unwrap();
6673 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
6674 }
6675
6676 fn capability_grammar_hello_frame(
6677 capabilities: Value,
6678 runtime_computed: Option<Value>,
6679 corr: u64,
6680 ) -> Frame {
6681 let mut body = serde_json::to_value(ModuleHelloBody {
6682 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
6683 protocol_ver: PROTOCOL_VERSION,
6684 control_ops: None,
6685 launch_nonce: None,
6686 })
6687 .expect("HELLO body serializes");
6688 body["manifest"]["capabilities"] = capabilities;
6689 if let Some(runtime_computed) = runtime_computed {
6690 body["runtime_computed"] = runtime_computed;
6691 }
6692 Frame::build(
6693 FrameType::Hello,
6694 control_flags(),
6695 0,
6696 0,
6697 corr,
6698 serde_json::to_vec(&body).expect("HELLO body reserializes"),
6699 )
6700 .expect("HELLO frame builds")
6701 }
6702
6703 fn channel_request(channel: u16, corr: u64) -> Frame {
6704 Frame::build(
6705 FrameType::Request,
6706 Flags::new(true, Priority::Interactive, false),
6707 channel,
6708 0,
6709 corr,
6710 b"opaque".to_vec(),
6711 )
6712 .unwrap()
6713 }
6714
6715 fn route_ctx(
6716 connection_id: ConnectionId,
6717 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
6718 let (tx, rx) = mpsc::channel(8);
6719 (
6720 RouteCtx {
6721 connection_id,
6722 egress: FrameSink::new(tx),
6723 },
6724 rx,
6725 )
6726 }
6727
6728 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
6729 serde_json::from_slice(&frame.body).unwrap()
6730 }
6731
6732 async fn hello_via_sink(
6737 handler: &ControlHandler,
6738 ctx: &RouteCtx,
6739 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
6740 hello: Frame,
6741 ) -> Frame {
6742 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
6743 assert!(
6744 replies.is_empty(),
6745 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
6746 );
6747 let ack = rx
6748 .try_recv()
6749 .expect("HELLO_ACK is queued on the module sink")
6750 .frame;
6751 assert_eq!(ack.header.ty, FrameType::HelloAck);
6752 ack
6753 }
6754
6755 fn parse_error(frame: &Frame) -> Value {
6756 serde_json::from_slice(&frame.body).unwrap()
6757 }
6758
6759 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
6760 serde_json::from_slice(&frame.body).unwrap()
6761 }
6762
6763 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
6764 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
6765 route_channel,
6766 route_epoch: 0,
6767 kind,
6768 })
6769 .unwrap();
6770 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6771 }
6772
6773 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
6774 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
6775 module_id: module_id.to_string(),
6776 })
6777 .unwrap();
6778 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6779 }
6780
6781 fn route_open_frame(corr: u64, module_id: &str, project_root: TestTempDir) -> Frame {
6782 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
6783 }
6784
6785 fn route_open_frame_with_consumer_capabilities(
6786 corr: u64,
6787 module_id: &str,
6788 project_root: TestTempDir,
6789 consumer_capabilities: Option<Vec<String>>,
6790 ) -> Frame {
6791 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6792 target: RouteTarget::ToolProvider {
6793 module_id: module_id.to_string(),
6794 },
6795 identity: BindIdentity::new(
6796 project_root.path().to_path_buf(),
6797 "unit".to_string(),
6798 "session".to_string(),
6799 ),
6800 consumer_identity: None,
6801 consumer_capabilities,
6802 role_versions: None,
6803 admission_facts: None,
6804 scope: None,
6805 })
6806 .unwrap();
6807 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6808 }
6809
6810 fn route_open_frame_with_role_versions(
6811 corr: u64,
6812 module_id: &str,
6813 project_root: TestTempDir,
6814 role_versions: Option<BTreeMap<String, String>>,
6815 ) -> Frame {
6816 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6817 target: RouteTarget::ToolProvider {
6818 module_id: module_id.to_string(),
6819 },
6820 identity: BindIdentity::new(
6821 project_root.path().to_path_buf(),
6822 "unit".to_string(),
6823 format!("session-{corr}"),
6824 ),
6825 consumer_identity: None,
6826 consumer_capabilities: None,
6827 role_versions,
6828 admission_facts: None,
6829 scope: None,
6830 })
6831 .unwrap();
6832 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6833 }
6834
6835 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
6836 entries
6837 .iter()
6838 .map(|(role, version)| (role.to_string(), version.to_string()))
6839 .collect()
6840 }
6841
6842 fn route_open_frame_with_admission_facts(
6843 corr: u64,
6844 module_id: &str,
6845 project_root: TestTempDir,
6846 consumer_identity: Option<subc_control::ConsumerIdentity>,
6847 facts: Option<Value>,
6848 ) -> Frame {
6849 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6850 target: RouteTarget::ToolProvider {
6851 module_id: module_id.to_string(),
6852 },
6853 identity: BindIdentity::new(
6854 project_root.path().to_path_buf(),
6855 "unit".to_string(),
6856 format!("session-{corr}"),
6857 ),
6858 consumer_identity,
6859 consumer_capabilities: None,
6860 role_versions: None,
6861 admission_facts: facts,
6862 scope: None,
6863 })
6864 .unwrap();
6865 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6866 }
6867
6868 #[derive(Clone, Default)]
6869 struct EventCapture {
6870 events: Arc<Mutex<Vec<CapturedEvent>>>,
6871 }
6872
6873 #[derive(Clone, Debug)]
6874 struct CapturedEvent {
6875 target: String,
6876 level: tracing::Level,
6877 fields: BTreeMap<String, String>,
6878 }
6879
6880 impl EventCapture {
6881 fn events(&self) -> Vec<CapturedEvent> {
6882 self.events.lock().unwrap().clone()
6883 }
6884 }
6885
6886 impl<S> Layer<S> for EventCapture
6887 where
6888 S: Subscriber,
6889 {
6890 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
6891 let mut visitor = EventFieldVisitor::default();
6892 event.record(&mut visitor);
6893 self.events.lock().unwrap().push(CapturedEvent {
6894 target: event.metadata().target().to_string(),
6895 level: *event.metadata().level(),
6896 fields: visitor.fields,
6897 });
6898 }
6899 }
6900
6901 #[derive(Default)]
6902 struct EventFieldVisitor {
6903 fields: BTreeMap<String, String>,
6904 }
6905
6906 impl Visit for EventFieldVisitor {
6907 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
6908 self.fields
6909 .insert(field.name().to_string(), format!("{value:?}"));
6910 }
6911 }
6912
6913 fn health_response(corr: u64, status: HealthStatus) -> Frame {
6914 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
6915 status,
6916 detail: Some("warming".to_string()),
6917 metrics: Some(json!({"queue_depth": 3})),
6918 })
6919 .unwrap();
6920 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
6921 }
6922
6923 fn route_bind_ack(corr: u64) -> Frame {
6924 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
6925 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
6926 }
6927
6928 fn unique_project_root(label: &str) -> TestTempDir {
6929 TestTempDir::new(label)
6930 }
6931
6932 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
6933 match parse_route_poll(frame) {
6934 ClientControlResponse::RoutePoll {
6935 status: None,
6936 live: Some(live),
6937 ..
6938 } => assert_eq!(live, expected_live),
6939 other => panic!("unexpected route.poll response: {other:?}"),
6940 }
6941 }
6942
6943 fn bind_liveness_route(
6944 registry: &Registry,
6945 forwarding: &ForwardingTable,
6946 module_id: &str,
6947 ) -> (RouteCtx, u16, u32) {
6948 let module_connection = ConnectionId::new(101);
6949 let client_connection = ConnectionId::new(202);
6950 let registration = registry
6951 .register_with_control_ops(
6952 manifest(module_id, PROTOCOL_VERSION),
6953 PROTOCOL_VERSION,
6954 module_connection,
6955 module_baseline_control_ops(),
6956 )
6957 .unwrap();
6958 let (module_tx, _module_rx) = mpsc::channel(8);
6959 let endpoint = forwarding
6960 .register_module_connection(
6961 module_connection,
6962 module_id.to_string(),
6963 PROTOCOL_VERSION,
6964 manifest_concurrency(®istration.manifest),
6965 FrameSink::new(module_tx),
6966 )
6967 .unwrap();
6968 let (client_ctx, _client_rx) = route_ctx(client_connection);
6969 let pending = forwarding
6970 .begin_route_bind_relay_for_test(
6971 client_connection,
6972 client_ctx.egress.clone(),
6973 1,
6974 module_id,
6975 )
6976 .unwrap();
6977 assert_eq!(pending.endpoint, endpoint);
6978 let route_channel = pending.client_channel;
6979 let route_epoch = pending.client_epoch;
6980 forwarding
6981 .complete_pending_relay(
6982 module_connection,
6983 pending.corr,
6984 RouteBindRelayOutcome::Accepted,
6985 )
6986 .unwrap();
6987 (client_ctx, route_channel, route_epoch)
6988 }
6989
6990 struct FakeProcessLiveness {
6991 live: Option<bool>,
6992 }
6993
6994 impl ModuleProcessLiveness for FakeProcessLiveness {
6995 fn process_live(&self, _module_id: &str) -> Option<bool> {
6996 self.live
6997 }
6998 }
6999
7000 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7001 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
7002 {
7003 let registry = Arc::new(Registry::default());
7004 let supervisor_handle = SupervisorHandle::new();
7005 let supervisor = Supervisor::new_for_test(
7006 Arc::clone(®istry),
7007 RestartPolicy::new(1, Duration::from_millis(10)),
7008 )
7009 .with_handle(supervisor_handle.clone());
7010 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
7011 let module = supervisor
7012 .spawn(ModuleSpec {
7013 module_id: "stderr-tail-wire".to_string(),
7014 program: fake_aft_stub_path(),
7015 args: Vec::new(),
7016 env: vec![
7017 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
7018 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
7019 ],
7020 reserved: false,
7021 reserved_prefixes: Vec::new(),
7022 protocol: ModuleProtocol::Subc,
7023 overlap: Default::default(),
7024 })
7025 .unwrap();
7026
7027 let deadline = Instant::now() + Duration::from_secs(5);
7028 loop {
7029 let tail = module.stderr_tail(None, None);
7030 if tail
7031 .entries
7032 .iter()
7033 .any(|entry| matches!(entry, TailEntry::ProcessStart))
7034 && tail.entries.iter().any(|entry| {
7035 matches!(
7036 entry,
7037 TailEntry::Line {
7038 truncated: true,
7039 ..
7040 }
7041 )
7042 })
7043 {
7044 break;
7045 }
7046 assert!(
7047 Instant::now() < deadline,
7048 "module did not produce a truncated line and restart boundary: {tail:?}"
7049 );
7050 sleep(Duration::from_millis(10)).await;
7051 }
7052
7053 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
7054 let request = ClientControlRequest::SupervisorStderrTail {
7055 module_id: "stderr-tail-wire".to_string(),
7056 max_lines: None,
7057 max_bytes: None,
7058 };
7059 let frame = Frame::build(
7060 FrameType::Request,
7061 control_flags(),
7062 0,
7063 0,
7064 1,
7065 serde_json::to_vec(&request).unwrap(),
7066 )
7067 .unwrap();
7068 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7069 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
7070 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
7071 serde_json::from_slice(&responses[0].body).unwrap()
7072 else {
7073 panic!("expected supervisor.stderr_tail response");
7074 };
7075
7076 assert!(
7077 tail.entries
7078 .iter()
7079 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
7080 "the control response lost the restart boundary"
7081 );
7082 let Some(StderrTailEntry::Line {
7083 text,
7084 truncated,
7085 at_ms,
7086 }) = tail.entries.iter().find(|entry| {
7087 matches!(
7088 entry,
7089 StderrTailEntry::Line {
7090 truncated: true,
7091 ..
7092 }
7093 )
7094 })
7095 else {
7096 panic!("the control response lost the truncated line");
7097 };
7098 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
7099 assert!(*truncated);
7100 assert!(
7101 at_ms.is_some(),
7102 "the control response lost the line's capture time"
7103 );
7104 }
7105
7106 #[tokio::test(flavor = "current_thread")]
7111 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
7112 let dir = TestTempDir::new("terminals-off-worker");
7113 let journal_path = dir.join("terminals.jsonl");
7114 let registry = Arc::new(Registry::default());
7115 let supervisor_handle = SupervisorHandle::new();
7116 let supervisor =
7117 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
7118 .with_handle(supervisor_handle.clone())
7119 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
7120 let module = supervisor
7121 .spawn(ModuleSpec {
7122 module_id: "terminal-off-worker".to_string(),
7123 program: fake_aft_stub_path(),
7124 args: Vec::new(),
7125 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
7126 reserved: false,
7127 reserved_prefixes: Vec::new(),
7128 protocol: ModuleProtocol::Subc,
7129 overlap: Default::default(),
7130 })
7131 .unwrap();
7132 let deadline = Instant::now() + Duration::from_secs(5);
7133 while module.terminal_history().entries.len() != 2 {
7134 assert!(Instant::now() < deadline, "module did not record two exits");
7135 sleep(Duration::from_millis(10)).await;
7136 }
7137
7138 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
7139 let handler =
7140 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
7141 let frame = Frame::build(
7142 FrameType::Request,
7143 control_flags(),
7144 0,
7145 0,
7146 1,
7147 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
7148 module_id: "terminal-off-worker".to_string(),
7149 })
7150 .unwrap(),
7151 )
7152 .unwrap();
7153 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7154 let spawned_at = std::time::Instant::now();
7155 let read = tokio::spawn({
7156 let handler = Arc::clone(&handler);
7157 async move { handler.handle_control_frame(&ctx, frame).await }
7158 });
7159 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
7162 .await
7163 .unwrap()
7164 .expect("the history read reached its pause");
7165 let elapsed = spawned_at.elapsed();
7166 assert!(
7167 elapsed < Duration::from_secs(2) && !read.is_finished(),
7168 "this task could not run while the history read was paused \
7169 (resumed after {elapsed:?}, read finished: {})",
7170 read.is_finished()
7171 );
7172
7173 drop(release);
7174 let responses = read.await.unwrap().unwrap();
7175 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
7176 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
7177 panic!("expected supervisor.terminals response");
7178 };
7179 assert_eq!(terminals.entries.len(), 2);
7180 assert_eq!(terminals.journal_skipped_lines, 0);
7181 assert_eq!(terminals.journal_read_errors, 0);
7182 }
7183
7184 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7185 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
7186 let registry = Arc::new(Registry::default());
7187 let supervisor_handle = SupervisorHandle::new();
7188 let supervisor =
7189 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
7190 .with_handle(supervisor_handle.clone());
7191 let module = supervisor
7192 .spawn(ModuleSpec {
7193 module_id: "terminal-golden".to_string(),
7194 program: fake_aft_stub_path(),
7195 args: Vec::new(),
7196 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
7197 reserved: false,
7198 reserved_prefixes: Vec::new(),
7199 protocol: ModuleProtocol::Subc,
7200 overlap: Default::default(),
7201 })
7202 .unwrap();
7203
7204 let deadline = Instant::now() + Duration::from_secs(5);
7205 while module.terminal_history().entries.len() != 2 {
7206 assert!(
7207 Instant::now() < deadline,
7208 "module did not retain two terminal exits: {:?}",
7209 module.terminal_history()
7210 );
7211 sleep(Duration::from_millis(10)).await;
7212 }
7213
7214 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
7215 let request = ClientControlRequest::SupervisorTerminals {
7216 module_id: "terminal-golden".to_string(),
7217 };
7218 let frame = Frame::build(
7219 FrameType::Request,
7220 control_flags(),
7221 0,
7222 0,
7223 1,
7224 serde_json::to_vec(&request).unwrap(),
7225 )
7226 .unwrap();
7227 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7228 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
7229 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
7230 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
7231 panic!("expected supervisor.terminals response");
7232 };
7233 assert_eq!(terminals.entries.len(), 2);
7234 assert_eq!(terminals.dropped, 0);
7235
7236 let mut rendered = serde_json::to_value(response).unwrap();
7237 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
7240 for (index, entry) in rendered["entries"]
7241 .as_array_mut()
7242 .expect("terminal response entries array")
7243 .iter_mut()
7244 .enumerate()
7245 {
7246 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
7247 }
7248
7249 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
7250 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
7251 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
7252 if std::env::var_os("UPDATE_GOLDEN").is_some() {
7253 std::fs::write(&golden_path, &serialized).unwrap();
7254 }
7255 let expected: Value =
7256 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
7257 assert_eq!(rendered, expected);
7258 }
7259
7260 #[test]
7261 fn hello_registers_manifest_and_returns_ack() {
7262 let registry = Arc::new(Registry::default());
7263 let handler = ControlHandler::new(Arc::clone(®istry));
7264 let conn = ConnectionId::new(1);
7265
7266 let responses = handler
7267 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
7268 .unwrap();
7269
7270 assert_eq!(responses.len(), 1);
7271 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
7272 assert_eq!(responses[0].header.channel, 0);
7273 assert_eq!(responses[0].header.corr, 7);
7274 let ack = parse_ack(&responses[0]);
7275 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
7276 assert!(ack
7277 .subc_capabilities
7278 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
7279 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
7280 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
7281 assert!(ack
7282 .subc_ops
7283 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
7284 assert!(ack
7285 .subc_ops
7286 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
7287
7288 let registration = registry.get_module("aft").unwrap().unwrap();
7289 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
7290 assert_eq!(registration.state, ChannelState::Active);
7291 assert_eq!(registration.connection_id, conn);
7292 assert_eq!(registration.control_ops, module_baseline_control_ops());
7293 }
7294
7295 #[test]
7296 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
7297 let invalid_identifiers = [
7298 ("case_change", "credentials-Provider/v1"),
7299 ("leading_zero", "credentials-provider/v01"),
7300 ("trailing_hyphen", "credentials-provider-/v1"),
7301 ("consecutive_hyphens", "credentials--provider/v1"),
7302 ("uppercase", "Credentials-provider/v1"),
7303 ("missing_v", "credentials-provider/1"),
7304 ("whitespace", "credentials provider/v1"),
7305 ("zero_version", "credentials-provider/v0"),
7306 ("out_of_range_version", "credentials-provider/v4294967296"),
7307 (
7308 "overlength_name",
7309 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
7310 ),
7311 ];
7312 let mut cases = invalid_identifiers
7313 .into_iter()
7314 .map(|(name, identifier)| {
7315 (
7316 format!("identifier_{name}"),
7317 "capabilities.provides[0]".to_string(),
7318 identifier.to_string(),
7319 json!({ "provides": [identifier] }),
7320 None,
7321 )
7322 })
7323 .collect::<Vec<_>>();
7324 cases.extend([
7325 (
7326 "unknown_need".to_string(),
7327 "capabilities.requires[0].need".to_string(),
7328 "deferred".to_string(),
7329 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
7330 None,
7331 ),
7332 (
7333 "duplicate_provides".to_string(),
7334 "capabilities.provides[1]".to_string(),
7335 "credentials-provider/v1".to_string(),
7336 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
7337 None,
7338 ),
7339 (
7340 "duplicate_must_never_reach".to_string(),
7341 "capabilities.must_never_reach[1]".to_string(),
7342 "credentials-provider/v1".to_string(),
7343 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
7344 None,
7345 ),
7346 (
7347 "duplicate_requires_same_need".to_string(),
7348 "capabilities.requires[1]".to_string(),
7349 "credentials-provider/v1".to_string(),
7350 json!({ "requires": [
7351 { "capability": "credentials-provider/v1", "need": "required" },
7352 { "capability": "credentials-provider/v1", "need": "required" }
7353 ] }),
7354 None,
7355 ),
7356 (
7357 "duplicate_requires_conflicting_need".to_string(),
7358 "capabilities.requires[1]".to_string(),
7359 "credentials-provider/v1".to_string(),
7360 json!({ "requires": [
7361 { "capability": "credentials-provider/v1", "need": "required" },
7362 { "capability": "credentials-provider/v1", "need": "optional" }
7363 ] }),
7364 None,
7365 ),
7366 (
7367 "capabilities_root_pointer".to_string(),
7368 "runtime_computed[0]".to_string(),
7369 "/capabilities".to_string(),
7370 json!({}),
7371 Some(json!(["/capabilities"])),
7372 ),
7373 (
7374 "capabilities_descendant_pointer".to_string(),
7375 "runtime_computed[0]".to_string(),
7376 "/capabilities/provides".to_string(),
7377 json!({}),
7378 Some(json!(["/capabilities/provides"])),
7379 ),
7380 (
7381 "malformed_pointer_without_leading_slash".to_string(),
7382 "runtime_computed[0]".to_string(),
7383 "capabilities".to_string(),
7384 json!({}),
7385 Some(json!(["capabilities"])),
7386 ),
7387 (
7388 "malformed_pointer_escape".to_string(),
7389 "runtime_computed[0]".to_string(),
7390 "/roles/~2/tools".to_string(),
7391 json!({}),
7392 Some(json!(["/roles/~2/tools"])),
7393 ),
7394 (
7395 "unknown_capabilities_field".to_string(),
7396 "capabilities.future".to_string(),
7397 "<array>".to_string(),
7398 json!({ "future": [] }),
7399 None,
7400 ),
7401 ]);
7402
7403 for (index, (name, field, value, capabilities, runtime_computed)) in
7404 cases.into_iter().enumerate()
7405 {
7406 let registry = Arc::new(Registry::default());
7407 let handler = ControlHandler::new(Arc::clone(®istry));
7408 let response = handler
7409 .handle_control(
7410 ConnectionId::new((index + 1) as u64),
7411 capability_grammar_hello_frame(
7412 capabilities,
7413 runtime_computed,
7414 index as u64 + 1,
7415 ),
7416 )
7417 .expect("invalid HELLO returns a refusal");
7418
7419 assert_eq!(response.len(), 1, "{name} must emit one refusal");
7420 let error = parse_error(&response[0]);
7421 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
7422 let message = error["message"]
7423 .as_str()
7424 .expect("error message is a string");
7425 assert!(
7426 message.contains(&field),
7427 "{name}: field missing from {message}"
7428 );
7429 assert!(
7430 message.contains(&value),
7431 "{name}: value missing from {message}"
7432 );
7433 assert_eq!(
7434 registry
7435 .active_registration_count()
7436 .expect("registry reads"),
7437 0,
7438 "{name}: refused HELLO must not create a catalog entry"
7439 );
7440 }
7441 }
7442
7443 #[test]
7444 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
7445 let registry = Arc::new(Registry::default());
7446 let handler = ControlHandler::new(Arc::clone(®istry));
7447 let capabilities = json!({
7448 "provides": ["credentials-provider/v1"],
7449 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
7450 "must_never_reach": ["federation-transport/v1"]
7451 });
7452 let response = handler
7453 .handle_control(
7454 ConnectionId::new(99),
7455 capability_grammar_hello_frame(
7456 capabilities.clone(),
7457 Some(json!(["/roles/0/tools"])),
7458 99,
7459 ),
7460 )
7461 .expect("valid HELLO registers");
7462 assert_eq!(response[0].header.ty, FrameType::HelloAck);
7463
7464 let request = Frame::build(
7465 FrameType::Request,
7466 control_flags(),
7467 0,
7468 0,
7469 100,
7470 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7471 .expect("catalog request serializes"),
7472 )
7473 .expect("catalog request frame builds");
7474 let response = handler
7475 .handle_catalog_list(request, None)
7476 .expect("catalog list succeeds");
7477 let ClientControlResponse::CatalogList { modules, .. } =
7478 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
7479 else {
7480 panic!("catalog request must return catalog.list");
7481 };
7482 assert_eq!(modules.len(), 1);
7483 assert_eq!(
7484 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
7485 capabilities
7486 );
7487 }
7488
7489 #[test]
7490 fn catalog_list_mirrors_management_operation_description() {
7491 let registry = Arc::new(Registry::default());
7492 let handler = ControlHandler::new(Arc::clone(®istry));
7493 let description = "List managed records and return their identifiers and metadata.";
7494 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
7495 manifest.provides = vec![ProviderRole::ManagementSurface {
7496 operations: vec![ManagementOperation {
7497 name: "records.list".to_string(),
7498 kind: ManagementOperationKind::Query,
7499 description: Some(description.to_string()),
7500 }],
7501 config_schema: json!({"type": "object"}),
7502 observability: vec![ObservabilitySurface {
7503 name: "records.stats".to_string(),
7504 kind: ObservabilityKind::Snapshot,
7505 }],
7506 identity_scope: vec![IdentityScope::Project],
7507 concurrency: Concurrency::ModuleManaged,
7508 }];
7509 registry
7510 .register_with_control_ops(
7511 manifest,
7512 PROTOCOL_VERSION,
7513 ConnectionId::new(99),
7514 Vec::new(),
7515 )
7516 .expect("described management manifest registers");
7517
7518 let request = Frame::build(
7519 FrameType::Request,
7520 control_flags(),
7521 0,
7522 0,
7523 100,
7524 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7525 .expect("catalog request serializes"),
7526 )
7527 .expect("catalog request frame builds");
7528 let response = handler
7529 .handle_catalog_list(request, None)
7530 .expect("catalog list succeeds");
7531 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
7532 assert_eq!(
7533 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
7534 "catalog.list must preserve the declared operation description verbatim"
7535 );
7536 }
7537
7538 #[test]
7539 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
7540 let registry = Arc::new(Registry::default());
7541 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
7542 [("vault".to_string(), true), ("squatter".to_string(), true)],
7543 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
7544 );
7545 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
7546 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
7547 provides: vec!["credentials-provider/v1".to_string()],
7548 requires: Vec::new(),
7549 must_never_reach: Vec::new(),
7550 });
7551 let frame = Frame::build(
7552 FrameType::Hello,
7553 control_flags(),
7554 0,
7555 0,
7556 77,
7557 serde_json::to_vec(&ModuleHelloBody {
7558 manifest: squatter,
7559 protocol_ver: PROTOCOL_VERSION,
7560 control_ops: None,
7561 launch_nonce: None,
7562 })
7563 .expect("HELLO serializes"),
7564 )
7565 .expect("HELLO frame builds");
7566 let response = handler
7567 .handle_control(ConnectionId::new(77), frame)
7568 .expect("reserved claim receives a typed refusal");
7569 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
7570 assert_eq!(
7571 registry
7572 .active_registration_count()
7573 .expect("registry reads"),
7574 0,
7575 "a reserved capability refusal must not leave a catalog entry"
7576 );
7577 }
7578
7579 #[test]
7580 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
7581 for settlement in ["timeout", "inconclusive", "drop"] {
7582 let breakers = RouteBindBreakers::default();
7583 let RouteBindAdmission::Admitted {
7584 guard: mut old,
7585 probe: false,
7586 } = breakers.admit("prov")
7587 else {
7588 panic!("ordinary relay admitted")
7589 };
7590 let RouteBindAdmission::Admitted {
7591 guard: mut opener, ..
7592 } = breakers.admit("prov")
7593 else {
7594 panic!("second relay admitted")
7595 };
7596 assert!(
7597 !opener
7598 .record_timeout(1, Duration::ZERO)
7599 .unwrap()
7600 .reopened_after_probe
7601 );
7602 let RouteBindAdmission::Admitted {
7603 guard: mut probe,
7604 probe: true,
7605 } = breakers.admit("prov")
7606 else {
7607 panic!("one cooldown probe admitted")
7608 };
7609 match settlement {
7610 "timeout" => assert!(
7611 !old.record_timeout(1, Duration::ZERO)
7612 .unwrap()
7613 .reopened_after_probe
7614 ),
7615 "inconclusive" => old.record_inconclusive(),
7616 "drop" => drop(old),
7617 _ => unreachable!(),
7618 }
7619 assert!(
7620 matches!(
7621 breakers.admit("prov"),
7622 RouteBindAdmission::Refused {
7623 probe_in_flight: true,
7624 ..
7625 }
7626 ),
7627 "{settlement} of a pre-open relay cannot release the real probe"
7628 );
7629 assert!(
7630 probe
7631 .record_timeout(1, Duration::ZERO)
7632 .unwrap()
7633 .reopened_after_probe
7634 );
7635 assert!(matches!(
7636 breakers.admit("prov"),
7637 RouteBindAdmission::Admitted { probe: true, .. }
7638 ));
7639 }
7640 let breakers = RouteBindBreakers::default();
7641 let admit = || match breakers.admit("prov") {
7642 RouteBindAdmission::Admitted { guard, .. } => guard,
7643 _ => panic!("relay admitted"),
7644 };
7645 admit().record_timeout(1, Duration::ZERO);
7646 let mut old_probe = admit();
7647 breakers.reset_for_new_module_connection("prov");
7648 admit().record_timeout(1, Duration::ZERO);
7649 let _new_probe = admit();
7650 old_probe.record_inconclusive();
7651 assert!(matches!(
7652 breakers.admit("prov"),
7653 RouteBindAdmission::Refused {
7654 probe_in_flight: true,
7655 ..
7656 }
7657 ));
7658 }
7659
7660 #[tokio::test]
7661 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
7662 for candidate in [false, true] {
7663 let registry = Arc::new(Registry::default());
7664 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
7665 [("vault".to_string(), true), ("squatter".to_string(), true)],
7666 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
7667 );
7668 let conn = ConnectionId::new(77);
7669 let (ctx, mut rx) = route_ctx(conn);
7670 let initial = capability_manifest("squatter", &[], &[]);
7671 if candidate {
7672 registry
7673 .register_candidate_with_control_ops(
7674 initial.clone(),
7675 PROTOCOL_VERSION,
7676 conn,
7677 module_baseline_control_ops(),
7678 )
7679 .unwrap();
7680 handler
7681 .forwarding
7682 .register_candidate_module_connection(
7683 conn,
7684 "squatter".to_string(),
7685 PROTOCOL_VERSION,
7686 manifest_concurrency(&initial),
7687 ctx.egress.clone(),
7688 )
7689 .unwrap();
7690 } else {
7691 hello_via_sink(
7692 &handler,
7693 &ctx,
7694 &mut rx,
7695 hello_frame_with_manifest(initial.clone(), 1),
7696 )
7697 .await;
7698 }
7699 let response = handler
7700 .handle_control_frame(
7701 &ctx,
7702 catalog_update_with_capabilities_frame(
7703 2,
7704 capability_manifest("squatter", &["credentials-provider/v1"], &[])
7705 .capabilities
7706 .unwrap(),
7707 ),
7708 )
7709 .await
7710 .unwrap();
7711 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
7712 assert_eq!(
7713 registry
7714 .get_module_by_connection(conn)
7715 .unwrap()
7716 .unwrap()
7717 .manifest,
7718 initial
7719 );
7720 }
7721 }
7722
7723 #[test]
7724 fn server_describe_surfaces_required_capability_verdict_fields() {
7725 let registry = Arc::new(Registry::default());
7726 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
7727 [
7728 ("consumer".to_string(), true),
7729 ("provider".to_string(), false),
7730 ],
7731 BTreeMap::new(),
7732 );
7733 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
7734 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
7735 provides: Vec::new(),
7736 requires: vec![subc_protocol::manifest::CapabilityRequirement {
7737 capability: "credentials-provider/v1".to_string(),
7738 need: subc_protocol::manifest::CapabilityNeed::Required,
7739 }],
7740 must_never_reach: Vec::new(),
7741 });
7742 let hello = Frame::build(
7743 FrameType::Hello,
7744 control_flags(),
7745 0,
7746 0,
7747 78,
7748 serde_json::to_vec(&ModuleHelloBody {
7749 manifest: consumer,
7750 protocol_ver: PROTOCOL_VERSION,
7751 control_ops: None,
7752 launch_nonce: None,
7753 })
7754 .expect("HELLO serializes"),
7755 )
7756 .expect("HELLO frame builds");
7757 handler
7758 .handle_control(ConnectionId::new(78), hello)
7759 .expect("consumer registers");
7760 let describe = Frame::build(
7761 FrameType::Request,
7762 control_flags(),
7763 0,
7764 0,
7765 79,
7766 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
7767 .expect("request serializes"),
7768 )
7769 .expect("describe frame builds");
7770 let response = handler
7771 .handle_server_describe(describe)
7772 .expect("server.describe succeeds");
7773 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
7774 let requirement = &rendered["capability_requirements"][0];
7775 assert_eq!(requirement["consumer"], "consumer");
7776 assert_eq!(requirement["verdict"], "never_provided");
7777 assert_eq!(requirement["episode_seq"], 1);
7778 assert_eq!(requirement["config_satisfiable"], false);
7779 assert_eq!(requirement["runtime_available"], false);
7780 assert!(requirement["detail"]
7781 .as_str()
7782 .expect("detail string")
7783 .contains("credentials-provider/v1"));
7784 }
7785
7786 #[test]
7787 fn catalog_list_omits_capabilities_for_legacy_manifest() {
7788 let registry = Arc::new(Registry::default());
7789 let handler = ControlHandler::new(Arc::clone(®istry));
7790 let hello = handler
7791 .handle_control(
7792 ConnectionId::new(101),
7793 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
7794 )
7795 .expect("legacy HELLO registers");
7796 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
7797
7798 let request = Frame::build(
7799 FrameType::Request,
7800 control_flags(),
7801 0,
7802 0,
7803 102,
7804 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7805 .expect("catalog request serializes"),
7806 )
7807 .expect("catalog request frame builds");
7808 let response = handler
7809 .handle_catalog_list(request, None)
7810 .expect("catalog list succeeds");
7811 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
7812 assert!(
7813 body["modules"][0].get("capabilities").is_none(),
7814 "legacy manifest must retain an absent capabilities field on catalog.list"
7815 );
7816 }
7817
7818 #[test]
7819 fn hello_ack_omits_storage_when_no_storage_config() {
7820 let registry = Arc::new(Registry::default());
7821 let handler = ControlHandler::new(Arc::clone(®istry));
7822 let responses = handler
7823 .handle_control(
7824 ConnectionId::new(1),
7825 hello_frame("aft", PROTOCOL_VERSION, 7),
7826 )
7827 .unwrap();
7828 let ack = parse_ack(&responses[0]);
7829 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
7830 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
7831 }
7832
7833 #[tokio::test]
7834 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
7835 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
7836 let registry = Arc::new(Registry::default());
7837 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
7838 let responses = handler
7839 .handle_control(
7840 ConnectionId::new(1),
7841 hello_frame("aft", PROTOCOL_VERSION, 7),
7842 )
7843 .unwrap();
7844 let ack = parse_ack(&responses[0]);
7845 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
7846
7847 let described = handler
7848 .handle_control_frame(
7849 &route_ctx(ConnectionId::new(2)).0,
7850 Frame::build(
7851 FrameType::Request,
7852 control_flags(),
7853 0,
7854 0,
7855 9,
7856 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
7857 )
7858 .unwrap(),
7859 )
7860 .await
7861 .unwrap();
7862 let ClientControlResponse::ServerDescribe { machine_id, .. } =
7863 serde_json::from_slice(&described[0].body).unwrap()
7864 else {
7865 panic!("server.describe answered with another shape");
7866 };
7867 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
7868 }
7869
7870 #[test]
7871 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
7872 let registry = Arc::new(Registry::default());
7875 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
7876 crate::daemon_config::StorageConfig::Sqlite {
7877 data_home: std::path::PathBuf::from("/data"),
7878 },
7879 ));
7880
7881 let responses = handler
7882 .handle_control(
7883 ConnectionId::new(1),
7884 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
7885 )
7886 .unwrap();
7887 let ack = parse_ack(&responses[0]);
7888 assert_eq!(
7889 ack.storage,
7890 Some(serde_json::json!({
7891 "module_id": "alfonso-routing",
7892 "storage_namespace": "default",
7893 "isolation": { "kind": "module" },
7894 "backend": {
7895 "backend": "sqlite",
7896 "path": "/data/cortexkit/alfonso-routing/store.db"
7897 }
7898 })),
7899 "the delivered descriptor is the module's own sqlite store path"
7900 );
7901 }
7902
7903 #[test]
7904 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
7905 let registry = Arc::new(Registry::default());
7906 let handler = ControlHandler::new(Arc::clone(®istry));
7907 let conn = ConnectionId::new(1);
7908 let responses = handler
7909 .handle_control(
7910 conn,
7911 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
7912 )
7913 .unwrap();
7914 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
7915 let registration = registry.get_module("aft").unwrap().unwrap();
7916 assert_eq!(registration.control_ops, module_baseline_control_ops());
7917
7918 let frame =
7919 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
7920 assert!(handler
7921 .guard_module_control_op(&frame, "aft", "route.bind")
7922 .unwrap()
7923 .is_none());
7924 let error = handler
7925 .guard_module_control_op(&frame, "aft", "test.synthetic")
7926 .unwrap()
7927 .expect("synthetic ungranted op should be rejected");
7928 assert_eq!(error.header.ty, FrameType::Error);
7929 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
7930 }
7931
7932 #[test]
7933 fn hello_control_ops_some_adds_optional_grants() {
7934 let registry = Arc::new(Registry::default());
7935 let handler = ControlHandler::new(Arc::clone(®istry));
7936 handler
7937 .handle_control(
7938 ConnectionId::new(1),
7939 hello_frame_with_control_ops(
7940 "aft",
7941 PROTOCOL_VERSION,
7942 7,
7943 Some(vec![
7944 "future.synthetic".to_string(),
7945 "route.bind".to_string(),
7946 ]),
7947 ),
7948 )
7949 .unwrap();
7950 let registration = registry.get_module("aft").unwrap().unwrap();
7951 assert_eq!(
7952 registration.control_ops,
7953 vec![
7954 "route.bind".to_string(),
7955 "route.status".to_string(),
7956 "future.synthetic".to_string(),
7957 ]
7958 );
7959 let frame =
7960 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
7961 assert!(handler
7962 .guard_module_control_op(&frame, "aft", "future.synthetic")
7963 .unwrap()
7964 .is_none());
7965 }
7966
7967 #[tokio::test]
7968 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
7969 let registry = Arc::new(Registry::default());
7970 let forwarding = Arc::new(ForwardingTable::default());
7971 let handler =
7972 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
7973 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
7974 hello_via_sink(
7975 &handler,
7976 &module_ctx,
7977 &mut module_rx,
7978 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
7979 )
7980 .await;
7981
7982 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
7983 let responses = handler
7984 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
7985 .await
7986 .unwrap();
7987 assert_eq!(responses.len(), 1);
7988 assert_eq!(responses[0].header.ty, FrameType::Error);
7989 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
7990 assert!(module_rx.try_recv().is_err());
7991 }
7992
7993 #[tokio::test]
7994 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
7995 let registry = Arc::new(Registry::default());
7996 let forwarding = Arc::new(ForwardingTable::default());
7997 let handler =
7998 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
7999 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
8000 hello_via_sink(
8001 &handler,
8002 &module_ctx,
8003 &mut module_rx,
8004 hello_frame_with_control_ops(
8005 "aft",
8006 PROTOCOL_VERSION,
8007 7,
8008 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
8009 ),
8010 )
8011 .await;
8012
8013 let project_root = unique_project_root("demux");
8014 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
8015 let route_handler = handler.clone();
8016 let route_task = tokio::spawn(async move {
8017 route_handler
8018 .handle_control_frame(
8019 &route_client_ctx,
8020 route_open_frame(100, "aft", project_root),
8021 )
8022 .await
8023 .unwrap()
8024 });
8025 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8026 .await
8027 .unwrap()
8028 .unwrap();
8029 assert!(matches!(
8030 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
8031 ModuleControlRequest::RouteBind { .. }
8032 ));
8033
8034 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
8035 let health_handler = handler.clone();
8036 let health_task = tokio::spawn(async move {
8037 health_handler
8038 .handle_control_frame(
8039 &health_client_ctx,
8040 supervisor_health_probe_frame(101, "aft"),
8041 )
8042 .await
8043 .unwrap()
8044 });
8045 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8046 .await
8047 .unwrap()
8048 .unwrap();
8049 assert_eq!(
8050 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
8051 ModuleControlRequest::HealthCheck {}
8052 );
8053
8054 handler
8055 .handle_control_frame(
8056 &module_ctx,
8057 health_response(health_frame.header.corr, HealthStatus::Degraded),
8058 )
8059 .await
8060 .unwrap();
8061 let health_response = health_task.await.unwrap();
8062 assert_eq!(health_response.len(), 1);
8063 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
8064 ClientControlResponse::SupervisorHealthProbe {
8065 module_id,
8066 status,
8067 detail,
8068 metrics,
8069 } => {
8070 assert_eq!(module_id, "aft");
8071 assert_eq!(status, HealthStatus::Degraded);
8072 assert_eq!(detail.as_deref(), Some("warming"));
8073 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
8074 }
8075 other => panic!("unexpected health response: {other:?}"),
8076 }
8077
8078 handler
8079 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8080 .await
8081 .unwrap();
8082 let route_response = route_task.await.unwrap();
8083 assert!(route_response.is_empty());
8084 let published = route_client_rx.recv().await.unwrap();
8085 assert!(matches!(
8086 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8087 ClientControlResponse::RouteOpen { .. }
8088 ));
8089 }
8090
8091 async fn relay_route_open(
8096 handler: &ControlHandler,
8097 client_connection: ConnectionId,
8098 client_egress: &FrameSink,
8099 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
8100 corr: u64,
8101 module_id: &str,
8102 project_root_label: &str,
8103 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
8104 let ctx = RouteCtx {
8105 connection_id: client_connection,
8106 egress: client_egress.clone(),
8107 };
8108 let handler = handler.clone();
8109 let project_root = unique_project_root(project_root_label);
8110 let module_id = module_id.to_string();
8111 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
8112 let task = tokio::spawn(async move {
8113 let _guard = tracing::dispatcher::set_default(&dispatch);
8114 handler
8115 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
8116 .await
8117 .unwrap()
8118 });
8119 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
8120 .await
8121 .expect("module receives the relayed route.bind")
8122 .expect("module egress is open");
8123 (task, bind.frame)
8124 }
8125
8126 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
8127 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
8128 ModuleControlRequest::RouteBind {
8129 route_channel,
8130 epoch,
8131 ..
8132 } => (route_channel, epoch),
8133 other => panic!("expected a route.bind request, got {other:?}"),
8134 }
8135 }
8136
8137 fn published_route(frame: &Frame) -> (u16, u32) {
8138 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
8139 ClientControlResponse::RouteOpen {
8140 route_channel,
8141 route_epoch,
8142 } => (route_channel, route_epoch),
8143 other => panic!("expected a route.open response, got {other:?}"),
8144 }
8145 }
8146
8147 #[tokio::test]
8167 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
8168 let registry = Arc::new(Registry::default());
8169 let forwarding = Arc::new(ForwardingTable::default());
8170 let handler =
8171 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8172
8173 let module_connection = ConnectionId::new(30);
8174 let (module_ctx, mut module_rx) = route_ctx(module_connection);
8175 hello_via_sink(
8176 &handler,
8177 &module_ctx,
8178 &mut module_rx,
8179 hello_frame("aft", PROTOCOL_VERSION, 7),
8180 )
8181 .await;
8182
8183 let dying_client = ConnectionId::new(31);
8184 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
8185
8186 let (first_task, first_bind) = relay_route_open(
8189 &handler,
8190 dying_client,
8191 &dying_ctx.egress,
8192 &mut module_rx,
8193 100,
8194 "aft",
8195 "closing-first",
8196 )
8197 .await;
8198 handler
8199 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
8200 .await
8201 .unwrap();
8202 assert!(first_task.await.unwrap().is_empty());
8203 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
8204
8205 let (second_task, second_bind) = relay_route_open(
8207 &handler,
8208 dying_client,
8209 &dying_ctx.egress,
8210 &mut module_rx,
8211 101,
8212 "aft",
8213 "closing-second",
8214 )
8215 .await;
8216 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
8217
8218 assert!(forwarding
8221 .escalate_client_delivery_failure(
8222 dying_client,
8223 first_channel,
8224 first_epoch,
8225 CloseReason::new(
8226 "module_to_client_delivery_failed",
8227 "client egress refused a module frame",
8228 ),
8229 crate::forwarding::UndeliveredFrame {
8230 module_id: None,
8231 sink: &dying_ctx.egress,
8232 },
8233 )
8234 .unwrap());
8235 assert!(!dying_ctx.egress.is_closed());
8236
8237 let ack = handler
8239 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
8240 .await;
8241 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
8242 if module_loop_error.is_some() {
8243 handler.cleanup_connection(module_connection).unwrap();
8247 }
8248 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8252 .await
8253 .ok()
8254 .flatten();
8255
8256 assert!(
8258 registry
8259 .get_module_by_connection(module_connection)
8260 .unwrap()
8261 .is_some(),
8262 "one client's closing connection ended the shared module connection: \
8263 {module_loop_error:?}"
8264 );
8265 let cotenant = ConnectionId::new(32);
8267 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
8268 let (cotenant_task, cotenant_bind) = relay_route_open(
8269 &handler,
8270 cotenant,
8271 &cotenant_ctx.egress,
8272 &mut module_rx,
8273 102,
8274 "aft",
8275 "closing-cotenant",
8276 )
8277 .await;
8278 handler
8279 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
8280 .await
8281 .unwrap();
8282 assert!(cotenant_task.await.unwrap().is_empty());
8283 let (cotenant_channel, cotenant_epoch) =
8284 published_route(&cotenant_rx.recv().await.unwrap());
8285 assert!(matches!(
8286 forwarding
8287 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
8288 .unwrap(),
8289 DataRoute::Client(DataRouteState::Bound(_))
8290 ));
8291
8292 let goodbye = post_ack_module_frame
8295 .expect("module receives a GOODBYE for the abandoned route channel");
8296 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
8297 assert_eq!(goodbye.header.channel, abandoned_channel);
8298 assert_eq!(goodbye.header.epoch, abandoned_epoch);
8299
8300 assert!(dying_rx.try_recv().is_err());
8304 let second_response = second_task.await.unwrap();
8305 assert_eq!(second_response.len(), 1);
8306 assert_eq!(
8307 parse_error(&second_response[0])["code"],
8308 "target_unavailable"
8309 );
8310 }
8311
8312 #[test]
8319 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
8320 let handler = ControlHandler::default();
8321 let module_connection = ConnectionId::new(30);
8322 let client_connection = ConnectionId::new(31);
8323
8324 handler
8325 .refuse_to_end_module_connection_for_a_client(
8326 module_connection,
8327 77,
8328 ForwardingError::ConnectionClosing {
8329 connection_id: client_connection,
8330 },
8331 )
8332 .expect("a closing client must never end the module connection");
8333
8334 assert!(matches!(
8335 handler.refuse_to_end_module_connection_for_a_client(
8336 module_connection,
8337 78,
8338 ForwardingError::ConnectionClosing {
8339 connection_id: module_connection,
8340 },
8341 ),
8342 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
8343 connection_id
8344 })) if connection_id == module_connection
8345 ));
8346 assert!(matches!(
8347 handler.refuse_to_end_module_connection_for_a_client(
8348 module_connection,
8349 79,
8350 ForwardingError::Poisoned,
8351 ),
8352 Err(RouterError::Forwarding(ForwardingError::Poisoned))
8353 ));
8354 assert!(matches!(
8355 handler.refuse_to_end_module_connection_for_a_client(
8356 module_connection,
8357 80,
8358 ForwardingError::StaleModuleEndpoint,
8359 ),
8360 Err(RouterError::Forwarding(
8361 ForwardingError::StaleModuleEndpoint
8362 ))
8363 ));
8364 }
8365
8366 #[tokio::test]
8379 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
8380 let registry = Arc::new(Registry::default());
8381 let forwarding = Arc::new(ForwardingTable::default());
8382 let supervisor = SupervisorHandle::new();
8383 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8384 let handler =
8385 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8386 .with_supervisor(supervisor);
8387
8388 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
8389 hello_via_sink(
8390 &handler,
8391 &target_ctx,
8392 &mut target_rx,
8393 hello_frame("target", PROTOCOL_VERSION, 1),
8394 )
8395 .await;
8396
8397 let wrong_nonce = handler
8401 .handle_control_frame(
8402 &route_ctx(ConnectionId::new(91)).0,
8403 route_open_frame_with_admission_facts(
8404 20,
8405 "target",
8406 unique_project_root("admission-facts"),
8407 Some(subc_control::ConsumerIdentity {
8408 module_id: "fed".to_string(),
8409 launch_nonce: "not-the-real-nonce".to_string(),
8410 }),
8411 None,
8412 ),
8413 )
8414 .await
8415 .unwrap();
8416 assert_eq!(
8417 parse_error(&wrong_nonce[0])["code"],
8418 "bad_consumer_identity",
8419 "a mismatched launch nonce must be refused, not stamped Reserved"
8420 );
8421
8422 let never_spawned = handler
8426 .handle_control_frame(
8427 &route_ctx(ConnectionId::new(92)).0,
8428 route_open_frame_with_admission_facts(
8429 21,
8430 "target",
8431 unique_project_root("admission-facts"),
8432 Some(subc_control::ConsumerIdentity {
8433 module_id: "never-spawned".to_string(),
8434 launch_nonce: "any-nonce".to_string(),
8435 }),
8436 None,
8437 ),
8438 )
8439 .await
8440 .unwrap();
8441 assert_eq!(
8442 parse_error(&never_spawned[0])["code"],
8443 "bad_consumer_identity",
8444 "an unspawned module_id must be refused rather than accepted for lack of a record"
8445 );
8446 }
8447
8448 #[tokio::test]
8471 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
8472 let registry = Arc::new(Registry::default());
8473 let forwarding = Arc::new(ForwardingTable::default());
8474 let supervisor = SupervisorHandle::new();
8475 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8476 let handler =
8477 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8478 .with_supervisor(supervisor);
8479
8480 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
8481 hello_via_sink(
8482 &handler,
8483 &target_ctx,
8484 &mut target_rx,
8485 hello_frame("target", PROTOCOL_VERSION, 1),
8486 )
8487 .await;
8488
8489 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
8490 let route_handler = handler.clone();
8491 let route_task = tokio::spawn(async move {
8492 route_handler
8493 .handle_control_frame(
8494 &client_ctx,
8495 route_open_frame_with_admission_facts(
8496 30,
8497 "target",
8498 unique_project_root("admission-facts"),
8499 Some(subc_control::ConsumerIdentity {
8500 module_id: "fed".to_string(),
8501 launch_nonce: "fed-nonce".to_string(),
8502 }),
8503 None,
8504 ),
8505 )
8506 .await
8507 .unwrap()
8508 });
8509
8510 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8517 .await
8518 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
8519 .expect("module control channel closed before route.bind");
8520 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8521 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
8522 panic!("expected route.bind")
8523 };
8524 assert_eq!(
8525 principal,
8526 Some(Principal::Reserved {
8527 module_id: "fed".to_string()
8528 }),
8529 "a correctly attested consumer must be stamped Reserved for its own id"
8530 );
8531
8532 handler
8533 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8534 .await
8535 .unwrap();
8536 assert!(route_task.await.unwrap().is_empty());
8537 assert!(
8538 matches!(
8539 serde_json::from_slice::<ClientControlResponse>(
8540 &client_rx.recv().await.unwrap().body
8541 )
8542 .unwrap(),
8543 ClientControlResponse::RouteOpen { .. }
8544 ),
8545 "the route must actually open, not merely avoid an error"
8546 );
8547 }
8548
8549 #[tokio::test(start_paused = true)]
8550 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
8551 let registry = Arc::new(Registry::default());
8552 let forwarding = Arc::new(ForwardingTable::default());
8553 let supervisor = SupervisorHandle::new();
8554 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8555 let handler =
8556 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8557 .with_supervisor(supervisor);
8558
8559 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
8560 hello_via_sink(
8561 &handler,
8562 &target_ctx,
8563 &mut target_rx,
8564 hello_frame("target", PROTOCOL_VERSION, 1),
8565 )
8566 .await;
8567
8568 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
8569 let direct_handler = handler.clone();
8570 let direct_open = tokio::spawn(async move {
8571 direct_handler
8572 .handle_control_frame(
8573 &direct_ctx,
8574 route_open_frame(2, "target", unique_project_root("route-census-direct")),
8575 )
8576 .await
8577 .unwrap()
8578 });
8579 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8580 .await
8581 .expect("no direct route.bind within 5s")
8582 .expect("target control channel closed before direct route.bind");
8583 handler
8584 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
8585 .await
8586 .unwrap();
8587 assert!(direct_open.await.unwrap().is_empty());
8588 let _ = direct_rx.recv().await.unwrap();
8589
8590 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
8591 let reserved_handler = handler.clone();
8592 let reserved_open = tokio::spawn(async move {
8593 reserved_handler
8594 .handle_control_frame(
8595 &reserved_ctx,
8596 route_open_frame_with_admission_facts(
8597 3,
8598 "target",
8599 unique_project_root("admission-facts"),
8600 Some(ConsumerIdentity {
8601 module_id: "fed".to_string(),
8602 launch_nonce: "fed-nonce".to_string(),
8603 }),
8604 None,
8605 ),
8606 )
8607 .await
8608 .unwrap()
8609 });
8610 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8611 .await
8612 .expect("no reserved route.bind within 5s")
8613 .expect("target control channel closed before reserved route.bind");
8614 handler
8615 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
8616 .await
8617 .unwrap();
8618 assert!(reserved_open.await.unwrap().is_empty());
8619 let _ = reserved_rx.recv().await.unwrap();
8620
8621 forwarding
8622 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
8623 .unwrap();
8624 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
8625 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
8626 module_id: Some("target".to_string()),
8627 })
8628 .unwrap();
8629 let census_frame =
8630 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
8631 let response = handler
8632 .handle_control_frame(&census_ctx, census_frame)
8633 .await
8634 .unwrap()
8635 .pop()
8636 .unwrap();
8637 let actual: Value = serde_json::from_slice(&response.body).unwrap();
8638 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
8639 assert!(matches!(
8640 decoded,
8641 ClientControlResponse::SupervisorRoutes { .. }
8642 ));
8643 let routes = actual["modules"][0]["routes"].as_array().unwrap();
8644 assert_eq!(routes.len(), 2);
8645 assert!(routes.iter().all(|route| route["draining"] == true));
8646 assert!(
8649 routes.iter().all(|route| route["drain_reason"] == "reload"),
8650 "draining routes must name the drain's reason: {routes:?}"
8651 );
8652 assert!(routes.iter().any(|route| {
8653 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
8654 }));
8655 assert!(routes.iter().any(|route| {
8656 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
8657 }));
8658
8659 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8660 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
8661 if std::env::var_os("UPDATE_GOLDEN").is_some() {
8662 std::fs::write(
8663 &golden_path,
8664 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
8665 )
8666 .unwrap();
8667 }
8668 let expected: Value =
8669 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
8670 assert_eq!(actual, expected);
8671 }
8672
8673 async fn query_live_roots(
8674 handler: &ControlHandler,
8675 module_ctx: &RouteCtx,
8676 ) -> ModuleControlResponseToModule {
8677 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
8678 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
8679 let response = handler
8680 .handle_control_frame(module_ctx, frame)
8681 .await
8682 .unwrap()
8683 .pop()
8684 .unwrap();
8685 serde_json::from_slice(&response.body).unwrap()
8686 }
8687
8688 #[tokio::test(start_paused = true)]
8689 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
8690 let registry = Arc::new(Registry::default());
8691 let forwarding = Arc::new(ForwardingTable::default());
8692 let handler = ControlHandler::with_forwarding(registry, forwarding);
8693 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
8694 hello_via_sink(
8695 &handler,
8696 &target_ctx,
8697 &mut target_rx,
8698 hello_frame("target", PROTOCOL_VERSION, 1),
8699 )
8700 .await;
8701 let root = unique_project_root("live-roots-known");
8702 let path = ProjectRootId::from_path_allowing_missing(root.path())
8703 .unwrap()
8704 .as_path()
8705 .to_path_buf();
8706 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
8707 let open_handler = handler.clone();
8708 let opened = tokio::spawn(async move {
8709 open_handler
8710 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
8711 .await
8712 .unwrap()
8713 });
8714 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8715 .await
8716 .unwrap()
8717 .unwrap();
8718 handler
8719 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
8720 .await
8721 .unwrap();
8722 assert!(opened.await.unwrap().is_empty());
8723 let _ = client_rx.recv().await.unwrap();
8724
8725 let root = unique_project_root("live-roots-pending");
8726 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
8727 .unwrap()
8728 .as_path()
8729 .to_path_buf();
8730 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
8731 let open_handler = handler.clone();
8732 let pending = tokio::spawn(async move {
8733 open_handler
8734 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
8735 .await
8736 .unwrap()
8737 });
8738 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8739 .await
8740 .unwrap()
8741 .unwrap();
8742 let actual = query_live_roots(&handler, &target_ctx).await;
8743 let ModuleControlResponseToModule::LiveRoots {
8744 roots,
8745 unknown_root_bindings,
8746 total_bindings,
8747 } = actual
8748 else {
8749 panic!("expected live roots")
8750 };
8751 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
8752 assert_eq!(unknown_root_bindings, 0);
8753 assert_eq!(
8754 roots.len(),
8755 2,
8756 "root-known arm must retain each canonical root"
8757 );
8758 assert_eq!(
8759 total_bindings,
8760 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
8761 );
8762 let counts = roots
8763 .iter()
8764 .map(|root| (root.project_root.clone(), root.bound, root.pending))
8765 .collect::<Vec<_>>();
8766 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
8767 expected.sort_by(|a, b| a.0.cmp(&b.0));
8768 assert_eq!(
8769 counts, expected,
8770 "roots must sort by path and count pending separately"
8771 );
8772 handler
8773 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
8774 .await
8775 .unwrap();
8776 assert!(pending.await.unwrap().is_empty());
8777 }
8778
8779 #[tokio::test(start_paused = true)]
8780 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
8781 let forwarding = Arc::new(ForwardingTable::default());
8782 let handler =
8783 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
8784 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
8785 hello_via_sink(
8786 &handler,
8787 &target_ctx,
8788 &mut target_rx,
8789 hello_frame("target", PROTOCOL_VERSION, 1),
8790 )
8791 .await;
8792 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
8793 let pending = forwarding
8794 .begin_route_bind_relay_for_test(
8795 client_ctx.connection_id,
8796 client_ctx.egress.clone(),
8797 2,
8798 "target",
8799 )
8800 .unwrap();
8801 forwarding
8802 .complete_pending_relay(
8803 target_ctx.connection_id,
8804 pending.corr,
8805 RouteBindRelayOutcome::Accepted,
8806 )
8807 .unwrap();
8808 let actual = query_live_roots(&handler, &target_ctx).await;
8809 let ModuleControlResponseToModule::LiveRoots {
8810 roots,
8811 unknown_root_bindings,
8812 total_bindings,
8813 } = actual
8814 else {
8815 panic!("expected live roots")
8816 };
8817 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
8818 assert_eq!(
8819 unknown_root_bindings, 1,
8820 "unknown-root arm must not read as no bindings"
8821 );
8822 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
8823 assert_eq!(
8824 total_bindings,
8825 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
8826 );
8827 }
8828
8829 #[tokio::test(start_paused = true)]
8837 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
8838 let forwarding = Arc::new(ForwardingTable::default());
8839 let handler =
8840 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
8841 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
8842 let replies = handler
8843 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
8844 .await
8845 .unwrap();
8846 let queued_by_hello = module_rx.len();
8847
8848 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
8849 let open_handler = handler.clone();
8850 let open = tokio::spawn(async move {
8851 open_handler
8852 .handle_control_frame(
8853 &client_ctx,
8854 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
8855 )
8856 .await
8857 .unwrap()
8858 });
8859 let mut spins = 0;
8861 while module_rx.len() == queued_by_hello {
8862 spins += 1;
8863 assert!(spins < 10_000, "route.open never queued a route.bind");
8864 tokio::task::yield_now().await;
8865 }
8866
8867 for reply in replies {
8869 module_ctx.egress.send(reply).await.unwrap();
8870 }
8871
8872 let first = module_rx.recv().await.unwrap().frame;
8873 assert_eq!(
8874 first.header.ty,
8875 FrameType::HelloAck,
8876 "the first frame a registering module reads must be its HELLO_ACK"
8877 );
8878 assert_eq!(first.header.corr, 7);
8879 let second = module_rx.recv().await.unwrap().frame;
8880 assert_eq!(second.header.ty, FrameType::Request);
8881 assert!(
8882 matches!(
8883 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
8884 ModuleControlRequest::RouteBind { .. }
8885 ),
8886 "the route.bind follows the ack"
8887 );
8888 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
8889
8890 handler
8891 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
8892 .await
8893 .unwrap();
8894 assert!(open.await.unwrap().is_empty());
8895 let _ = client_rx.recv().await.unwrap();
8896 }
8897
8898 #[tokio::test(start_paused = true)]
8899 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
8900 let handler = ControlHandler::with_forwarding(
8901 Arc::new(Registry::default()),
8902 Arc::new(ForwardingTable::default()),
8903 );
8904 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
8905 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
8906 hello_via_sink(
8907 &handler,
8908 &first_ctx,
8909 &mut first_rx,
8910 hello_frame("first", PROTOCOL_VERSION, 1),
8911 )
8912 .await;
8913 hello_via_sink(
8914 &handler,
8915 &second_ctx,
8916 &mut second_rx,
8917 hello_frame("second", PROTOCOL_VERSION, 2),
8918 )
8919 .await;
8920 let root = unique_project_root("second-only");
8921 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
8922 let cloned = handler.clone();
8923 let open = tokio::spawn(async move {
8924 cloned
8925 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
8926 .await
8927 .unwrap()
8928 });
8929 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
8930 .await
8931 .unwrap()
8932 .unwrap();
8933 let first = query_live_roots(&handler, &first_ctx).await;
8934 let second = query_live_roots(&handler, &second_ctx).await;
8935 assert!(
8936 matches!(
8937 first,
8938 ModuleControlResponseToModule::LiveRoots {
8939 total_bindings: 0,
8940 ..
8941 }
8942 ),
8943 "cross-module scope must not expose another module's roots"
8944 );
8945 assert!(
8946 matches!(
8947 second,
8948 ModuleControlResponseToModule::LiveRoots {
8949 total_bindings: 1,
8950 ..
8951 }
8952 ),
8953 "second module must see its pending route"
8954 );
8955 handler
8956 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
8957 .await
8958 .unwrap();
8959 assert!(open.await.unwrap().is_empty());
8960 }
8961
8962 #[tokio::test(start_paused = true)]
8963 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
8964 let handler = ControlHandler::with_forwarding(
8965 Arc::new(Registry::default()),
8966 Arc::new(ForwardingTable::default()),
8967 );
8968 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
8969 hello_via_sink(
8970 &handler,
8971 &target_ctx,
8972 &mut target_rx,
8973 hello_frame("target", PROTOCOL_VERSION, 1),
8974 )
8975 .await;
8976 let actual = query_live_roots(&handler, &target_ctx).await;
8977 let ModuleControlResponseToModule::LiveRoots {
8978 roots,
8979 unknown_root_bindings,
8980 total_bindings,
8981 } = actual
8982 else {
8983 panic!("expected live roots")
8984 };
8985 assert!(roots.is_empty());
8986 assert_eq!(unknown_root_bindings, 0);
8987 assert_eq!(total_bindings, 0);
8988 assert_eq!(
8989 total_bindings,
8990 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
8991 );
8992 }
8993
8994 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
9001 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9002 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
9003 let text = std::fs::read_to_string(&path)
9004 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
9005 let vectors: Vec<(String, Value)> = text
9006 .lines()
9007 .filter(|line| !line.trim().is_empty())
9008 .map(|line| {
9009 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
9010 let id = entry["corpus_id"]
9011 .as_str()
9012 .expect("every vector carries a corpus_id")
9013 .to_string();
9014 (id, entry["package"].clone())
9015 })
9016 .collect();
9017 assert_eq!(
9021 vectors.len(),
9022 3,
9023 "vendored fed corpus changed size; re-sync from subc-federation"
9024 );
9025
9026 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
9041 let richest = vectors
9042 .iter()
9043 .filter_map(|(_, package)| package.as_object())
9044 .map(|object| {
9045 object
9046 .keys()
9047 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
9048 .count()
9049 })
9050 .max()
9051 .unwrap_or(0);
9052 assert!(
9053 richest >= 2,
9054 "vendored corpus no longer carries a package with unmodelled fields, \
9055 so the relay test can no longer distinguish a verbatim relay from a lossy one"
9056 );
9057
9058 vectors
9059 }
9060
9061 #[tokio::test]
9069 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
9070 for (corpus_id, package) in fed_admission_facts_vectors() {
9071 let registry = Arc::new(Registry::default());
9072 let forwarding = Arc::new(ForwardingTable::default());
9073 let supervisor = SupervisorHandle::new();
9074 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9075 let handler =
9076 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9077 .with_supervisor(supervisor)
9078 .with_admission_facts_config(
9079 Some("fed".to_string()),
9080 Some(vec!["target".to_string()]),
9081 );
9082
9083 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
9084 hello_via_sink(
9085 &handler,
9086 &target_ctx,
9087 &mut target_rx,
9088 hello_frame("target", PROTOCOL_VERSION, 1),
9089 )
9090 .await;
9091
9092 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
9093 let route_handler = handler.clone();
9094 let expected = package.clone();
9095 let route_task = tokio::spawn(async move {
9096 route_handler
9097 .handle_control_frame(
9098 &client_ctx,
9099 route_open_frame_with_admission_facts(
9100 20,
9101 "target",
9102 unique_project_root("admission-facts"),
9103 Some(subc_control::ConsumerIdentity {
9104 module_id: "fed".to_string(),
9105 launch_nonce: "fed-nonce".to_string(),
9106 }),
9107 Some(package),
9108 ),
9109 )
9110 .await
9111 .unwrap()
9112 });
9113
9114 let bind_frame = target_rx.recv().await.unwrap();
9115 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9116 let ModuleControlRequest::RouteBind {
9117 admission_facts, ..
9118 } = bind
9119 else {
9120 panic!("{corpus_id}: expected route.bind")
9121 };
9122 assert_eq!(
9123 admission_facts,
9124 Some(expected),
9125 "{corpus_id}: relay must not add, drop or reshape any field"
9126 );
9127
9128 handler
9129 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
9130 .await
9131 .unwrap();
9132 route_task.await.unwrap();
9133 }
9134 }
9135
9136 #[tokio::test]
9137 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
9138 let registry = Arc::new(Registry::default());
9139 let forwarding = Arc::new(ForwardingTable::default());
9140 let supervisor = SupervisorHandle::new();
9141 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9142 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
9143 let handler =
9144 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9145 .with_supervisor(supervisor)
9146 .with_admission_facts_config(
9147 Some("fed".to_string()),
9148 Some(vec!["target".to_string()]),
9149 );
9150
9151 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
9152 hello_via_sink(
9153 &handler,
9154 &target_ctx,
9155 &mut target_rx,
9156 hello_frame("target", PROTOCOL_VERSION, 1),
9157 )
9158 .await;
9159 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
9160 hello_via_sink(
9161 &handler,
9162 &other_ctx,
9163 &mut other_rx,
9164 hello_frame("other", PROTOCOL_VERSION, 2),
9165 )
9166 .await;
9167
9168 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
9169 let expected_facts = facts.clone();
9170 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
9171 let route_handler = handler.clone();
9172 let route_task = tokio::spawn(async move {
9173 route_handler
9174 .handle_control_frame(
9175 &client_ctx,
9176 route_open_frame_with_admission_facts(
9177 10,
9178 "target",
9179 unique_project_root("admission-facts"),
9180 Some(subc_control::ConsumerIdentity {
9181 module_id: "fed".to_string(),
9182 launch_nonce: "fed-nonce".to_string(),
9183 }),
9184 Some(facts.clone()),
9185 ),
9186 )
9187 .await
9188 .unwrap()
9189 });
9190 let bind_frame = target_rx.recv().await.unwrap();
9191 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9192 let ModuleControlRequest::RouteBind {
9193 admission_facts, ..
9194 } = bind
9195 else {
9196 panic!("expected route.bind")
9197 };
9198 assert_eq!(admission_facts, Some(expected_facts));
9199 handler
9200 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
9201 .await
9202 .unwrap();
9203 assert!(route_task.await.unwrap().is_empty());
9204 assert!(matches!(
9205 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
9206 .unwrap(),
9207 ClientControlResponse::RouteOpen { .. }
9208 ));
9209
9210 let direct = handler
9211 .handle_control_frame(
9212 &route_ctx(ConnectionId::new(73)).0,
9213 route_open_frame_with_admission_facts(
9214 11,
9215 "target",
9216 unique_project_root("admission-facts"),
9217 None,
9218 Some(json!({"x": 1})),
9219 ),
9220 )
9221 .await
9222 .unwrap();
9223 assert_eq!(
9224 parse_error(&direct[0])["code"],
9225 "admission_facts_not_permitted"
9226 );
9227
9228 let different_reserved = handler
9229 .handle_control_frame(
9230 &route_ctx(ConnectionId::new(77)).0,
9231 route_open_frame_with_admission_facts(
9232 15,
9233 "target",
9234 unique_project_root("admission-facts"),
9235 Some(subc_control::ConsumerIdentity {
9236 module_id: "other".to_string(),
9237 launch_nonce: "other-nonce".to_string(),
9238 }),
9239 Some(json!({"x": 1})),
9240 ),
9241 )
9242 .await
9243 .unwrap();
9244 assert_eq!(
9245 parse_error(&different_reserved[0])["code"],
9246 "admission_facts_not_permitted"
9247 );
9248
9249 let other_target = handler
9250 .handle_control_frame(
9251 &route_ctx(ConnectionId::new(74)).0,
9252 route_open_frame_with_admission_facts(
9253 12,
9254 "other",
9255 unique_project_root("admission-facts"),
9256 Some(subc_control::ConsumerIdentity {
9257 module_id: "fed".to_string(),
9258 launch_nonce: "fed-nonce".to_string(),
9259 }),
9260 Some(json!({"x": 1})),
9261 ),
9262 )
9263 .await
9264 .unwrap();
9265 assert_eq!(
9266 parse_error(&other_target[0])["code"],
9267 "admission_facts_target_not_allowed"
9268 );
9269
9270 let nonexistent = handler
9271 .handle_control_frame(
9272 &route_ctx(ConnectionId::new(75)).0,
9273 route_open_frame_with_admission_facts(
9274 13,
9275 "missing",
9276 unique_project_root("admission-facts"),
9277 None,
9278 Some(json!({"x": 1})),
9279 ),
9280 )
9281 .await
9282 .unwrap();
9283 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
9284
9285 let described = handler
9286 .handle_control_frame(
9287 &route_ctx(ConnectionId::new(76)).0,
9288 Frame::build(
9289 FrameType::Request,
9290 control_flags(),
9291 0,
9292 0,
9293 14,
9294 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
9295 )
9296 .unwrap(),
9297 )
9298 .await
9299 .unwrap();
9300 let ClientControlResponse::ServerDescribe { capabilities, .. } =
9301 serde_json::from_slice(&described[0].body).unwrap()
9302 else {
9303 panic!("expected server.describe response")
9304 };
9305 assert!(capabilities
9306 .iter()
9307 .any(|cap| cap == "admission_facts_relay_v1"));
9308 }
9309
9310 #[tokio::test]
9311 async fn admission_facts_without_configured_carrier_are_rejected() {
9312 let registry = Arc::new(Registry::default());
9313 let forwarding = Arc::new(ForwardingTable::default());
9314 let handler = ControlHandler::with_forwarding(registry, forwarding);
9315 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
9316 hello_via_sink(
9317 &handler,
9318 &target_ctx,
9319 &mut target_rx,
9320 hello_frame("target", PROTOCOL_VERSION, 1),
9321 )
9322 .await;
9323
9324 let responses = handler
9325 .handle_control_frame(
9326 &route_ctx(ConnectionId::new(79)).0,
9327 route_open_frame_with_admission_facts(
9328 16,
9329 "target",
9330 unique_project_root("admission-facts"),
9331 None,
9332 Some(json!({"x": 1})),
9333 ),
9334 )
9335 .await
9336 .unwrap();
9337 assert_eq!(
9338 parse_error(&responses[0])["code"],
9339 "admission_facts_not_permitted"
9340 );
9341 }
9342
9343 #[tokio::test]
9344 async fn route_open_relays_consumer_capabilities_verbatim() {
9345 let registry = Arc::new(Registry::default());
9346 let forwarding = Arc::new(ForwardingTable::default());
9347 let handler =
9348 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9349 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
9350 hello_via_sink(
9351 &handler,
9352 &module_ctx,
9353 &mut module_rx,
9354 hello_frame("aft", PROTOCOL_VERSION, 7),
9355 )
9356 .await;
9357
9358 let expected = vec!["elicitation".to_string(), "roots".to_string()];
9359 let expected_for_request = expected.clone();
9360 let project_root = unique_project_root("consumer-capabilities-present");
9361 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
9362 let route_handler = handler.clone();
9363 let route_task = tokio::spawn(async move {
9364 route_handler
9365 .handle_control_frame(
9366 &client_ctx,
9367 route_open_frame_with_consumer_capabilities(
9368 401,
9369 "aft",
9370 project_root,
9371 Some(expected_for_request),
9372 ),
9373 )
9374 .await
9375 .unwrap()
9376 });
9377 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9378 .await
9379 .unwrap()
9380 .unwrap();
9381 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9382 let ModuleControlRequest::RouteBind {
9383 consumer_capabilities,
9384 ..
9385 } = bind
9386 else {
9387 panic!("expected route.bind request, got {bind:?}");
9388 };
9389 assert_eq!(consumer_capabilities, Some(expected.clone()));
9390
9391 handler
9392 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9393 .await
9394 .unwrap();
9395 let route_response = route_task.await.unwrap();
9396 assert!(route_response.is_empty());
9397 let published = client_rx.recv().await.unwrap();
9398 assert!(matches!(
9399 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9400 ClientControlResponse::RouteOpen { .. }
9401 ));
9402 }
9403
9404 #[tokio::test]
9405 async fn route_open_without_consumer_capabilities_relays_none() {
9406 let registry = Arc::new(Registry::default());
9407 let forwarding = Arc::new(ForwardingTable::default());
9408 let handler =
9409 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9410 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
9411 hello_via_sink(
9412 &handler,
9413 &module_ctx,
9414 &mut module_rx,
9415 hello_frame("aft", PROTOCOL_VERSION, 7),
9416 )
9417 .await;
9418
9419 let project_root = unique_project_root("consumer-capabilities-absent");
9420 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
9421 let route_handler = handler.clone();
9422 let route_task = tokio::spawn(async move {
9423 route_handler
9424 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
9425 .await
9426 .unwrap()
9427 });
9428 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9429 .await
9430 .unwrap()
9431 .unwrap();
9432 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9433 let ModuleControlRequest::RouteBind {
9434 consumer_capabilities,
9435 ..
9436 } = bind
9437 else {
9438 panic!("expected route.bind request, got {bind:?}");
9439 };
9440 assert_eq!(consumer_capabilities, None);
9441
9442 handler
9443 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9444 .await
9445 .unwrap();
9446 let route_response = route_task.await.unwrap();
9447 assert!(route_response.is_empty());
9448 let published = client_rx.recv().await.unwrap();
9449 assert!(matches!(
9450 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9451 ClientControlResponse::RouteOpen { .. }
9452 ));
9453 }
9454
9455 async fn bind_role_versions_for(
9459 sent: Option<BTreeMap<String, String>>,
9460 connection: u64,
9461 ) -> Option<BTreeMap<String, String>> {
9462 let registry = Arc::new(Registry::default());
9463 let forwarding = Arc::new(ForwardingTable::default());
9464 let handler =
9465 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9466 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
9467 hello_via_sink(
9468 &handler,
9469 &module_ctx,
9470 &mut module_rx,
9471 hello_frame("aft", PROTOCOL_VERSION, 7),
9472 )
9473 .await;
9474 let project_root = unique_project_root("role-versions");
9475 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
9476 let route_handler = handler.clone();
9477 let route_task = tokio::spawn(async move {
9478 route_handler
9479 .handle_control_frame(
9480 &client_ctx,
9481 route_open_frame_with_role_versions(403, "aft", project_root, sent),
9482 )
9483 .await
9484 .unwrap()
9485 });
9486 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9487 .await
9488 .expect("a well-formed route.open reaches the module as a bind")
9489 .unwrap();
9490 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9491 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
9492 panic!("expected route.bind request, got {bind:?}");
9493 };
9494 handler
9495 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9496 .await
9497 .unwrap();
9498 assert!(route_task.await.unwrap().is_empty());
9499 let published = client_rx.recv().await.unwrap();
9500 assert!(matches!(
9501 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9502 ClientControlResponse::RouteOpen { .. }
9503 ));
9504 role_versions
9505 }
9506
9507 #[tokio::test]
9508 async fn route_open_relays_role_versions_verbatim() {
9509 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
9510 assert_eq!(
9511 bind_role_versions_for(Some(sent.clone()), 141).await,
9512 Some(sent)
9513 );
9514 }
9515
9516 #[tokio::test]
9519 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
9520 assert_eq!(bind_role_versions_for(None, 143).await, None);
9521 assert_eq!(
9522 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
9523 None
9524 );
9525 }
9526
9527 #[tokio::test]
9528 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
9529 let registry = Arc::new(Registry::default());
9530 let forwarding = Arc::new(ForwardingTable::default());
9531 let handler =
9532 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9533 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
9534 hello_via_sink(
9535 &handler,
9536 &module_ctx,
9537 &mut module_rx,
9538 hello_frame("aft", PROTOCOL_VERSION, 7),
9539 )
9540 .await;
9541
9542 let nine: BTreeMap<String, String> = (0..9)
9543 .map(|index| (format!("role-{index}"), "v1".to_string()))
9544 .collect();
9545 for (label, malformed) in [
9546 (
9547 "invalid role name",
9548 role_versions(&[("Tool_Provider", "v1")]),
9549 ),
9550 ("invalid version", role_versions(&[("tool-provider", "v0")])),
9551 ("nine entries", nine),
9552 ] {
9553 let responses = tokio::time::timeout(
9556 Duration::from_secs(1),
9557 handler.handle_control_frame(
9558 &route_ctx(ConnectionId::new(148)).0,
9559 route_open_frame_with_role_versions(
9560 404,
9561 "aft",
9562 unique_project_root("role-versions-malformed"),
9563 Some(malformed),
9564 ),
9565 ),
9566 )
9567 .await
9568 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
9569 .unwrap();
9570 assert_eq!(responses.len(), 1, "{label}");
9571 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
9572 let error = parse_error(&responses[0]);
9573 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
9574 assert_eq!(
9575 error["detail"]["field"], "role_versions",
9576 "{label}: {error}"
9577 );
9578 assert!(
9579 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
9580 "{label}: a malformed declaration is terminal"
9581 );
9582 assert!(
9583 module_rx.try_recv().is_err(),
9584 "{label}: the module must never see a bind"
9585 );
9586 }
9587 }
9588
9589 #[tokio::test]
9593 async fn route_role_versions_capability_is_advertised() {
9594 let handler = ControlHandler::new(Arc::new(Registry::default()));
9595 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
9596 let ack = hello_via_sink(
9597 &handler,
9598 &ctx,
9599 &mut rx,
9600 hello_frame("m", PROTOCOL_VERSION, 1),
9601 )
9602 .await;
9603 let ack = parse_ack(&ack);
9604 assert!(
9605 ack.subc_capabilities
9606 .iter()
9607 .any(|c| c == "route-role-versions/v1"),
9608 "{:?}",
9609 ack.subc_capabilities
9610 );
9611
9612 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
9613 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
9614 let reply = handler
9615 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
9616 .await
9617 .unwrap()
9618 .pop()
9619 .unwrap();
9620 let ClientControlResponse::ServerDescribe { capabilities, .. } =
9621 serde_json::from_slice(&reply.body).unwrap()
9622 else {
9623 panic!("not a server.describe reply");
9624 };
9625 assert!(
9626 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
9627 "{capabilities:?}"
9628 );
9629 }
9630
9631 #[tokio::test]
9632 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
9633 let registry = Arc::new(Registry::default());
9634 let forwarding = Arc::new(ForwardingTable::default());
9635 let handler =
9636 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9637 .with_health_probe_timeout(Duration::from_secs(5));
9638 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
9639 hello_via_sink(
9640 &handler,
9641 &module_ctx,
9642 &mut module_rx,
9643 non_routable_hello_frame_with_control_ops(
9644 "mcp",
9645 300,
9646 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9647 ),
9648 )
9649 .await;
9650 assert!(registry
9651 .get_module("mcp")
9652 .unwrap()
9653 .unwrap()
9654 .manifest
9655 .provides
9656 .is_empty());
9657
9658 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
9659 let route_response = handler
9660 .handle_control_frame(
9661 &route_client_ctx,
9662 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
9663 )
9664 .await
9665 .unwrap();
9666 assert_eq!(route_response[0].header.ty, FrameType::Error);
9667 assert_eq!(
9668 parse_error(&route_response[0])["code"],
9669 "target_unavailable"
9670 );
9671 assert!(parse_error(&route_response[0])["message"]
9672 .as_str()
9673 .unwrap()
9674 .contains("does not provide the requested target"));
9675 assert!(module_rx.try_recv().is_err());
9676
9677 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
9678 let health_handler = handler.clone();
9679 let health_task = tokio::spawn(async move {
9680 health_handler
9681 .handle_control_frame(
9682 &health_client_ctx,
9683 supervisor_health_probe_frame(302, "mcp"),
9684 )
9685 .await
9686 .unwrap()
9687 });
9688 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9689 .await
9690 .unwrap()
9691 .unwrap();
9692 assert_eq!(
9693 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
9694 ModuleControlRequest::HealthCheck {}
9695 );
9696 handler
9697 .handle_control_frame(
9698 &module_ctx,
9699 health_response(health_frame.header.corr, HealthStatus::Ok),
9700 )
9701 .await
9702 .unwrap();
9703 let health_response = health_task.await.unwrap();
9704 assert_eq!(health_response[0].header.ty, FrameType::Response);
9705 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
9706 ClientControlResponse::SupervisorHealthProbe {
9707 module_id, status, ..
9708 } => {
9709 assert_eq!(module_id, "mcp");
9710 assert_eq!(status, HealthStatus::Ok);
9711 }
9712 other => panic!("unexpected health response: {other:?}"),
9713 }
9714
9715 forwarding
9720 .cleanup_connection(module_ctx.connection_id)
9721 .unwrap();
9722 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
9723 let cleanup_response = tokio::time::timeout(
9724 Duration::from_millis(200),
9725 handler.handle_control_frame(
9726 &cleanup_probe_ctx,
9727 supervisor_health_probe_frame(303, "mcp"),
9728 ),
9729 )
9730 .await
9731 .expect("probe should fail immediately when the control lane is gone")
9732 .unwrap();
9733 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
9734 assert_eq!(
9735 parse_error(&cleanup_response[0])["code"],
9736 "target_unavailable"
9737 );
9738 assert!(parse_error(&cleanup_response[0])["message"]
9739 .as_str()
9740 .unwrap()
9741 .contains("no module connection"));
9742
9743 handler
9744 .cleanup_connection(module_ctx.connection_id)
9745 .unwrap();
9746 }
9747
9748 #[tokio::test]
9749 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
9750 let registry = Arc::new(Registry::default());
9751 let supervisor_handle = SupervisorHandle::new();
9752 let supervisor =
9753 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
9754 .with_handle(supervisor_handle.clone())
9755 .with_connection_file_path(
9756 std::env::temp_dir()
9757 .join(format!("subc-route-open-warming-{}", std::process::id())),
9758 );
9759 let module = supervisor
9760 .supervise_configured(
9761 ModuleSpec {
9762 module_id: "warming".to_string(),
9763 program: fake_aft_stub_path(),
9764 args: Vec::new(),
9765 env: Vec::new(),
9766 reserved: false,
9767 reserved_prefixes: Vec::new(),
9768 protocol: ModuleProtocol::Subc,
9769 overlap: Default::default(),
9770 },
9771 true,
9772 )
9773 .unwrap();
9774 assert_eq!(module.state().unwrap(), ModuleState::Running);
9775
9776 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9777 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
9778 let response = handler
9779 .handle_control_frame(
9780 &ctx,
9781 route_open_frame(304, "warming", unique_project_root("warming")),
9782 )
9783 .await
9784 .unwrap();
9785 module.stop().await.unwrap();
9786
9787 assert_eq!(response[0].header.ty, FrameType::Error);
9788 let error = parse_error(&response[0]);
9789 assert_eq!(error["code"], "module_warming");
9790 assert!(error["message"]
9791 .as_str()
9792 .unwrap()
9793 .contains("state=running, enabled=true, live=false"));
9794 }
9795
9796 #[test]
9797 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
9798 let handler = ControlHandler::new(Arc::new(Registry::default()));
9799 let capture = EventCapture::default();
9800 let _subscriber =
9801 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9802 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
9803 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
9804 let pending = (0..limit).collect::<Vec<_>>();
9805 let response = handler
9806 .route_open_capacity_refusal(
9807 &ctx,
9808 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
9809 "busy",
9810 pending.len(),
9811 limit,
9812 )
9813 .unwrap();
9814 assert_eq!(parse_error(&response)["code"], "target_unavailable");
9815 let event = capture
9816 .events()
9817 .into_iter()
9818 .find(|event| {
9819 event.target == "control"
9820 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
9821 })
9822 .expect("connection admission refusal event");
9823 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
9824 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
9825 }
9826
9827 #[test]
9828 fn route_open_target_cap_logs_admission_reason_and_capacity() {
9829 let handler = ControlHandler::new(Arc::new(Registry::default()));
9830 let capture = EventCapture::default();
9831 let _subscriber =
9832 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9833 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
9834 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
9835 let guards = (0..limit)
9836 .map(|_| {
9837 handler
9838 .route_bind_concurrency
9839 .try_admit("busy", limit)
9840 .unwrap()
9841 })
9842 .collect::<Vec<_>>();
9843 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
9844 Err(in_flight) => in_flight,
9845 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
9846 };
9847 let response = handler
9848 .route_open_target_capacity_refusal(
9849 &ctx,
9850 &route_open_frame(397, "busy", unique_project_root("target-cap")),
9851 "busy",
9852 in_flight,
9853 )
9854 .unwrap();
9855 assert_eq!(parse_error(&response)["code"], "target_unavailable");
9856 let event = capture
9857 .events()
9858 .into_iter()
9859 .find(|event| {
9860 event.target == "control"
9861 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
9862 })
9863 .expect("target admission refusal event");
9864 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
9865 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
9866 drop(guards);
9867 }
9868
9869 #[tokio::test]
9874 async fn route_open_refusal_names_the_check_that_refused() {
9875 let handler = ControlHandler::new(Arc::new(Registry::default()));
9876 let capture = EventCapture::default();
9877 let _subscriber =
9878 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9879 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
9880 let response = handler
9881 .handle_control_frame(
9882 &ctx,
9883 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
9884 )
9885 .await
9886 .unwrap();
9887
9888 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
9889 let event = capture
9890 .events()
9891 .into_iter()
9892 .find(|event| {
9893 event.target == "control"
9894 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
9895 })
9896 .expect("route.open refusal event");
9897 assert_eq!(
9898 event.fields.get("reason"),
9899 Some(&"\"not_registered\"".to_string())
9900 );
9901 }
9902
9903 #[tokio::test]
9904 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
9905 let registry = Arc::new(Registry::default());
9906 let supervisor_handle = SupervisorHandle::new();
9907 let supervisor =
9908 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
9909 .with_handle(supervisor_handle.clone())
9910 .with_connection_file_path(std::env::temp_dir().join(format!(
9911 "subc-route-open-refusal-info-{}",
9912 std::process::id()
9913 )));
9914 let module = supervisor
9915 .supervise_configured(
9916 ModuleSpec {
9917 module_id: "warming".to_string(),
9918 program: fake_aft_stub_path(),
9919 args: Vec::new(),
9920 env: Vec::new(),
9921 reserved: false,
9922 reserved_prefixes: Vec::new(),
9923 protocol: ModuleProtocol::Subc,
9924 overlap: Default::default(),
9925 },
9926 true,
9927 )
9928 .unwrap();
9929 assert_eq!(module.state().unwrap(), ModuleState::Running);
9930
9931 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9932 assert!(handler
9933 .counters()
9934 .snapshot()
9935 .get("route_open_refused_by_code")
9936 .is_none());
9937 let capture = EventCapture::default();
9938 let _subscriber =
9939 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9940 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
9941 let response = handler
9942 .handle_control_frame(
9943 &ctx,
9944 route_open_frame(394, "warming", unique_project_root("refusal-info")),
9945 )
9946 .await
9947 .unwrap();
9948 module.stop().await.unwrap();
9949
9950 assert_eq!(parse_error(&response[0])["code"], "module_warming");
9951 let event = capture
9952 .events()
9953 .into_iter()
9954 .find(|event| {
9955 event.target == "control"
9956 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
9957 })
9958 .expect("route.open refusal event");
9959 assert_eq!(
9960 event.fields.get("module_id"),
9961 Some(&"\"warming\"".to_string())
9962 );
9963 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
9964 assert_eq!(
9965 event.fields.get("reason"),
9966 Some(&"\"supervised_not_registered\"".to_string())
9967 );
9968 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
9969 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
9970 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
9971 assert_eq!(
9972 handler.counters().snapshot()["route_open_refused_by_code"],
9973 json!({ "module_warming": 1 })
9974 );
9975 }
9976
9977 const OUTAGE_START: &str = "route.open refusing module: not serving";
9978 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
9979
9980 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
9981 capture
9982 .events()
9983 .into_iter()
9984 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
9985 .collect()
9986 }
9987
9988 fn supervise_stub(
9989 registry: &Arc<Registry>,
9990 module_id: &str,
9991 enabled: bool,
9992 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
9993 let supervisor_handle = SupervisorHandle::new();
9994 let supervisor =
9995 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
9996 .with_handle(supervisor_handle.clone())
9997 .with_connection_file_path(std::env::temp_dir().join(format!(
9998 "subc-route-outage-{module_id}-{}",
9999 std::process::id()
10000 )));
10001 let module = supervisor
10002 .supervise_configured(
10003 ModuleSpec {
10004 module_id: module_id.to_string(),
10005 program: fake_aft_stub_path(),
10006 args: Vec::new(),
10007 env: Vec::new(),
10008 reserved: false,
10009 reserved_prefixes: Vec::new(),
10010 protocol: ModuleProtocol::Subc,
10011 overlap: Default::default(),
10012 },
10013 enabled,
10014 )
10015 .unwrap();
10016 (supervisor_handle, module)
10017 }
10018
10019 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
10020 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
10021 module_id: module_id.to_string(),
10022 drain_timeout_ms: Some(50),
10023 })
10024 .unwrap();
10025 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
10026 }
10027
10028 #[test]
10032 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
10033 let registry = Arc::new(Registry::default());
10034 let forwarding = Arc::new(ForwardingTable::default());
10035 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10036 let second = ControlHandler::with_forwarding(registry, forwarding);
10037 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
10038 }
10039
10040 #[tokio::test(flavor = "current_thread")]
10044 async fn route_open_unknown_module_refusals_add_no_outage_state() {
10045 let handler = ControlHandler::new(Arc::new(Registry::default()));
10046 let capture = EventCapture::default();
10047 let _subscriber =
10048 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10049 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
10050 for corr in 0..8 {
10051 let response = handler
10052 .handle_control_frame(
10053 &ctx,
10054 route_open_frame(
10055 380 + corr,
10056 &format!("nobody-{corr}"),
10057 unique_project_root("outage-unknown"),
10058 ),
10059 )
10060 .await
10061 .unwrap();
10062 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
10063 }
10064
10065 assert_eq!(handler.route_outages.tracked_module_count(), 0);
10066 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
10067 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
10068 }
10069
10070 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
10075 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
10076 let registry = Arc::new(Registry::default());
10077 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
10078 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10079 let capture = EventCapture::default();
10080 let _subscriber =
10081 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10082 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
10083 handler.route_outages.record_accepted("outage-restart");
10086
10087 let response = handler
10088 .handle_control_frame(
10089 &ctx,
10090 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
10091 )
10092 .await
10093 .unwrap();
10094 assert_eq!(response[0].header.ty, FrameType::Error);
10095 let starts = outage_lines(&capture, OUTAGE_START);
10096 assert_eq!(starts.len(), 1, "{starts:?}");
10097 assert_eq!(starts[0].level, tracing::Level::WARN);
10098 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
10099 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
10100 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
10101 handler.route_outages.record_accepted("outage-restart");
10102 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
10103
10104 let restart = handler
10105 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
10106 .await
10107 .unwrap();
10108 assert_eq!(
10109 restart[0].header.ty,
10110 FrameType::Response,
10111 "{:?}",
10112 parse_error(&restart[0])
10113 );
10114 handler
10115 .handle_control_frame(
10116 &ctx,
10117 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
10118 )
10119 .await
10120 .unwrap();
10121 module.stop().await.unwrap();
10122
10123 let starts = outage_lines(&capture, OUTAGE_START);
10124 assert_eq!(starts.len(), 2, "{starts:?}");
10125 assert_eq!(starts[1].level, tracing::Level::INFO);
10126 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
10127 }
10128
10129 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
10133 async fn failed_operator_restart_leaves_no_operator_mark() {
10134 let registry = Arc::new(Registry::default());
10135 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
10136 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10137 let capture = EventCapture::default();
10138 let _subscriber =
10139 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10140 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
10141 handler.route_outages.record_accepted("outage-disabled");
10142
10143 let restart = handler
10144 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
10145 .await
10146 .unwrap();
10147 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
10148 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
10149
10150 handler
10151 .handle_control_frame(
10152 &ctx,
10153 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
10154 )
10155 .await
10156 .unwrap();
10157 let starts = outage_lines(&capture, OUTAGE_START);
10158 assert_eq!(starts.len(), 1, "{starts:?}");
10159 assert_eq!(starts[0].level, tracing::Level::WARN);
10160 }
10161
10162 #[tokio::test(flavor = "current_thread")]
10163 async fn route_open_unknown_module_escapes_target_module_id() {
10164 let handler = ControlHandler::new(Arc::new(Registry::default()));
10165 let capture = EventCapture::default();
10166 let _subscriber =
10167 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10168 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
10169 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
10170 let response = handler
10171 .handle_control_frame(
10172 &ctx,
10173 route_open_frame(
10174 395,
10175 hostile_module_id,
10176 unique_project_root("hostile-target-module-id"),
10177 ),
10178 )
10179 .await
10180 .unwrap();
10181
10182 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
10183 let event = capture
10184 .events()
10185 .into_iter()
10186 .find(|event| {
10187 event.target == "control"
10188 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
10189 })
10190 .expect("route.open unknown-module refusal event");
10191 let logged = event.fields.get("module_id").expect("module_id field");
10192 assert!(!logged.bytes().any(|byte| byte < 0x20));
10193 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
10194 }
10195
10196 #[tokio::test(flavor = "current_thread")]
10197 async fn route_open_module_rejection_uses_daemon_counter_key() {
10198 let registry = Arc::new(Registry::default());
10199 let forwarding = Arc::new(ForwardingTable::default());
10200 let handler =
10201 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10202 let module_connection = ConnectionId::new(95);
10203 let (module_ctx, mut module_rx) = route_ctx(module_connection);
10204 hello_via_sink(
10205 &handler,
10206 &module_ctx,
10207 &mut module_rx,
10208 hello_frame("aft", PROTOCOL_VERSION, 395),
10209 )
10210 .await;
10211
10212 let client_connection = ConnectionId::new(96);
10213 let (client_ctx, _client_rx) = route_ctx(client_connection);
10214 let capture = EventCapture::default();
10215 let _subscriber =
10216 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10217 let (route_task, bind) = relay_route_open(
10218 &handler,
10219 client_connection,
10220 &client_ctx.egress,
10221 &mut module_rx,
10222 396,
10223 "aft",
10224 "hostile-module-code",
10225 )
10226 .await;
10227 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
10228 let rejection = Frame::build(
10229 FrameType::Error,
10230 control_flags(),
10231 0,
10232 0,
10233 bind.header.corr,
10234 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
10235 )
10236 .unwrap();
10237 handler
10238 .handle_control_frame(&module_ctx, rejection)
10239 .await
10240 .unwrap();
10241
10242 let response = route_task.await.unwrap();
10243 assert_eq!(parse_error(&response[0])["code"], hostile_code);
10244 let counters = handler.counters().snapshot();
10245 assert_eq!(
10246 counters["route_open_refused_by_code"],
10247 json!({ "module_rejected": 1 })
10248 );
10249 assert!(counters["route_open_refused_by_code"]
10250 .get(hostile_code)
10251 .is_none());
10252
10253 let event = capture
10254 .events()
10255 .into_iter()
10256 .find(|event| {
10257 event.target == "control"
10258 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
10259 })
10260 .expect("route.open module-rejection refusal event");
10261 let logged = event.fields.get("module_code").expect("module_code field");
10262 assert!(!logged.bytes().any(|byte| byte < 0x20));
10263 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
10264 }
10265
10266 #[tokio::test]
10267 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
10268 let registry = Arc::new(Registry::default());
10269 let supervisor_handle = SupervisorHandle::new();
10270 let missing_program = std::env::temp_dir().join(format!(
10271 "subc-route-open-missing-program-{}",
10272 std::process::id()
10273 ));
10274 let supervisor =
10275 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
10276 .with_handle(supervisor_handle.clone());
10277 let module = supervisor
10278 .supervise_configured(
10279 ModuleSpec {
10280 module_id: "failed".to_string(),
10281 program: missing_program,
10282 args: Vec::new(),
10283 env: Vec::new(),
10284 reserved: false,
10285 reserved_prefixes: Vec::new(),
10286 protocol: ModuleProtocol::Subc,
10287 overlap: Default::default(),
10288 },
10289 true,
10290 )
10291 .unwrap();
10292 assert_eq!(module.state().unwrap(), ModuleState::Failed);
10293
10294 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10295 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
10296 let response = handler
10297 .handle_control_frame(
10298 &ctx,
10299 route_open_frame(305, "failed", unique_project_root("failed")),
10300 )
10301 .await
10302 .unwrap();
10303
10304 assert_eq!(response[0].header.ty, FrameType::Error);
10305 let error = parse_error(&response[0]);
10306 assert_eq!(error["code"], "target_unavailable");
10307 assert!(error["message"]
10308 .as_str()
10309 .unwrap()
10310 .contains("state=failed, enabled=true, live=false"));
10311 }
10312
10313 #[tokio::test]
10314 async fn route_open_role_mismatch_remains_target_unavailable() {
10315 let registry = Arc::new(Registry::default());
10316 let handler = ControlHandler::new(Arc::clone(®istry));
10317 handler
10318 .handle_control(
10319 ConnectionId::new(41),
10320 non_routable_hello_frame_with_control_ops("health-only", 306, None),
10321 )
10322 .unwrap();
10323
10324 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
10325 let response = handler
10326 .handle_control_frame(
10327 &ctx,
10328 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
10329 )
10330 .await
10331 .unwrap();
10332
10333 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10334 assert!(parse_error(&response[0])["message"]
10335 .as_str()
10336 .unwrap()
10337 .contains("does not provide the requested target"));
10338 }
10339
10340 #[tokio::test]
10341 async fn route_open_inactive_registration_remains_target_unavailable() {
10342 let registry = Arc::new(Registry::default());
10343 let handler = ControlHandler::new(Arc::clone(®istry));
10344 handler
10345 .handle_control(
10346 ConnectionId::new(43),
10347 hello_frame("inactive", PROTOCOL_VERSION, 308),
10348 )
10349 .unwrap();
10350 assert!(registry
10351 .set_module_state_for_test("inactive", ChannelState::Closed)
10352 .unwrap());
10353
10354 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
10355 let response = handler
10356 .handle_control_frame(
10357 &ctx,
10358 route_open_frame(309, "inactive", unique_project_root("inactive")),
10359 )
10360 .await
10361 .unwrap();
10362
10363 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10364 assert!(parse_error(&response[0])["message"]
10365 .as_str()
10366 .unwrap()
10367 .contains("is not active"));
10368 }
10369
10370 #[tokio::test]
10371 async fn late_health_reply_is_recorded_through_the_module_response_path() {
10372 let registry = Arc::new(Registry::default());
10373 let forwarding = Arc::new(ForwardingTable::default());
10374 let supervisor_handle = SupervisorHandle::new();
10375 let supervisor =
10376 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
10377 .with_forwarding(Arc::clone(&forwarding))
10378 .with_handle(supervisor_handle.clone());
10379 let module = supervisor
10380 .supervise_configured(
10381 crate::ModuleSpec {
10382 module_id: "late-health-response".to_string(),
10383 program: PathBuf::from("disabled-module"),
10384 args: Vec::new(),
10385 env: Vec::new(),
10386 reserved: false,
10387 reserved_prefixes: Vec::new(),
10388 protocol: ModuleProtocol::Subc,
10389 overlap: Default::default(),
10390 },
10391 false,
10392 )
10393 .unwrap();
10394 let handler =
10395 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10396 .with_supervisor(supervisor_handle);
10397 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
10398 handler
10399 .handle_control_frame(
10400 &module_ctx,
10401 hello_frame_with_control_ops(
10402 "late-health-response",
10403 PROTOCOL_VERSION,
10404 7,
10405 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10406 ),
10407 )
10408 .await
10409 .unwrap();
10410 let probe_started_at = Instant::now() - Duration::from_millis(80);
10411 let pending = forwarding
10412 .begin_health_probe_rpc_for(
10413 "late-health-response",
10414 MODULE_CONTROL_OP_HEALTH_CHECK,
10415 probe_started_at,
10416 Instant::now() - Duration::from_millis(1),
10417 )
10418 .unwrap();
10419 assert!(forwarding
10420 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
10421 .unwrap());
10422
10423 let responses = handler
10424 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
10425 .await
10426 .unwrap();
10427
10428 assert!(responses.is_empty());
10429 let health = module.status().unwrap().health;
10430 assert_eq!(health.late_answer_count, 1);
10431 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
10432 }
10433
10434 #[tokio::test]
10435 async fn health_probe_timeout_and_module_death_are_typed() {
10436 let registry = Arc::new(Registry::default());
10437 let forwarding = Arc::new(ForwardingTable::default());
10438 let handler =
10439 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10440 .with_health_probe_timeout(Duration::from_millis(50));
10441 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
10442 hello_via_sink(
10443 &handler,
10444 &module_ctx,
10445 &mut module_rx,
10446 hello_frame_with_control_ops(
10447 "aft",
10448 PROTOCOL_VERSION,
10449 7,
10450 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10451 ),
10452 )
10453 .await;
10454
10455 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10456 let responses = handler
10457 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
10458 .await
10459 .unwrap();
10460 assert_eq!(responses[0].header.ty, FrameType::Error);
10461 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
10462 let _ = module_rx.try_recv();
10463
10464 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
10465 let health_handler = handler.clone();
10466 let death_task = tokio::spawn(async move {
10467 health_handler
10468 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
10469 .await
10470 .unwrap()
10471 });
10472 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10473 .await
10474 .unwrap()
10475 .unwrap();
10476 handler
10477 .cleanup_connection(module_ctx.connection_id)
10478 .unwrap();
10479 let responses = death_task.await.unwrap();
10480 assert_eq!(responses[0].header.ty, FrameType::Error);
10481 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
10482 }
10483
10484 #[test]
10485 fn hello_requires_exact_protocol_version() {
10486 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
10487 let registry = Arc::new(Registry::default());
10488 let handler = ControlHandler::new(Arc::clone(®istry));
10489 let responses = handler
10490 .handle_control(
10491 ConnectionId::new(connection),
10492 hello_frame("aft", offered, 9),
10493 )
10494 .unwrap();
10495
10496 assert_eq!(responses.len(), 1);
10497 assert_eq!(responses[0].header.ty, FrameType::Error);
10498 let error = parse_error(&responses[0]);
10499 assert_eq!(error["code"], "version_unsupported");
10500 assert!(registry.get_module("aft").unwrap().is_none());
10501 assert_eq!(registry.active_registration_count().unwrap(), 0);
10502 }
10503 }
10504
10505 #[test]
10506 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
10507 let registry = Arc::new(Registry::default());
10508 let forwarding = Arc::new(ForwardingTable::default());
10509 let handler =
10510 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10511 let module_connection = ConnectionId::new(301);
10512 let registration = registry
10513 .register_with_control_ops(
10514 manifest("aft-push", PROTOCOL_VERSION),
10515 PROTOCOL_VERSION,
10516 module_connection,
10517 module_baseline_control_ops(),
10518 )
10519 .unwrap();
10520 let (module_tx, _module_rx) = mpsc::channel(8);
10521 let endpoint = forwarding
10522 .register_module_connection(
10523 module_connection,
10524 "aft-push".to_string(),
10525 PROTOCOL_VERSION,
10526 manifest_concurrency(®istration.manifest),
10527 FrameSink::new(module_tx),
10528 )
10529 .unwrap();
10530
10531 let unknown = Frame::build(
10533 FrameType::Push,
10534 control_flags(),
10535 0,
10536 0,
10537 5,
10538 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
10539 )
10540 .unwrap();
10541 let out = handler.handle_status_update(endpoint, unknown).unwrap();
10542 assert!(
10543 out.is_empty(),
10544 "unknown push op must be ignored, got {out:?}"
10545 );
10546
10547 let malformed = Frame::build(
10549 FrameType::Push,
10550 control_flags(),
10551 0,
10552 0,
10553 6,
10554 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
10555 )
10556 .unwrap();
10557 let out = handler.handle_status_update(endpoint, malformed).unwrap();
10558 assert_eq!(out.len(), 1);
10559 assert_eq!(out[0].header.ty, FrameType::Error);
10560 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
10561 }
10562
10563 #[test]
10564 fn hello_rejected_when_connection_already_owns_client_routes() {
10565 let registry = Arc::new(Registry::default());
10566 let forwarding = Arc::new(ForwardingTable::default());
10567 let handler =
10568 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10569 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
10571 let client_connection = ConnectionId::new(202);
10572
10573 let responses = handler
10576 .handle_control(
10577 client_connection,
10578 hello_frame("aft-second", PROTOCOL_VERSION, 9),
10579 )
10580 .unwrap();
10581 assert_eq!(responses[0].header.ty, FrameType::Error);
10582 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
10583 assert!(registry.get_module("aft-second").unwrap().is_none());
10584 }
10585
10586 #[tokio::test]
10587 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
10588 let registry = Arc::new(Registry::default());
10589 let forwarding = Arc::new(ForwardingTable::default());
10590 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
10591 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
10592 hello_via_sink(
10593 &handler,
10594 &module_ctx,
10595 &mut module_rx,
10596 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
10597 )
10598 .await;
10599 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
10600 let pending = forwarding
10601 .begin_route_bind_relay_for_test(
10602 client_ctx.connection_id,
10603 client_ctx.egress.clone(),
10604 2,
10605 "alpha",
10606 )
10607 .unwrap();
10608 forwarding
10609 .complete_pending_relay(
10610 module_ctx.connection_id,
10611 pending.corr,
10612 RouteBindRelayOutcome::Accepted,
10613 )
10614 .unwrap();
10615 client_rx.try_recv().unwrap();
10616 for module_id in ["beta", "alpha"] {
10617 let replies = handler
10618 .handle_control_frame(
10619 &module_ctx,
10620 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
10621 )
10622 .await
10623 .unwrap();
10624 assert_eq!(replies.len(), 1, "second HELLO must be refused");
10625 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
10626 }
10627 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
10628 assert!(registry.get_module("beta").unwrap().is_none());
10629 assert!(matches!(
10630 forwarding
10631 .lookup_data_route(
10632 client_ctx.connection_id,
10633 pending.client_channel,
10634 pending.client_epoch,
10635 )
10636 .unwrap(),
10637 DataRoute::Client(DataRouteState::Bound(_))
10638 ));
10639 assert!(handler
10640 .hello_launch_nonces
10641 .lock()
10642 .unwrap()
10643 .presented(module_ctx.connection_id, Some("alpha-nonce")));
10644 assert!(module_rx.try_recv().is_err());
10645 }
10646
10647 #[test]
10648 fn reserved_module_hello_requires_matching_launch_nonce() {
10649 let registry = Arc::new(Registry::default());
10650 let supervisor = SupervisorHandle::new();
10651 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
10654 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
10655
10656 let no_nonce = handler
10658 .handle_control(
10659 ConnectionId::new(1),
10660 hello_frame("vault", PROTOCOL_VERSION, 1),
10661 )
10662 .unwrap();
10663 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
10664 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
10665 assert!(registry.get_module("vault").unwrap().is_none());
10666
10667 let wrong = handler
10669 .handle_control(
10670 ConnectionId::new(2),
10671 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
10672 )
10673 .unwrap();
10674 assert_eq!(wrong[0].header.ty, FrameType::Error);
10675 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
10676 assert!(registry.get_module("vault").unwrap().is_none());
10677
10678 let ok = handler
10680 .handle_control(
10681 ConnectionId::new(3),
10682 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
10683 )
10684 .unwrap();
10685 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
10686 assert!(registry.get_module("vault").unwrap().is_some());
10687 }
10688
10689 #[test]
10690 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
10691 let registry = Arc::new(Registry::default());
10692 let supervisor = SupervisorHandle::new();
10693 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
10694 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
10695 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
10696
10697 let squat = handler
10698 .handle_control(
10699 ConnectionId::new(1),
10700 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
10701 )
10702 .unwrap();
10703 assert_eq!(squat[0].header.ty, FrameType::Error);
10704 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
10705 assert!(parse_error(&squat[0])["message"]
10706 .as_str()
10707 .unwrap()
10708 .contains("fed:"));
10709
10710 let accepted_peer = handler
10711 .handle_control(
10712 ConnectionId::new(2),
10713 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
10714 )
10715 .unwrap();
10716 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
10717
10718 let accepted_short = handler
10719 .handle_control(
10720 ConnectionId::new(3),
10721 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
10722 )
10723 .unwrap();
10724 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
10725
10726 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
10727 let response = handler
10728 .handle_control(
10729 ConnectionId::new(conn),
10730 hello_frame(module_id, PROTOCOL_VERSION, conn),
10731 )
10732 .unwrap();
10733 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
10734 }
10735 }
10736
10737 #[test]
10738 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
10739 let registry = Arc::new(Registry::default());
10740 let supervisor = SupervisorHandle::new();
10741 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
10742 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
10743 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
10744 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
10745
10746 let owner_nonce = handler
10747 .handle_control(
10748 ConnectionId::new(1),
10749 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
10750 )
10751 .unwrap();
10752 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
10753 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
10754 assert!(registry.get_module("fed:special").unwrap().is_none());
10755
10756 let exact_nonce = handler
10757 .handle_control(
10758 ConnectionId::new(2),
10759 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
10760 )
10761 .unwrap();
10762 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
10763 assert!(registry.get_module("fed:special").unwrap().is_some());
10764 }
10765
10766 #[test]
10767 fn non_reserved_module_ignores_launch_nonce() {
10768 let registry = Arc::new(Registry::default());
10769 let handler = ControlHandler::new(Arc::clone(®istry));
10772 let no_nonce = handler
10773 .handle_control(
10774 ConnectionId::new(1),
10775 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
10776 )
10777 .unwrap();
10778 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
10779 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
10780
10781 let echoed_nonce = handler
10782 .handle_control(
10783 ConnectionId::new(2),
10784 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
10785 )
10786 .unwrap();
10787 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
10788 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
10789 }
10790
10791 #[test]
10792 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
10793 let handler = ControlHandler::default();
10794 let conn = ConnectionId::new(1);
10795 let malformed = Frame::build(
10796 FrameType::Hello,
10797 control_flags(),
10798 0,
10799 0,
10800 3,
10801 b"{not json".to_vec(),
10802 )
10803 .unwrap();
10804
10805 let error = handler.handle_control(conn, malformed).unwrap();
10806 assert_eq!(error[0].header.ty, FrameType::Error);
10807 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
10808
10809 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
10810 let pong = handler.handle_control(conn, ping).unwrap();
10811 assert_eq!(pong[0].header.ty, FrameType::Pong);
10812 assert_eq!(pong[0].header.corr, 4);
10813 }
10814
10815 #[test]
10816 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
10817 let registry = Arc::new(Registry::default());
10818 let handler = ControlHandler::new(Arc::clone(®istry));
10819
10820 handler
10821 .handle_control(
10822 ConnectionId::new(1),
10823 hello_frame("aft", PROTOCOL_VERSION, 1),
10824 )
10825 .unwrap();
10826 let duplicate = handler
10827 .handle_control(
10828 ConnectionId::new(2),
10829 hello_frame("aft", PROTOCOL_VERSION, 2),
10830 )
10831 .unwrap();
10832
10833 assert_eq!(duplicate[0].header.ty, FrameType::Error);
10834 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
10835 let registration = registry.get_module("aft").unwrap().unwrap();
10836 assert_eq!(registration.connection_id, ConnectionId::new(1));
10837 }
10838
10839 #[test]
10840 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
10841 let registry = Arc::new(Registry::default());
10842 let forwarding = Arc::new(ForwardingTable::default());
10843 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
10844 let handler =
10845 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10846 .with_process_liveness(process_liveness);
10847 let (ctx, route_channel, route_epoch) =
10848 bind_liveness_route(®istry, &forwarding, "aft-dead");
10849 let responses = handler
10850 .handle_route_poll(
10851 &ctx,
10852 route_poll_frame(41, PollKind::Liveness, route_channel),
10853 route_channel,
10854 route_epoch,
10855 PollKind::Liveness,
10856 )
10857 .unwrap();
10858
10859 assert_eq!(responses.len(), 1);
10860 assert_eq!(responses[0].header.ty, FrameType::Response);
10861 assert_route_poll_liveness(&responses[0], false);
10862 }
10863
10864 #[test]
10865 fn liveness_poll_without_process_source_uses_bound_route() {
10866 let registry = Arc::new(Registry::default());
10867 let forwarding = Arc::new(ForwardingTable::default());
10868 let handler =
10869 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10870 let (ctx, route_channel, route_epoch) =
10871 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
10872 let responses = handler
10873 .handle_route_poll(
10874 &ctx,
10875 route_poll_frame(42, PollKind::Liveness, route_channel),
10876 route_channel,
10877 route_epoch,
10878 PollKind::Liveness,
10879 )
10880 .unwrap();
10881
10882 assert_route_poll_liveness(&responses[0], true);
10883 }
10884
10885 #[test]
10886 fn liveness_poll_untracked_process_source_uses_bound_route() {
10887 let registry = Arc::new(Registry::default());
10888 let forwarding = Arc::new(ForwardingTable::default());
10889 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
10890 let handler =
10891 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10892 .with_process_liveness(process_liveness);
10893 let (ctx, route_channel, route_epoch) =
10894 bind_liveness_route(®istry, &forwarding, "aft-untracked");
10895 let responses = handler
10896 .handle_route_poll(
10897 &ctx,
10898 route_poll_frame(43, PollKind::Liveness, route_channel),
10899 route_channel,
10900 route_epoch,
10901 PollKind::Liveness,
10902 )
10903 .unwrap();
10904
10905 assert_route_poll_liveness(&responses[0], true);
10906 }
10907
10908 #[tokio::test]
10909 async fn unknown_op_returns_unknown_control_op() {
10910 let handler = ControlHandler::default();
10911 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
10912 let request = Frame::build(
10913 FrameType::Request,
10914 control_flags(),
10915 0,
10916 0,
10917 55,
10918 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
10919 )
10920 .unwrap();
10921
10922 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
10923
10924 assert_eq!(response.len(), 1);
10925 assert_eq!(response[0].header.ty, FrameType::Error);
10926 assert_eq!(response[0].header.corr, 55);
10927 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
10928 }
10929
10930 #[tokio::test]
10931 async fn supervisor_provenance_rejects_unknown_exact_module() {
10932 let handler = ControlHandler::default();
10933 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
10934 let request = Frame::build(
10935 FrameType::Request,
10936 control_flags(),
10937 0,
10938 0,
10939 57,
10940 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
10941 )
10942 .unwrap();
10943
10944 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
10945
10946 assert_eq!(response.len(), 1);
10947 assert_eq!(response[0].header.ty, FrameType::Error);
10948 assert_eq!(response[0].header.corr, 57);
10949 let error = parse_error(&response[0]);
10950 assert_eq!(error["code"], "unknown_module");
10951 assert_eq!(error["message"], "module_id 'missing' is not supervised");
10952 }
10953
10954 #[test]
10955 fn provenance_probe_override_keeps_handler_tests_deterministic() {
10956 let expected = subc_control::RunningImageAgreement::Unavailable {
10957 reason: subc_control::RunningImageUnavailableReason::HashFailed,
10958 };
10959 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
10960 assert_eq!(handler.provenance_probe_override, Some(expected));
10961 }
10962
10963 #[test]
10964 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
10965 let verdict = reload_verdict(
10966 std::path::Path::new("/bin/new"),
10967 Some(std::path::Path::new("/bin/old")),
10968 subc_control::RunningImageAgreement::Unavailable {
10969 reason: subc_control::RunningImageUnavailableReason::HashFailed,
10970 },
10971 );
10972 assert!(matches!(
10973 verdict.path,
10974 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
10975 if configured == std::path::Path::new("/bin/new")
10976 && spawned_from == std::path::Path::new("/bin/old")
10977 ));
10978 }
10979
10980 #[test]
10981 fn reload_verdict_detects_replaced_image_at_same_path() {
10982 let image = subc_control::RunningImageAgreement::Mismatch {
10983 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
10984 digest: "old".into(),
10985 },
10986 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
10987 digest: "new".into(),
10988 },
10989 };
10990 let verdict = reload_verdict(
10991 std::path::Path::new("/bin/same"),
10992 Some(std::path::Path::new("/bin/same")),
10993 image.clone(),
10994 );
10995 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
10996 assert_eq!(verdict.image, image);
10997 }
10998
10999 #[test]
11000 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
11001 let image = subc_control::RunningImageAgreement::Unavailable {
11002 reason: subc_control::RunningImageUnavailableReason::NotRunning,
11003 };
11004 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
11005 assert_eq!(
11006 verdict.path,
11007 subc_control::ReloadPathAgreement::Unavailable {
11008 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
11009 }
11010 );
11011 assert_eq!(verdict.image, image);
11012
11013 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
11014 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
11015 };
11016 let verdict = reload_verdict(
11017 std::path::Path::new("/bin/same"),
11018 Some(std::path::Path::new("/bin/same")),
11019 unconfirmed.clone(),
11020 );
11021 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
11022 assert_eq!(verdict.image, unconfirmed);
11023 }
11024
11025 #[test]
11026 fn reload_verdict_preserves_each_image_unavailability_reason() {
11027 use subc_control::RunningImageUnavailableReason as Reason;
11028
11029 for reason in [
11030 Reason::NotRunning,
11031 Reason::UnsupportedPlatform,
11032 Reason::RunningExecutableUnreadable,
11033 Reason::SpawnedPathUnreadable,
11034 Reason::HashFailed,
11035 Reason::ProcessIdentityUnconfirmed,
11036 Reason::Unknown("future_probe_reason".to_string()),
11037 ] {
11038 let image = subc_control::RunningImageAgreement::Unavailable {
11039 reason: reason.clone(),
11040 };
11041 let verdict = reload_verdict(
11042 std::path::Path::new("/bin/same"),
11043 Some(std::path::Path::new("/bin/same")),
11044 image.clone(),
11045 );
11046 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
11047 assert_eq!(verdict.image, image, "{reason:?}");
11048 }
11049 }
11050
11051 #[tokio::test]
11052 async fn malformed_control_bodies_return_invalid_control_body() {
11053 let handler = ControlHandler::default();
11054 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
11055
11056 for (corr, body) in [
11057 (56, br#"{"route_channel":1}"#.as_slice()),
11058 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
11059 (
11060 58,
11061 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
11062 ),
11063 ] {
11064 let request = Frame::build(
11065 FrameType::Request,
11066 control_flags(),
11067 0,
11068 0,
11069 corr,
11070 body.to_vec(),
11071 )
11072 .unwrap();
11073 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
11074
11075 assert_eq!(response.len(), 1);
11076 assert_eq!(response[0].header.ty, FrameType::Error);
11077 assert_eq!(response[0].header.corr, corr);
11078 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
11079 }
11080 }
11081
11082 #[tokio::test]
11083 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
11084 let registry = Arc::new(Registry::default());
11085 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
11086 let router = Router::with_control_handler(Arc::clone(&control));
11087 let connection = router.begin_connection();
11088 let (ctx, mut rx) = route_ctx(connection.id());
11089
11090 router
11091 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
11092 .await
11093 .unwrap();
11094 let response = rx.recv().await.unwrap();
11095 let ack = parse_ack(&response);
11096 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
11097 let channel = 1;
11098
11099 let goodbye =
11100 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
11101 router.route_for_connection(&ctx, goodbye).await.unwrap();
11102 assert!(rx.try_recv().is_err());
11103 assert!(registry.get_module("aft").unwrap().is_none());
11104
11105 router
11106 .route_for_connection(&ctx, channel_request(channel, 13))
11107 .await
11108 .unwrap();
11109 let error_frame = rx.recv().await.unwrap();
11110 assert_eq!(error_frame.header.ty, FrameType::Error);
11111 assert_eq!(error_frame.header.channel, channel);
11112 }
11113
11114 #[tokio::test]
11115 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
11116 let registry = Arc::new(Registry::default());
11117 let handler = ControlHandler::new(registry).with_capability_config(
11118 [("prov".to_string(), true), ("cons".to_string(), true)],
11119 BTreeMap::new(),
11120 );
11121 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
11122 register_capability_manifest(
11123 &handler,
11124 &provider_ctx,
11125 &mut provider_rx,
11126 capability_manifest("prov", &["thing/v1"], &[]),
11127 1,
11128 )
11129 .await;
11130 let mut consumer = capability_manifest("cons", &[], &[]);
11131 consumer.capabilities.as_mut().unwrap().requires.push(
11132 subc_protocol::manifest::CapabilityRequirement {
11133 capability: "thing/v1".to_string(),
11134 need: subc_protocol::manifest::CapabilityNeed::Required,
11135 },
11136 );
11137 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
11138 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
11139 assert_eq!(
11140 handler.capability_evaluator.verdict("cons", "thing/v1"),
11141 Some(CapabilityVerdict::Provided)
11142 );
11143 let (mut client_rx, _) = open_route_for_capability_test(
11144 &handler,
11145 &provider_ctx,
11146 &mut provider_rx,
11147 703,
11148 3,
11149 "prov",
11150 None,
11151 )
11152 .await;
11153 let goodbye =
11154 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
11155 handler
11156 .handle_control_frame(&provider_ctx, goodbye)
11157 .await
11158 .unwrap();
11159 assert_eq!(
11160 handler.capability_evaluator.verdict("cons", "thing/v1"),
11161 Some(CapabilityVerdict::NeverProvided)
11162 );
11163 let closed = client_rx
11164 .try_recv()
11165 .expect("GOODBYE pushes route.closed before route GOODBYE");
11166 assert!(
11167 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
11168 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
11169 );
11170 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
11171 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
11172 }
11173
11174 #[tokio::test]
11175 async fn dropping_router_connection_releases_registration() {
11176 let registry = Arc::new(Registry::default());
11177 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
11178 let router = Router::with_control_handler(control);
11179 let connection = router.begin_connection();
11180 let (ctx, mut rx) = route_ctx(connection.id());
11181
11182 router
11183 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
11184 .await
11185 .unwrap();
11186 let response = rx.recv().await.unwrap();
11187 let ack = parse_ack(&response);
11188 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
11189 assert!(registry.get_module("aft").unwrap().is_some());
11190
11191 drop(connection);
11192
11193 assert!(registry.get_module("aft").unwrap().is_none());
11194 assert_eq!(registry.active_registration_count().unwrap(), 0);
11195 }
11196
11197 fn capability_manifest(
11198 module_id: &str,
11199 provides: &[&str],
11200 must_never_reach: &[&str],
11201 ) -> ModuleManifest {
11202 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
11203 manifest.capabilities = Some(CapabilityDeclarations {
11204 provides: provides
11205 .iter()
11206 .map(|capability| (*capability).to_string())
11207 .collect(),
11208 requires: Vec::new(),
11209 must_never_reach: must_never_reach
11210 .iter()
11211 .map(|capability| (*capability).to_string())
11212 .collect(),
11213 });
11214 manifest
11215 }
11216
11217 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
11218 Frame::build(
11219 FrameType::Hello,
11220 control_flags(),
11221 0,
11222 0,
11223 corr,
11224 serde_json::to_vec(&ModuleHelloBody {
11225 protocol_ver: manifest.protocol_ver,
11226 manifest,
11227 control_ops: None,
11228 launch_nonce: None,
11229 })
11230 .expect("capability test HELLO serializes"),
11231 )
11232 .expect("capability test HELLO frame builds")
11233 }
11234
11235 fn catalog_update_with_capabilities_frame(
11236 corr: u64,
11237 capabilities: CapabilityDeclarations,
11238 ) -> Frame {
11239 Frame::build(
11240 FrameType::Request,
11241 control_flags(),
11242 0,
11243 0,
11244 corr,
11245 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
11246 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
11247 capabilities: Some(capabilities),
11248 ready: None,
11249 })
11250 .expect("capability catalog.update serializes"),
11251 )
11252 .expect("capability catalog.update frame builds")
11253 }
11254
11255 async fn register_capability_manifest(
11256 handler: &ControlHandler,
11257 ctx: &RouteCtx,
11258 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
11259 manifest: ModuleManifest,
11260 corr: u64,
11261 ) {
11262 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
11263 }
11264
11265 async fn open_route_for_capability_test(
11266 handler: &ControlHandler,
11267 target_ctx: &RouteCtx,
11268 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
11269 client_connection_id: u64,
11270 corr: u64,
11271 target_module_id: &str,
11272 consumer_identity: Option<ConsumerIdentity>,
11273 ) -> (
11274 mpsc::Receiver<crate::router::OutboundFrame>,
11275 ModuleControlRequest,
11276 ) {
11277 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
11278 let route_handler = handler.clone();
11279 let target_module_id = target_module_id.to_string();
11280 let route_task = tokio::spawn(async move {
11281 route_handler
11282 .handle_control_frame(
11283 &client_ctx,
11284 route_open_frame_with_admission_facts(
11285 corr,
11286 &target_module_id,
11287 unique_project_root("admission-facts"),
11288 consumer_identity,
11289 None,
11290 ),
11291 )
11292 .await
11293 .expect("capability test route.open succeeds")
11294 });
11295 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
11296 .await
11297 .expect("capability test route.open must reach route.bind")
11298 .expect("target control receiver stays open");
11299 let bind_request: ModuleControlRequest =
11300 serde_json::from_slice(&bind.body).expect("route.bind decodes");
11301 handler
11302 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
11303 .await
11304 .expect("capability test route.bind ACK succeeds");
11305 assert!(route_task.await.expect("route.open task joins").is_empty());
11306 let opened = client_rx
11307 .recv()
11308 .await
11309 .expect("successful route.open publishes a response");
11310 assert!(matches!(
11311 serde_json::from_slice::<ClientControlResponse>(&opened.body),
11312 Ok(ClientControlResponse::RouteOpen { .. })
11313 ));
11314 (client_rx, bind_request)
11315 }
11316
11317 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
11318 assert_eq!(frame.header.ty, FrameType::Push);
11319 assert_eq!(frame.header.channel, 0);
11320 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
11321 .expect("route.closed control push decodes");
11322 let ClientControlPush::RouteClosed { channels, .. } = &push else {
11323 panic!("expected route.closed");
11324 };
11325 assert_eq!(channels.len(), 1, "exactly one violating route closed");
11326 let channels = channels.clone();
11327 assert_eq!(
11328 push,
11329 ClientControlPush::RouteClosed {
11330 module_id: target_module_id.to_string(),
11331 channels,
11332 reason: RouteCloseReason::CapabilityDenied,
11333 drained: false,
11334 abandoned: 0,
11335 excluded_subscriptions: 0,
11336 terminal: Some(false),
11337 }
11338 );
11339 }
11340
11341 #[tokio::test]
11342 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
11343 let registry = Arc::new(Registry::default());
11344 let forwarding = Arc::new(ForwardingTable::default());
11345 let supervisor = SupervisorHandle::new();
11346 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11347 let handler =
11348 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11349 .with_supervisor(supervisor);
11350 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
11351 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
11352 register_capability_manifest(
11353 &handler,
11354 &target_ctx,
11355 &mut target_rx,
11356 capability_manifest("target", &["credentials-provider/v1"], &[]),
11357 1,
11358 )
11359 .await;
11360 register_capability_manifest(
11361 &handler,
11362 &opener_ctx,
11363 &mut opener_rx,
11364 capability_manifest("opener", &[], &["credentials-provider/v1"]),
11365 2,
11366 )
11367 .await;
11368
11369 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
11370 let replies = handler
11371 .handle_control_frame(
11372 &client_ctx,
11373 route_open_frame_with_admission_facts(
11374 3,
11375 "target",
11376 unique_project_root("admission-facts"),
11377 Some(ConsumerIdentity {
11378 module_id: "opener".to_string(),
11379 launch_nonce: "opener-nonce".to_string(),
11380 }),
11381 None,
11382 ),
11383 )
11384 .await
11385 .expect("denied route.open returns a typed frame");
11386 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
11387 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
11388 assert!(
11389 target_rx.try_recv().is_err(),
11390 "forbidden route.open must not relay route.bind"
11391 );
11392 }
11393
11394 #[tokio::test]
11395 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
11396 let registry = Arc::new(Registry::default());
11397 let forwarding = Arc::new(ForwardingTable::default());
11398 let supervisor = SupervisorHandle::new();
11399 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11400 let handler =
11401 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11402 .with_supervisor(supervisor);
11403 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
11404 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
11405 register_capability_manifest(
11406 &handler,
11407 &target_ctx,
11408 &mut target_rx,
11409 capability_manifest("target", &["credentials-provider/v1"], &[]),
11410 1,
11411 )
11412 .await;
11413 register_capability_manifest(
11414 &handler,
11415 &old_opener_ctx,
11416 &mut old_opener_rx,
11417 capability_manifest("opener", &[], &[]),
11418 2,
11419 )
11420 .await;
11421 let (mut client_rx, _) = open_route_for_capability_test(
11422 &handler,
11423 &target_ctx,
11424 &mut target_rx,
11425 712,
11426 3,
11427 "target",
11428 Some(ConsumerIdentity {
11429 module_id: "opener".to_string(),
11430 launch_nonce: "opener-nonce".to_string(),
11431 }),
11432 )
11433 .await;
11434 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
11435
11436 handler
11437 .cleanup_connection(old_opener_ctx.connection_id)
11438 .expect("old opener registration cleans up");
11439 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
11440 register_capability_manifest(
11441 &handler,
11442 &new_opener_ctx,
11443 &mut new_opener_rx,
11444 capability_manifest("opener", &[], &["credentials-provider/v1"]),
11445 4,
11446 )
11447 .await;
11448
11449 assert_capability_denied_push(
11450 client_rx
11451 .try_recv()
11452 .expect("HELLO deny addition must emit route.closed")
11453 .frame,
11454 "target",
11455 );
11456 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
11457 assert!(matches!(
11458 target_rx.try_recv(),
11459 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
11460 ));
11461 }
11462
11463 #[tokio::test]
11464 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
11465 let registry = Arc::new(Registry::default());
11466 let forwarding = Arc::new(ForwardingTable::default());
11467 let supervisor = SupervisorHandle::new();
11468 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11469 let handler =
11470 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11471 .with_supervisor(supervisor);
11472 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
11473 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
11474 register_capability_manifest(
11475 &handler,
11476 &target_ctx,
11477 &mut target_rx,
11478 capability_manifest("target", &[], &[]),
11479 1,
11480 )
11481 .await;
11482 register_capability_manifest(
11483 &handler,
11484 &opener_ctx,
11485 &mut opener_rx,
11486 capability_manifest("opener", &[], &["credentials-provider/v1"]),
11487 2,
11488 )
11489 .await;
11490 let (mut client_rx, _) = open_route_for_capability_test(
11491 &handler,
11492 &target_ctx,
11493 &mut target_rx,
11494 722,
11495 3,
11496 "target",
11497 Some(ConsumerIdentity {
11498 module_id: "opener".to_string(),
11499 launch_nonce: "opener-nonce".to_string(),
11500 }),
11501 )
11502 .await;
11503 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
11504
11505 let replies = handler
11506 .handle_control_frame(
11507 &target_ctx,
11508 catalog_update_with_capabilities_frame(
11509 4,
11510 CapabilityDeclarations {
11511 provides: vec!["credentials-provider/v1".to_string()],
11512 requires: Vec::new(),
11513 must_never_reach: Vec::new(),
11514 },
11515 ),
11516 )
11517 .await
11518 .expect("claim catalog.update succeeds");
11519 assert!(matches!(
11520 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
11521 Ok(ModuleControlResponseToModule::CatalogUpdate {})
11522 ));
11523 assert_capability_denied_push(
11524 client_rx
11525 .try_recv()
11526 .expect("claim addition must emit route.closed")
11527 .frame,
11528 "target",
11529 );
11530 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
11531 assert!(matches!(
11532 target_rx.try_recv(),
11533 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
11534 ));
11535 }
11536
11537 #[tokio::test]
11538 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
11539 let registry = Arc::new(Registry::default());
11540 let forwarding = Arc::new(ForwardingTable::default());
11541 let supervisor = SupervisorHandle::new();
11542 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11543 let handler =
11544 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11545 .with_supervisor(supervisor);
11546 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
11547 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
11548 register_capability_manifest(
11549 &handler,
11550 &target_ctx,
11551 &mut target_rx,
11552 capability_manifest("target", &["credentials-provider/v1"], &[]),
11553 1,
11554 )
11555 .await;
11556 register_capability_manifest(
11557 &handler,
11558 &opener_ctx,
11559 &mut opener_rx,
11560 capability_manifest("opener", &[], &[]),
11561 2,
11562 )
11563 .await;
11564 let (mut client_rx, _) = open_route_for_capability_test(
11565 &handler,
11566 &target_ctx,
11567 &mut target_rx,
11568 732,
11569 3,
11570 "target",
11571 Some(ConsumerIdentity {
11572 module_id: "opener".to_string(),
11573 launch_nonce: "opener-nonce".to_string(),
11574 }),
11575 )
11576 .await;
11577 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
11578
11579 handler
11580 .handle_control_frame(
11581 &target_ctx,
11582 catalog_update_with_capabilities_frame(
11583 4,
11584 CapabilityDeclarations {
11585 provides: Vec::new(),
11586 requires: Vec::new(),
11587 must_never_reach: Vec::new(),
11588 },
11589 ),
11590 )
11591 .await
11592 .expect("claim removal catalog.update succeeds");
11593 assert_eq!(
11594 forwarding.active_binding_count().unwrap(),
11595 1,
11596 "removing an attested target claim must leave the route census unchanged"
11597 );
11598 assert!(
11599 client_rx.try_recv().is_err(),
11600 "claim removal must not emit route.closed capability_denied"
11601 );
11602 assert!(
11603 target_rx.try_recv().is_err(),
11604 "claim removal must not send the target a route GOODBYE"
11605 );
11606 }
11607
11608 #[tokio::test]
11611 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
11612 let registry = Arc::new(Registry::default());
11613 let forwarding = Arc::new(ForwardingTable::default());
11614 let supervisor = SupervisorHandle::new();
11615 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11616 let handler =
11617 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11618 .with_supervisor(supervisor);
11619 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
11620 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
11621 register_capability_manifest(
11622 &handler,
11623 &target_ctx,
11624 &mut target_rx,
11625 capability_manifest("target", &["credentials-provider/v1"], &[]),
11626 1,
11627 )
11628 .await;
11629 register_capability_manifest(
11630 &handler,
11631 &opener_ctx,
11632 &mut opener_rx,
11633 capability_manifest("opener", &[], &["credentials-provider/v1"]),
11634 2,
11635 )
11636 .await;
11637
11638 let (_client_rx, bind) = open_route_for_capability_test(
11639 &handler,
11640 &target_ctx,
11641 &mut target_rx,
11642 742,
11643 3,
11644 "target",
11645 None,
11646 )
11647 .await;
11648 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
11649 panic!("direct scope-honesty route must bind");
11650 };
11651 assert_eq!(principal, Some(Principal::Direct));
11652 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
11653 }
11654
11655 #[tokio::test]
11658 async fn must_never_reach_self_route_is_capability_forbidden() {
11659 let registry = Arc::new(Registry::default());
11660 let forwarding = Arc::new(ForwardingTable::default());
11661 let supervisor = SupervisorHandle::new();
11662 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
11663 let handler =
11664 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11665 .with_supervisor(supervisor);
11666 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
11667 register_capability_manifest(
11668 &handler,
11669 &self_ctx,
11670 &mut self_rx,
11671 capability_manifest(
11672 "self-provider",
11673 &["credentials-provider/v1"],
11674 &["credentials-provider/v1"],
11675 ),
11676 1,
11677 )
11678 .await;
11679
11680 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
11681 let replies = handler
11682 .handle_control_frame(
11683 &client_ctx,
11684 route_open_frame_with_admission_facts(
11685 2,
11686 "self-provider",
11687 unique_project_root("admission-facts"),
11688 Some(ConsumerIdentity {
11689 module_id: "self-provider".to_string(),
11690 launch_nonce: "self-nonce".to_string(),
11691 }),
11692 None,
11693 ),
11694 )
11695 .await
11696 .expect("self-route refusal returns a typed frame");
11697 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
11698 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
11699 assert!(
11700 self_rx.try_recv().is_err(),
11701 "self denial must not relay route.bind"
11702 );
11703 }
11704
11705 #[test]
11706 fn unsupported_channel_zero_frame_returns_error() {
11707 let handler = ControlHandler::default();
11708 let request = Frame::build(
11709 FrameType::Request,
11710 control_flags(),
11711 0,
11712 0,
11713 21,
11714 b"opaque".to_vec(),
11715 )
11716 .unwrap();
11717
11718 let response = handler
11719 .handle_control(ConnectionId::new(1), request)
11720 .unwrap();
11721
11722 assert_eq!(response[0].header.ty, FrameType::Error);
11723 assert_eq!(
11724 parse_error(&response[0])["code"],
11725 "unsupported_control_frame"
11726 );
11727 }
11728
11729 mod swap {
11734 use super::*;
11735
11736 const INCUMBENT: ConnectionId = ConnectionId::new(30);
11737 const CANDIDATE: ConnectionId = ConnectionId::new(40);
11738
11739 struct Swap {
11740 registry: Arc<Registry>,
11741 forwarding: Arc<ForwardingTable>,
11742 handler: ControlHandler,
11743 incumbent_ctx: RouteCtx,
11744 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
11745 candidate_ctx: RouteCtx,
11746 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
11747 }
11748
11749 async fn swap_with_incumbent() -> Swap {
11750 let registry = Arc::new(Registry::default());
11751 let forwarding = Arc::new(ForwardingTable::default());
11752 let handler =
11753 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11754 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
11755 hello_via_sink(
11756 &handler,
11757 &incumbent_ctx,
11758 &mut incumbent_rx,
11759 hello_frame("aft", PROTOCOL_VERSION, 7),
11760 )
11761 .await;
11762 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
11763 Swap {
11764 registry,
11765 forwarding,
11766 handler,
11767 incumbent_ctx,
11768 incumbent_rx,
11769 candidate_ctx,
11770 candidate_rx,
11771 }
11772 }
11773
11774 fn register_candidate(swap: &Swap, ready: Option<bool>) {
11775 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
11776 candidate_manifest.ready = ready;
11777 let registration = swap
11778 .registry
11779 .register_candidate_with_control_ops(
11780 candidate_manifest,
11781 PROTOCOL_VERSION,
11782 CANDIDATE,
11783 module_baseline_control_ops(),
11784 )
11785 .unwrap();
11786 swap.forwarding
11787 .register_candidate_module_connection(
11788 CANDIDATE,
11789 "aft".to_string(),
11790 PROTOCOL_VERSION,
11791 manifest_concurrency(®istration.manifest),
11792 swap.candidate_ctx.egress.clone(),
11793 )
11794 .unwrap();
11795 }
11796
11797 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
11798 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
11799 swap.registry.promote_candidate("aft").unwrap().unwrap();
11800 cutover.incumbent.unwrap()
11801 }
11802
11803 fn keyed_total(counters: &Value, key: &str) -> u64 {
11804 counters[key]
11805 .as_object()
11806 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
11807 .unwrap_or(0)
11808 }
11809
11810 #[tokio::test]
11816 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
11817 let mut swap = swap_with_incumbent().await;
11818 let handler = swap.handler.clone();
11819
11820 let cotenant = ConnectionId::new(31);
11822 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
11823 let (cotenant_task, cotenant_bind) = relay_route_open(
11824 &handler,
11825 cotenant,
11826 &cotenant_ctx.egress,
11827 &mut swap.incumbent_rx,
11828 100,
11829 "aft",
11830 "swap-cotenant",
11831 )
11832 .await;
11833 handler
11834 .handle_control_frame(
11835 &swap.incumbent_ctx,
11836 route_bind_ack(cotenant_bind.header.corr),
11837 )
11838 .await
11839 .unwrap();
11840 assert!(cotenant_task.await.unwrap().is_empty());
11841 let (cotenant_channel, cotenant_epoch) =
11842 published_route(&cotenant_rx.recv().await.unwrap());
11843
11844 let caller = ConnectionId::new(32);
11846 let (caller_ctx, mut caller_rx) = route_ctx(caller);
11847 let (caller_task, caller_bind) = relay_route_open(
11848 &handler,
11849 caller,
11850 &caller_ctx.egress,
11851 &mut swap.incumbent_rx,
11852 101,
11853 "aft",
11854 "swap-caller",
11855 )
11856 .await;
11857 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
11858
11859 register_candidate(&swap, None);
11860 cutover(&swap);
11861
11862 let ack = handler
11864 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
11865 .await;
11866 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
11867 if module_loop_error.is_some() {
11868 handler.cleanup_connection(INCUMBENT).unwrap();
11871 }
11872
11873 assert!(
11875 cotenant_rx.try_recv().is_err(),
11876 "the co-tenant route on the incumbent was torn down by one late ack: \
11877 {module_loop_error:?}"
11878 );
11879 assert!(matches!(
11880 swap.forwarding
11881 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
11882 .unwrap(),
11883 DataRoute::Client(DataRouteState::Bound(_))
11884 ));
11885 assert_eq!(module_loop_error, None);
11886 assert!(swap
11887 .registry
11888 .get_module_by_connection(INCUMBENT)
11889 .unwrap()
11890 .is_some());
11891
11892 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
11894 .await
11895 .expect("the incumbent is told to drop the abandoned binding")
11896 .unwrap()
11897 .frame;
11898 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11899 assert_eq!(goodbye.header.channel, abandoned_channel);
11900 assert_eq!(goodbye.header.epoch, abandoned_epoch);
11901 assert!(swap.incumbent_rx.try_recv().is_err());
11902
11903 let response = caller_task.await.unwrap();
11905 assert_eq!(response.len(), 1);
11906 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
11907 assert!(caller_rx.try_recv().is_err());
11908
11909 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
11913 let counters = handler.counters().snapshot();
11914 assert_eq!(
11915 keyed_total(&counters, "route_open_accepted_by_principal"),
11916 1
11917 );
11918 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
11919 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
11920 }
11921
11922 #[tokio::test]
11926 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
11927 let mut swap = swap_with_incumbent().await;
11928 register_candidate(&swap, None);
11929 let incumbent = cutover(&swap);
11930 swap.forwarding
11931 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
11932 .unwrap()
11933 .expect("the incumbent is still registered");
11934
11935 let client = ConnectionId::new(33);
11936 let (client_ctx, mut client_rx) = route_ctx(client);
11937 let route_handler = swap.handler.clone();
11938 let open_ctx = RouteCtx {
11939 connection_id: client,
11940 egress: client_ctx.egress.clone(),
11941 };
11942 let mut route_task = tokio::spawn(async move {
11943 route_handler
11944 .handle_control_frame(
11945 &open_ctx,
11946 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
11947 )
11948 .await
11949 .unwrap()
11950 });
11951 let bind = tokio::select! {
11952 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
11953 response = &mut route_task => {
11954 let response = response.unwrap();
11955 panic!(
11956 "post-cutover route.open was refused instead of relayed to the candidate: {}",
11957 parse_error(&response[0])["code"]
11958 );
11959 }
11960 };
11961 swap.handler
11962 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
11963 .await
11964 .unwrap();
11965 assert!(route_task.await.unwrap().is_empty());
11966 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
11967 match swap
11968 .forwarding
11969 .lookup_data_route(client, channel, epoch)
11970 .unwrap()
11971 {
11972 DataRoute::Client(DataRouteState::Bound(route)) => {
11973 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
11974 }
11975 other => panic!("expected a bound route on the candidate, got {other:?}"),
11976 }
11977 assert!(swap.incumbent_rx.try_recv().is_err());
11978 }
11979
11980 #[tokio::test]
11985 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
11986 let swap = swap_with_incumbent().await;
11987 register_candidate(&swap, Some(false));
11988 let update = Frame::build(
11989 FrameType::Request,
11990 control_flags(),
11991 0,
11992 0,
11993 55,
11994 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
11995 provides: manifest("aft", PROTOCOL_VERSION).provides,
11996 capabilities: None,
11997 ready: Some(true),
11998 })
11999 .unwrap(),
12000 )
12001 .unwrap();
12002
12003 let replies = swap
12004 .handler
12005 .handle_control_frame(&swap.candidate_ctx, update)
12006 .await
12007 .unwrap();
12008
12009 assert_eq!(replies.len(), 1);
12010 assert_eq!(
12011 replies[0].header.ty,
12012 FrameType::Response,
12013 "candidate catalog.update was refused: {:?}",
12014 serde_json::from_slice::<Value>(&replies[0].body).ok()
12015 );
12016 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
12017 assert_eq!(
12018 swap.registry
12019 .get_module("aft")
12020 .unwrap()
12021 .unwrap()
12022 .connection_id,
12023 INCUMBENT
12024 );
12025 }
12026 }
12027
12028 mod swap_admission {
12032 use super::*;
12033
12034 const INCUMBENT_NONCE: &str = "incumbent-nonce";
12035 const CANDIDATE_NONCE: &str = "candidate-nonce";
12036
12037 fn handler_with_incumbent(
12038 module_id: &str,
12039 reserved: bool,
12040 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
12041 let registry = Arc::new(Registry::default());
12042 let supervisor = SupervisorHandle::new();
12043 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
12044 if reserved {
12045 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
12046 }
12047 let handler =
12048 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
12049 let incumbent = handler
12050 .handle_control(
12051 ConnectionId::new(1),
12052 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
12053 )
12054 .unwrap();
12055 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
12056 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
12057 (registry, supervisor, handler)
12058 }
12059
12060 #[test]
12066 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
12067 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
12068
12069 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
12070 let replies = handler
12071 .handle_control(
12072 ConnectionId::new(connection),
12073 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
12074 )
12075 .unwrap();
12076 assert_eq!(replies[0].header.ty, FrameType::Error);
12077 assert_eq!(
12078 parse_error(&replies[0])["code"],
12079 "swap_token_invalid",
12080 "nonce {nonce:?}"
12081 );
12082 }
12083 assert!(registry.get_candidate("aft").unwrap().is_none());
12084 assert_eq!(
12085 registry.get_module("aft").unwrap().unwrap().connection_id,
12086 ConnectionId::new(1)
12087 );
12088
12089 let admitted = handler
12092 .handle_control(
12093 ConnectionId::new(4),
12094 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
12095 )
12096 .unwrap();
12097 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
12098 assert_eq!(
12099 registry
12100 .get_candidate("aft")
12101 .unwrap()
12102 .unwrap()
12103 .connection_id,
12104 ConnectionId::new(4)
12105 );
12106 assert_eq!(
12107 registry.get_module("aft").unwrap().unwrap().connection_id,
12108 ConnectionId::new(1),
12109 "the candidate must not take the active slot"
12110 );
12111 let replayed = handler
12112 .handle_control(
12113 ConnectionId::new(5),
12114 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
12115 )
12116 .unwrap();
12117 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
12118
12119 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
12123 let squatter = handler
12124 .handle_control(
12125 ConnectionId::new(6),
12126 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
12127 )
12128 .unwrap();
12129 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
12130 assert!(
12131 registry.get_module("aft").unwrap().is_none(),
12132 "a squatter took the active slot of an id being swapped"
12133 );
12134 }
12135
12136 #[test]
12141 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
12142 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
12143
12144 let replies = handler
12145 .handle_control(
12146 ConnectionId::new(2),
12147 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
12148 )
12149 .unwrap();
12150
12151 assert_eq!(
12152 replies[0].header.ty,
12153 FrameType::HelloAck,
12154 "reserved candidate refused: {:?}",
12155 serde_json::from_slice::<Value>(&replies[0].body).ok()
12156 );
12157 assert_eq!(
12158 registry
12159 .get_candidate("vault")
12160 .unwrap()
12161 .unwrap()
12162 .connection_id,
12163 ConnectionId::new(2)
12164 );
12165 }
12166
12167 #[test]
12170 fn without_an_open_swap_the_ordinary_gates_decide() {
12171 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
12172 supervisor.close_swap("vault");
12173
12174 let candidate = handler
12175 .handle_control(
12176 ConnectionId::new(2),
12177 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
12178 )
12179 .unwrap();
12180 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
12181 let duplicate = handler
12182 .handle_control(
12183 ConnectionId::new(3),
12184 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
12185 )
12186 .unwrap();
12187 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
12188 assert!(registry.get_candidate("vault").unwrap().is_none());
12189 }
12190 }
12191
12192 mod scopes {
12196 use subc_protocol::scope::{
12197 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
12198 ScopeStatus,
12199 };
12200
12201 use super::*;
12202
12203 const OWNER: &str = "prefrontal-core";
12204
12205 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
12206 ScopeRecord {
12207 scope_ref: scope_ref.to_string(),
12208 scope_epoch,
12209 kind: ScopeKind::Head,
12210 parent: None,
12211 child_owners: Vec::new(),
12212 carriers: Vec::new(),
12213 attributes: Default::default(),
12214 }
12215 }
12216
12217 async fn call(
12218 handler: &ControlHandler,
12219 ctx: &RouteCtx,
12220 request: &ModuleControlRequestFromModule,
12221 ) -> Frame {
12222 let body = serde_json::to_vec(request).unwrap();
12223 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
12224 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
12225 assert_eq!(replies.len(), 1, "{replies:?}");
12226 replies.pop().unwrap()
12227 }
12228
12229 async fn sync(
12230 handler: &ControlHandler,
12231 ctx: &RouteCtx,
12232 generation: u64,
12233 scopes: Vec<ScopeRecord>,
12234 ) -> Result<ModuleControlResponseToModule, String> {
12235 let reply = call(
12236 handler,
12237 ctx,
12238 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
12239 )
12240 .await;
12241 match reply.header.ty {
12242 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
12243 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
12244 }
12245 }
12246
12247 async fn describe(
12248 handler: &ControlHandler,
12249 ctx: &RouteCtx,
12250 owner: &str,
12251 scope_ref: &str,
12252 ) -> ModuleControlResponseToModule {
12253 let reply = call(
12254 handler,
12255 ctx,
12256 &ModuleControlRequestFromModule::ScopeDescribe {
12257 owner: Principal::Reserved {
12258 module_id: owner.to_string(),
12259 },
12260 scope_ref: scope_ref.to_string(),
12261 },
12262 )
12263 .await;
12264 assert_eq!(
12265 reply.header.ty,
12266 FrameType::Response,
12267 "{:?}",
12268 parse_error(&reply)
12269 );
12270 serde_json::from_slice(&reply.body).unwrap()
12271 }
12272
12273 async fn module(
12275 handler: &ControlHandler,
12276 connection: u64,
12277 module_id: &str,
12278 nonce: Option<&str>,
12279 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
12280 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
12281 hello_via_sink(
12282 handler,
12283 &ctx,
12284 &mut rx,
12285 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
12286 )
12287 .await;
12288 (ctx, rx)
12289 }
12290
12291 #[tokio::test]
12294 async fn a_client_connection_cannot_sync_or_describe() {
12295 let handler = ControlHandler::new(Arc::new(Registry::default()));
12296 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
12297 for request in [
12298 ModuleControlRequestFromModule::ScopeSync {
12299 generation: 1,
12300 scopes: vec![head("s", 1)],
12301 },
12302 ModuleControlRequestFromModule::ScopeDescribe {
12303 owner: Principal::Direct,
12304 scope_ref: "s".to_string(),
12305 },
12306 ] {
12307 let reply = call(&handler, &ctx, &request).await;
12308 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
12309 }
12310 assert!(
12311 !handler
12312 .scopes
12313 .read()
12314 .unwrap()
12315 .describe(
12316 &Principal::Reserved {
12317 module_id: OWNER.to_string()
12318 },
12319 "s"
12320 )
12321 .owner_synced
12322 );
12323 }
12324
12325 #[tokio::test]
12328 async fn a_module_without_a_supervised_launch_cannot_sync() {
12329 let handler = ControlHandler::new(Arc::new(Registry::default()));
12330 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
12331 assert_eq!(
12332 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
12333 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
12334 );
12335 }
12336
12337 #[tokio::test]
12338 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
12339 let supervisor = SupervisorHandle::new();
12340 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
12341 let handler = ControlHandler::new(Arc::new(Registry::default()))
12342 .with_supervisor(supervisor.clone());
12343 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
12344 sync(&handler, &incumbent, 1, vec![head("s", 1)])
12345 .await
12346 .expect("the current launch syncs");
12347
12348 supervisor.open_swap(OWNER, "n2".to_string());
12351 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
12352 assert_eq!(
12353 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
12354 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
12355 );
12356 sync(&handler, &incumbent, 2, vec![head("s", 1)])
12357 .await
12358 .expect("the serving owner syncs during the swap");
12359
12360 supervisor.close_swap(OWNER);
12363 assert_eq!(
12364 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
12365 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
12366 );
12367 sync(&handler, &incumbent, 3, vec![head("s", 1)])
12368 .await
12369 .expect("the serving owner syncs after the rollback");
12370 handler.cleanup_connection(candidate.connection_id).unwrap();
12371
12372 supervisor.open_swap(OWNER, "n3".to_string());
12377 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
12378 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
12379 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
12380 .await
12381 .expect("the promoted launch takes authority");
12382 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
12383 panic!("unexpected reply {reply:?}");
12384 };
12385 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
12386 assert_eq!(
12387 sync(&handler, &incumbent, 4, Vec::new()).await,
12388 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
12389 );
12390 }
12391
12392 #[tokio::test]
12395 async fn closing_the_authority_connection_frees_sync_authority() {
12396 let supervisor = SupervisorHandle::new();
12397 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
12398 let handler = ControlHandler::new(Arc::new(Registry::default()))
12399 .with_supervisor(supervisor.clone());
12400 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
12401 sync(&handler, &first, 10, vec![head("s", 1)])
12402 .await
12403 .unwrap();
12404 handler.cleanup_connection(first.connection_id).unwrap();
12405
12406 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
12407 sync(&handler, &second, 1, vec![head("s", 1)])
12408 .await
12409 .expect("the next connection takes the released authority");
12410 }
12411
12412 #[tokio::test]
12413 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
12414 let supervisor = SupervisorHandle::new();
12415 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
12416 let handler =
12417 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
12418 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
12419 sync(&handler, &first, 10, vec![head("s", 1)])
12420 .await
12421 .unwrap();
12422 handler
12423 .handle_control_frame(
12424 &first,
12425 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
12426 )
12427 .await
12428 .unwrap();
12429 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
12430 sync(&handler, &second, 1, vec![head("s", 1)])
12431 .await
12432 .expect("GOODBYE releases authority even if the old socket remains open");
12433 }
12434
12435 #[tokio::test]
12436 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
12437 let registry = Arc::new(Registry::default());
12438 let supervisor_handle = SupervisorHandle::new();
12439 let supervisor =
12440 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
12441 .with_handle(supervisor_handle.clone())
12442 .with_daemon_incarnation("incarnation-7".to_string());
12443 supervisor
12446 .supervise_configured(
12447 ModuleSpec {
12448 module_id: OWNER.to_string(),
12449 program: PathBuf::from("/nonexistent/prefrontal-core"),
12450 args: Vec::new(),
12451 env: Vec::new(),
12452 reserved: false,
12453 reserved_prefixes: Vec::new(),
12454 protocol: ModuleProtocol::Subc,
12455 overlap: Default::default(),
12456 },
12457 false,
12458 )
12459 .unwrap();
12460 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
12461 let handler =
12462 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12463 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
12464
12465 let ModuleControlResponseToModule::ScopeDescribe {
12467 status,
12468 daemon_incarnation,
12469 owner_synced,
12470 owner_configured,
12471 scope,
12472 ..
12473 } = describe(&handler, &reader, OWNER, "s").await
12474 else {
12475 panic!("not a describe reply");
12476 };
12477 assert_eq!(status, ScopeStatus::NotLive);
12478 assert_eq!(daemon_incarnation, "incarnation-7");
12479 assert!(!owner_synced);
12480 assert!(owner_configured);
12481 assert!(scope.is_none());
12482
12483 let ModuleControlResponseToModule::ScopeDescribe {
12486 status,
12487 owner_configured,
12488 ..
12489 } = describe(&handler, &reader, "ghost", "s").await
12490 else {
12491 panic!("not a describe reply");
12492 };
12493 assert_eq!(status, ScopeStatus::NotLive);
12494 assert!(!owner_configured);
12495
12496 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
12498 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
12499 let ModuleControlResponseToModule::ScopeDescribe {
12500 status,
12501 scope_epoch,
12502 owner_synced,
12503 scope,
12504 ..
12505 } = describe(&handler, &reader, OWNER, "s").await
12506 else {
12507 panic!("not a describe reply");
12508 };
12509 assert_eq!(status, ScopeStatus::Live);
12510 assert_eq!(scope_epoch, Some(4));
12511 assert!(owner_synced);
12512 let stamp = scope.expect("a live scope carries its stamp");
12513 assert!(
12514 stamp.owner_authorized,
12515 "prefrontal-core is the default authority"
12516 );
12517 assert_eq!(stamp.kind, ScopeKind::Head);
12518 }
12519
12520 #[tokio::test]
12521 async fn scope_authority_owners_decides_owner_authorized() {
12522 let supervisor = SupervisorHandle::new();
12523 supervisor.set_spawn_nonce("broca", "b1".to_string());
12524 let handler = ControlHandler::new(Arc::new(Registry::default()))
12525 .with_supervisor(supervisor)
12526 .with_scope_authority_owners(vec!["broca".to_string()]);
12527 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
12528 let mut gated = head("s", 1);
12529 gated.attributes.agent_id = Some("agent".to_string());
12530 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
12531 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
12532 describe(&handler, &broca, "broca", "s").await
12533 else {
12534 panic!("not a describe reply");
12535 };
12536 assert!(scope.unwrap().owner_authorized);
12537 }
12538
12539 #[tokio::test]
12543 async fn scope_ops_and_the_scopes_capability_are_advertised() {
12544 let handler = ControlHandler::new(Arc::new(Registry::default()));
12545 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
12546 let ack = hello_via_sink(
12547 &handler,
12548 &ctx,
12549 &mut rx,
12550 hello_frame("m", PROTOCOL_VERSION, 1),
12551 )
12552 .await;
12553 let ack = parse_ack(&ack);
12554 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
12555 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
12556 }
12557 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
12558
12559 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
12560 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12561 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12562 let reply = handler
12563 .handle_control_frame(&client, frame)
12564 .await
12565 .unwrap()
12566 .pop()
12567 .unwrap();
12568 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12569 serde_json::from_slice(&reply.body).unwrap()
12570 else {
12571 panic!("not a server.describe reply");
12572 };
12573 assert!(
12574 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
12575 "{capabilities:?}"
12576 );
12577 }
12578
12579 const PLEXUS: &str = "plexus";
12582 const OTHER: &str = "other";
12583 const AFT: &str = "aft";
12584 const BROCA: &str = "broca";
12585 const MAGIC: &str = "magic-context";
12586
12587 fn nonce(module_id: &str) -> String {
12588 format!("nonce-{module_id}")
12589 }
12590
12591 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
12592 let (tx, rx) = mpsc::channel(64);
12593 (
12594 RouteCtx {
12595 connection_id: ConnectionId::new(connection),
12596 egress: FrameSink::new(tx),
12597 },
12598 rx,
12599 )
12600 }
12601
12602 struct Rig {
12606 handler: ControlHandler,
12607 forwarding: Arc<ForwardingTable>,
12608 owner: RouteCtx,
12609 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12610 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
12611 generation: u64,
12612 next_connection: u64,
12613 _supervisor: Supervisor,
12614 }
12615
12616 async fn rig() -> Rig {
12617 rig_with_flow_support(true).await
12618 }
12619
12620 async fn rig_with_flow_support(flow_support: bool) -> Rig {
12621 let registry = Arc::new(Registry::default());
12622 let forwarding = Arc::new(ForwardingTable::default());
12623 let supervisor_handle = SupervisorHandle::new();
12624 let supervisor =
12625 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
12626 .with_handle(supervisor_handle.clone());
12627 supervisor
12628 .supervise_configured(
12629 ModuleSpec {
12630 module_id: OWNER.to_string(),
12631 program: PathBuf::from("/nonexistent/prefrontal-core"),
12632 args: Vec::new(),
12633 env: Vec::new(),
12634 reserved: false,
12635 reserved_prefixes: Vec::new(),
12636 protocol: ModuleProtocol::Subc,
12637 overlap: Default::default(),
12638 },
12639 false,
12640 )
12641 .unwrap();
12642 for module_id in [OWNER, AFT, BROCA, MAGIC] {
12643 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
12644 }
12645 let handler =
12646 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12647 .with_supervisor(supervisor_handle);
12648 let (owner, mut owner_rx) = wide_ctx(1);
12649 hello_via_sink(
12650 &handler,
12651 &owner,
12652 &mut owner_rx,
12653 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
12654 )
12655 .await;
12656 let mut modules = BTreeMap::new();
12657 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
12658 let (ctx, mut rx) = wide_ctx(connection);
12659 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
12660 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
12661 body["manifest"]["provenance"] =
12665 serde_json::json!({"wire_crate_version": "0.29.0"});
12666 if flow_support {
12667 body["manifest"]["capabilities"] =
12668 serde_json::json!({"provides": ["flow-scopes/v1"]});
12669 }
12670 let hello = Frame::build(
12671 FrameType::Hello,
12672 control_flags(),
12673 0,
12674 0,
12675 connection,
12676 serde_json::to_vec(&body).unwrap(),
12677 )
12678 .unwrap();
12679 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
12680 modules.insert(module_id.to_string(), (ctx, rx));
12681 }
12682 Rig {
12683 handler,
12684 forwarding,
12685 owner,
12686 _owner_rx: owner_rx,
12687 modules,
12688 generation: 0,
12689 next_connection: 100,
12690 _supervisor: supervisor,
12691 }
12692 }
12693
12694 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
12695 ScopeCarrier {
12696 principal: Principal::Reserved {
12697 module_id: module_id.to_string(),
12698 },
12699 targets: targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()),
12700 }
12701 }
12702
12703 fn session(scope_epoch: u64) -> ScopeRecord {
12706 let mut record = head("s", scope_epoch);
12707 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
12708 record.attributes.agent_id = Some("agent-1".to_string());
12709 record.attributes.delegates = true;
12710 record
12711 }
12712
12713 impl Rig {
12714 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
12715 self.generation += 1;
12716 sync(&self.handler, &self.owner, self.generation, scopes)
12717 .await
12718 .expect("the owner's sync is accepted");
12719 }
12720
12721 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
12722 ScopeSelector {
12723 owner: Principal::Reserved {
12724 module_id: OWNER.to_string(),
12725 },
12726 scope_ref: scope_ref.to_string(),
12727 scope_epoch,
12728 }
12729 }
12730
12731 fn open_frame(
12732 &mut self,
12733 opener: Option<&str>,
12734 target: &str,
12735 scope: Option<ScopeSelector>,
12736 ) -> (
12737 RouteCtx,
12738 mpsc::Receiver<crate::router::OutboundFrame>,
12739 Frame,
12740 ) {
12741 self.next_connection += 1;
12742 let (ctx, rx) = wide_ctx(self.next_connection);
12743 let root = unique_project_root("scoped-open");
12744 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
12745 target: RouteTarget::ToolProvider {
12746 module_id: target.to_string(),
12747 },
12748 identity: BindIdentity::new(
12749 root.path().to_path_buf(),
12750 "unit".to_string(),
12751 "session".to_string(),
12752 ),
12753 consumer_identity: opener.map(|module_id| ConsumerIdentity {
12754 module_id: module_id.to_string(),
12755 launch_nonce: nonce(module_id),
12756 }),
12757 consumer_capabilities: None,
12758 role_versions: None,
12759 admission_facts: None,
12760 scope,
12761 })
12762 .unwrap();
12763 let frame = Frame::build(
12764 FrameType::Request,
12765 control_flags(),
12766 0,
12767 0,
12768 self.next_connection,
12769 body,
12770 )
12771 .unwrap();
12772 (ctx, rx, frame)
12773 }
12774
12775 async fn refused(
12777 &mut self,
12778 opener: Option<&str>,
12779 target: &str,
12780 scope: Option<ScopeSelector>,
12781 ) -> String {
12782 self.refusal_body(opener, target, scope).await["code"]
12783 .as_str()
12784 .unwrap()
12785 .to_string()
12786 }
12787
12788 async fn refusal_body(
12789 &mut self,
12790 opener: Option<&str>,
12791 target: &str,
12792 scope: Option<ScopeSelector>,
12793 ) -> Value {
12794 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
12795 let replies = tokio::time::timeout(
12796 Duration::from_secs(2),
12797 self.handler.handle_control_frame(&ctx, frame),
12798 )
12799 .await
12800 .expect("the open must be refused before waiting for a bind ack")
12801 .unwrap();
12802 assert_eq!(replies.len(), 1, "{replies:?}");
12803 assert_eq!(replies[0].header.ty, FrameType::Error);
12804 let (_, module_rx) = self.modules.get_mut(target).unwrap();
12805 assert!(
12806 module_rx.try_recv().is_err(),
12807 "a refused open relays nothing"
12808 );
12809 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
12810 parse_error(&replies[0])
12811 }
12812
12813 async fn relayed(
12815 &mut self,
12816 opener: Option<&str>,
12817 target: &str,
12818 scope: Option<ScopeSelector>,
12819 ) -> Relayed {
12820 let (ctx, rx, frame) = self.open_frame(opener, target, scope);
12821 let handler = self.handler.clone();
12822 let task_ctx = ctx.clone();
12823 let task = tokio::spawn(async move {
12824 handler
12825 .handle_control_frame(&task_ctx, frame)
12826 .await
12827 .unwrap()
12828 });
12829 let (_, module_rx) = self.modules.get_mut(target).unwrap();
12830 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
12831 .await
12832 .expect("the target receives the relayed route.bind")
12833 .unwrap()
12834 .frame;
12835 Relayed {
12836 target: target.to_string(),
12837 client: ctx,
12838 client_rx: rx,
12839 task,
12840 bind,
12841 }
12842 }
12843
12844 async fn ack(&self, relayed: &Relayed) {
12845 let (module, _) = &self.modules[&relayed.target];
12846 self.handler
12847 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
12848 .await
12849 .unwrap();
12850 }
12851
12852 async fn bound(
12854 &mut self,
12855 opener: Option<&str>,
12856 target: &str,
12857 scope: Option<ScopeSelector>,
12858 ) -> Bound {
12859 let relayed = self.relayed(opener, target, scope).await;
12860 self.ack(&relayed).await;
12861 let Relayed {
12862 target,
12863 client,
12864 mut client_rx,
12865 task,
12866 bind,
12867 } = relayed;
12868 assert!(
12869 task.await.unwrap().is_empty(),
12870 "the open is answered by commit"
12871 );
12872 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
12873 Bound {
12874 target,
12875 client,
12876 client_rx,
12877 channel,
12878 epoch,
12879 bind,
12880 }
12881 }
12882
12883 fn live(&self, route: &Bound) -> bool {
12884 matches!(
12885 self.forwarding
12886 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
12887 .unwrap(),
12888 DataRoute::Client(DataRouteState::Bound(_))
12889 )
12890 }
12891 }
12892
12893 struct Relayed {
12894 target: String,
12895 client: RouteCtx,
12896 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12897 task: tokio::task::JoinHandle<Vec<Frame>>,
12898 bind: Frame,
12899 }
12900
12901 struct Bound {
12902 target: String,
12903 client: RouteCtx,
12904 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12905 channel: u16,
12906 epoch: u32,
12907 bind: Frame,
12908 }
12909
12910 impl Bound {
12911 fn closed_reason(&mut self) -> RouteCloseReason {
12914 let mut reason = None;
12915 let mut goodbye = false;
12916 while let Ok(outbound) = self.client_rx.try_recv() {
12917 let frame = outbound.frame;
12918 match frame.header.ty {
12919 FrameType::Goodbye => {
12920 assert_eq!(
12921 (frame.header.channel, frame.header.epoch),
12922 (self.channel, self.epoch)
12923 );
12924 goodbye = true;
12925 }
12926 FrameType::Push => {
12927 let ClientControlPush::RouteClosed {
12928 reason: r,
12929 module_id,
12930 ..
12931 } = serde_json::from_slice(&frame.body).unwrap()
12932 else {
12933 panic!("unexpected push");
12934 };
12935 assert_eq!(module_id, self.target);
12936 reason = Some(r);
12937 }
12938 other => panic!("unexpected frame {other:?}"),
12939 }
12940 }
12941 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
12942 reason.expect("the client is told why the route closed")
12943 }
12944
12945 fn untouched(&mut self) -> bool {
12946 self.client_rx.try_recv().is_err()
12947 }
12948
12949 fn stamp(&self) -> Option<ScopeStamp> {
12950 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
12951 ModuleControlRequest::RouteBind { scope, .. } => scope,
12952 other => panic!("expected a route.bind, got {other:?}"),
12953 }
12954 }
12955 }
12956
12957 #[tokio::test]
12958 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
12959 ) {
12960 let mut rig = rig().await;
12961 let mut record = session(1);
12962 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
12963 record.child_owners = vec![Principal::Reserved {
12964 module_id: MAGIC.to_string(),
12965 }];
12966 rig.sync(vec![record]).await;
12967 let scope = || Some(rig_selector("s", Some(1)));
12968
12969 rig.bound(Some(OWNER), PLEXUS, scope()).await;
12972 rig.bound(Some(AFT), OTHER, scope()).await;
12973 rig.bound(Some(BROCA), PLEXUS, scope()).await;
12974
12975 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
12979 assert_eq!(
12980 rig.refused(opener, target, scope()).await,
12981 error_codes::SCOPE_NOT_CARRIER,
12982 "{opener:?} -> {target}"
12983 );
12984 }
12985 }
12986
12987 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
12988 ScopeSelector {
12989 owner: Principal::Reserved {
12990 module_id: OWNER.to_string(),
12991 },
12992 scope_ref: scope_ref.to_string(),
12993 scope_epoch,
12994 }
12995 }
12996
12997 #[tokio::test]
12998 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
12999 let mut rig = rig().await;
13000 rig.sync(vec![session(1)]).await;
13001 for opener in [OWNER, AFT] {
13002 assert_eq!(
13003 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
13004 .await,
13005 error_codes::SCOPE_EPOCH_REQUIRED,
13006 "{opener}"
13007 );
13008 }
13009 }
13010
13011 #[tokio::test]
13012 async fn admission_separates_not_synced_not_live_and_ended() {
13013 let mut rig = rig().await;
13014 let code = rig
13016 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13017 .await;
13018 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
13019 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
13020
13021 let ghost = ScopeSelector {
13023 owner: Principal::Reserved {
13024 module_id: "ghost".to_string(),
13025 },
13026 scope_ref: "s".to_string(),
13027 scope_epoch: Some(1),
13028 };
13029 assert_eq!(
13030 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
13031 error_codes::SCOPE_NOT_LIVE
13032 );
13033
13034 rig.sync(vec![session(2)]).await;
13035 assert_eq!(
13036 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
13037 .await,
13038 error_codes::SCOPE_NOT_LIVE
13039 );
13040 for epoch in [1, 3] {
13041 assert_eq!(
13042 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
13043 .await,
13044 error_codes::SCOPE_ENDED,
13045 "epoch {epoch}"
13046 );
13047 }
13048 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
13050 .await;
13051 }
13052
13053 #[tokio::test]
13054 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
13055 let mut rig = rig().await;
13056 rig.sync(vec![session(1)]).await;
13057 let route = rig
13058 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13059 .await;
13060 let stamp = route.stamp().expect("a scoped bind carries the stamp");
13061 assert_eq!(stamp.scope_ref, "s");
13062 assert_eq!(stamp.scope_epoch, 1);
13063 assert_eq!(stamp.kind, ScopeKind::Head);
13064 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
13065 assert!(stamp.attributes.delegates);
13066 assert!(stamp.owner_authorized);
13067 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
13068 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
13069
13070 let (broca, mut broca_rx) = wide_ctx(50);
13073 hello_via_sink(
13074 &rig.handler,
13075 &broca,
13076 &mut broca_rx,
13077 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
13078 )
13079 .await;
13080 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
13081 .await
13082 .unwrap();
13083 let own = ScopeSelector {
13084 owner: Principal::Reserved {
13085 module_id: BROCA.to_string(),
13086 },
13087 scope_ref: "b".to_string(),
13088 scope_epoch: Some(1),
13089 };
13090 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
13091 assert!(!route.stamp().unwrap().owner_authorized);
13092 }
13093
13094 #[tokio::test]
13095 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
13096 let mut rig = rig().await;
13097 let mut record = head("s", 1);
13098 record.carriers = vec![carrier(AFT, None)];
13099 let flow_id = "Flow:run-7/step_2!~";
13100 record.attributes.flow_id = Some(flow_id.to_string());
13101 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
13102 .await
13103 .unwrap();
13104 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
13105 panic!("not a sync reply");
13106 };
13107 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
13108 let route = rig
13109 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13110 .await;
13111 assert!(rig.live(&route), "the stamped bind committed");
13112 let stamp = route.stamp().expect("a flow scope carries a stamp");
13113 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
13114 assert_eq!(stamp.attributes.agent_id, None);
13115 assert!(!stamp.attributes.delegates);
13116 assert!(stamp.owner_authorized);
13117 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
13118 assert_eq!(unscoped.stamp(), None);
13119 }
13120
13121 #[tokio::test]
13122 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
13123 let mut rig = rig_with_flow_support(false).await;
13124 let mut record = session(1);
13125 record.attributes.flow_id = Some("flow:7".to_string());
13126 rig.sync(vec![record]).await;
13127 for opener in [OWNER, AFT] {
13128 let body = rig
13129 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
13130 .await;
13131 assert_eq!(body["code"], "target_flow_unsupported");
13132 let message = body["message"].as_str().unwrap();
13133 for required in [PLEXUS, "flow-scopes/v1"] {
13134 assert!(message.contains(required), "{message}");
13135 }
13136 }
13137 }
13138
13139 #[tokio::test]
13140 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
13141 let mut rig = rig_with_flow_support(true).await;
13142 let mut record = session(1);
13143 record.attributes.flow_id = Some("flow:7".to_string());
13144 rig.sync(vec![record]).await;
13145 for opener in [OWNER, AFT] {
13146 let route = rig
13147 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
13148 .await;
13149 assert!(rig.live(&route));
13150 assert_eq!(
13151 route.stamp().unwrap().attributes.flow_id.as_deref(),
13152 Some("flow:7")
13153 );
13154 }
13155 }
13156
13157 #[tokio::test]
13158 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
13159 use std::future::Future;
13160
13161 let mut rig = rig_with_flow_support(true).await;
13162 let mut record = session(1);
13163 record.attributes.flow_id = Some("flow:7".to_string());
13164 rig.sync(vec![record]).await;
13165 let (client, mut client_rx, frame) =
13166 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
13167 for _ in 0..64 {
13170 client.egress.try_send(route_bind_ack(1)).unwrap();
13171 }
13172 let handler = rig.handler.clone();
13173 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
13174 std::future::poll_fn(|cx| {
13175 assert!(open.as_mut().poll(cx).is_pending());
13176 std::task::Poll::Ready(())
13177 })
13178 .await;
13179 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
13180
13181 let old_connection = rig.modules[PLEXUS].0.connection_id;
13182 rig.handler.cleanup_connection(old_connection).unwrap();
13183 let (replacement, mut replacement_rx) = wide_ctx(200);
13184 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
13185 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
13186 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
13187 let hello = Frame::build(
13188 FrameType::Hello,
13189 control_flags(),
13190 0,
13191 0,
13192 200,
13193 serde_json::to_vec(&body).unwrap(),
13194 )
13195 .unwrap();
13196 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
13197
13198 client_rx.try_recv().unwrap();
13199 let replies = tokio::time::timeout(Duration::from_secs(2), open)
13200 .await
13201 .expect("the replacement is refused without waiting for a bind ack")
13202 .unwrap();
13203 assert_eq!(replies.len(), 1);
13204 let body = parse_error(&replies[0]);
13205 assert_eq!(body["code"], "target_flow_unsupported");
13206 for required in [PLEXUS, "flow-scopes/v1"] {
13207 assert!(body["message"].as_str().unwrap().contains(required));
13208 }
13209 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
13210 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
13211 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
13212 assert!(rig
13213 .handler
13214 .registry
13215 .get_module_by_connection(replacement.connection_id)
13216 .unwrap()
13217 .is_some());
13218 }
13219
13220 #[tokio::test]
13221 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
13222 {
13223 let mut rig = rig_with_flow_support(false).await;
13224 rig.sync(vec![session(1)]).await;
13225 let route = rig
13226 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13227 .await;
13228 assert!(rig.live(&route));
13229 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
13230 let mut record = session(1);
13231 record.attributes.flow_id = Some("flow:7".to_string());
13232 rig.sync(vec![record]).await;
13233 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
13234 assert!(rig.live(&unscoped));
13235 assert_eq!(unscoped.stamp(), None);
13236 }
13237
13238 #[tokio::test]
13239 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
13240 let mut rig = rig().await;
13241 let mut record = session(1);
13242 record.attributes.flow_id = Some("flow:7".to_string());
13243 rig.sync(vec![record.clone()]).await;
13244 let before = rig.forwarding.published_scope_tag(OWNER, "s");
13245 let mut owner_route = rig
13246 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
13247 .await;
13248 let mut carrier_route = rig
13249 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
13250 .await;
13251 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
13252 rig.sync(vec![record.clone()]).await;
13253 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
13254 assert!(rig.live(&owner_route) && owner_route.untouched());
13255 assert!(rig.live(&carrier_route) && carrier_route.untouched());
13256
13257 record.attributes.flow_id = Some("flow:8".to_string());
13258 rig.sync(vec![record]).await;
13259 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
13260 let before = before.unwrap();
13261 assert_eq!(after.scope_epoch, before.scope_epoch);
13262 assert!(after.version > before.version);
13263 for route in [&mut owner_route, &mut carrier_route] {
13264 assert!(!rig.live(route));
13265 assert_eq!(
13266 route.closed_reason(),
13267 RouteCloseReason::ScopeDelegationChanged
13268 );
13269 }
13270 assert!(rig.live(&unscoped) && unscoped.untouched());
13271 for target in [PLEXUS, OTHER] {
13274 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
13275 let goodbye = module_rx
13276 .try_recv()
13277 .expect("the provider sees the drain")
13278 .frame;
13279 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
13280 assert!(module_rx.try_recv().is_err());
13281 }
13282 let rebound = rig
13283 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13284 .await;
13285 assert_eq!(
13286 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
13287 Some("flow:8")
13288 );
13289 }
13290
13291 #[tokio::test]
13296 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
13297 let mut rig = rig().await;
13298 rig.sync(vec![session(1)]).await;
13299 let mut cotenant = rig
13300 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
13301 .await;
13302
13303 let mut changed = session(1);
13304 changed.child_owners.push(Principal::Reserved {
13305 module_id: MAGIC.to_string(),
13306 });
13307 let mut ended = None;
13308 for (code, next) in [
13309 (error_codes::SCOPE_CHANGED, vec![changed]),
13310 (error_codes::SCOPE_ENDED, Vec::new()),
13311 ] {
13312 let relayed = rig
13313 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13314 .await;
13315 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
13316 ended = Some(next.is_empty());
13317 rig.sync(next).await;
13318 rig.ack(&relayed).await;
13319 let replies = relayed.task.await.unwrap();
13320 assert_eq!(replies.len(), 1, "{replies:?}");
13321 assert_eq!(parse_error(&replies[0])["code"], code);
13322 assert_eq!(
13323 subc_protocol::error_codes::is_retryable_route_open(code),
13324 code == error_codes::SCOPE_CHANGED
13325 );
13326 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
13327 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
13329 let mut goodbyes = Vec::new();
13332 while let Ok(outbound) = plexus_rx.try_recv() {
13333 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
13334 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
13335 }
13336 assert!(
13337 goodbyes.contains(&(bind_channel, bind_epoch)),
13338 "{goodbyes:?}"
13339 );
13340 assert!(rig
13341 .handler
13342 .registry
13343 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
13344 .unwrap()
13345 .is_some());
13346 }
13347 assert_eq!(ended, Some(true));
13348 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
13351 }
13352
13353 #[tokio::test]
13356 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
13357 struct Case {
13358 name: &'static str,
13359 change: fn(&mut ScopeRecord),
13360 closed: [Option<RouteCloseReason>; 4],
13363 }
13364 use RouteCloseReason::*;
13365 let cases = [
13366 Case {
13367 name: "a carrier entry removed",
13368 change: |r| {
13369 r.carriers.retain(|c| {
13370 c.principal
13371 != Principal::Reserved {
13372 module_id: AFT.to_string(),
13373 }
13374 })
13375 },
13376 closed: [None, Some(ScopeCarrierRemoved), None, None],
13377 },
13378 Case {
13379 name: "a target removed from a carrier",
13380 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
13381 closed: [None, None, None, Some(ScopeCarrierRemoved)],
13382 },
13383 Case {
13384 name: "a bare carrier narrowed to targets",
13385 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
13386 closed: [None, Some(ScopeCarrierRemoved), None, None],
13387 },
13388 Case {
13389 name: "delegates turned off",
13390 change: |r| r.attributes.delegates = false,
13391 closed: [Some(ScopeDelegationChanged); 4],
13392 },
13393 Case {
13394 name: "agent_id changed",
13395 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
13396 closed: [Some(ScopeDelegationChanged); 4],
13397 },
13398 Case {
13399 name: "a carrier added, child owners changed, the record re-sent",
13400 change: |r| {
13401 r.carriers.push(carrier(MAGIC, None));
13402 r.child_owners.push(Principal::Reserved {
13403 module_id: MAGIC.to_string(),
13404 });
13405 },
13406 closed: [None; 4],
13407 },
13408 Case {
13409 name: "a target added",
13410 change: |r| {
13411 r.carriers[1]
13412 .targets
13413 .as_mut()
13414 .unwrap()
13415 .push("third".to_string())
13416 },
13417 closed: [None; 4],
13418 },
13419 Case {
13420 name: "delegates turned on",
13421 change: |r| r.attributes.delegates = true,
13422 closed: [None; 4],
13423 },
13424 ];
13425 for case in cases {
13426 let mut rig = rig().await;
13427 rig.sync(vec![session(1)]).await;
13428 let scope = || Some(rig_selector("s", Some(1)));
13429 let mut routes = [
13430 rig.bound(Some(OWNER), PLEXUS, scope()).await,
13431 rig.bound(Some(AFT), PLEXUS, scope()).await,
13432 rig.bound(Some(BROCA), PLEXUS, scope()).await,
13433 rig.bound(Some(BROCA), OTHER, scope()).await,
13434 ];
13435 let mut record = session(1);
13436 (case.change)(&mut record);
13437 rig.sync(vec![record]).await;
13438 for (index, expected) in case.closed.iter().enumerate() {
13439 let route = &mut routes[index];
13440 match expected {
13441 Some(reason) => {
13442 assert!(!rig.live(route), "{}: route {index} still live", case.name);
13443 assert_eq!(
13444 route.closed_reason(),
13445 *reason,
13446 "{}: route {index}",
13447 case.name
13448 );
13449 }
13450 None => {
13451 assert!(rig.live(route), "{}: route {index} closed", case.name);
13452 assert!(
13453 route.untouched(),
13454 "{}: route {index} was told something",
13455 case.name
13456 );
13457 }
13458 }
13459 }
13460 }
13461 }
13462
13463 #[tokio::test]
13464 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
13465 for next in [Vec::new(), vec![session(2)]] {
13467 let mut rig = rig().await;
13468 rig.sync(vec![session(1)]).await;
13469 let mut route = rig
13470 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13471 .await;
13472 rig.sync(next).await;
13473 assert!(!rig.live(&route));
13474 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
13475 }
13476
13477 let mut rig = rig().await;
13480 let mut child = session(1);
13481 child.scope_ref = "child".to_string();
13482 child.kind = ScopeKind::Worker;
13483 child.parent = Some(ScopeParent {
13484 owner: Principal::Reserved {
13485 module_id: OWNER.to_string(),
13486 },
13487 scope_ref: "s".to_string(),
13488 scope_epoch: 1,
13489 });
13490 rig.sync(vec![session(1), child.clone()]).await;
13491 let mut child_route = rig
13492 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
13493 .await;
13494 assert_eq!(
13495 child_route.stamp().unwrap().parent_state,
13496 Some(ParentState::Linked)
13497 );
13498 rig.sync(vec![child]).await;
13499 assert!(!rig.live(&child_route));
13500 assert_eq!(
13501 child_route.closed_reason(),
13502 RouteCloseReason::ScopeParentEnded
13503 );
13504 }
13505
13506 #[tokio::test]
13507 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
13508 ) {
13509 let mut rig = rig().await;
13510 rig.sync(vec![session(1)]).await;
13511 let mut route = rig
13512 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13513 .await;
13514 let before = rig.forwarding.published_scope_tag(OWNER, "s");
13515 rig.sync(vec![session(1)]).await;
13516 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
13517 assert!(rig.live(&route) && route.untouched());
13518
13519 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
13523 .forwarding
13524 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
13525 .unwrap()
13526 else {
13527 panic!("the route is bound");
13528 };
13529 binding.flow.acquire_tagged(9, false).await.unwrap();
13530 let mut widened = session(1);
13531 widened.carriers.push(carrier(MAGIC, None));
13532 rig.sync(vec![widened]).await;
13533 assert!(rig.live(&route) && route.untouched());
13534 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
13535 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
13536 assert_eq!(binding.flow.in_flight(), 1);
13539 binding
13540 .flow
13541 .acquire_tagged(10, false)
13542 .await
13543 .expect("the flow is still open");
13544 }
13545
13546 #[tokio::test]
13549 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
13550 let mut rig = rig().await;
13551 rig.sync(vec![session(1)]).await;
13552 let mut on_incumbent = rig
13553 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13554 .await;
13555
13556 let (candidate, _candidate_rx) = wide_ctx(9);
13559 let registration = rig
13560 .handler
13561 .registry
13562 .register_candidate_with_control_ops(
13563 manifest(PLEXUS, PROTOCOL_VERSION),
13564 PROTOCOL_VERSION,
13565 candidate.connection_id,
13566 module_baseline_control_ops(),
13567 )
13568 .unwrap();
13569 rig.forwarding
13570 .register_candidate_module_connection(
13571 candidate.connection_id,
13572 PLEXUS.to_string(),
13573 PROTOCOL_VERSION,
13574 manifest_concurrency(®istration.manifest),
13575 candidate.egress.clone(),
13576 )
13577 .unwrap();
13578 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
13579 rig.handler
13580 .registry
13581 .promote_candidate(PLEXUS)
13582 .unwrap()
13583 .unwrap();
13584 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
13585
13586 rig.sync(Vec::new()).await;
13587 assert!(!rig.live(&on_incumbent));
13588 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
13589 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
13590 let goodbye = incumbent_rx
13591 .try_recv()
13592 .expect("the superseded endpoint is told")
13593 .frame;
13594 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
13595 }
13596 }
13597}
13598
13599#[cfg(test)]
13600mod concurrency_default_exposure_tests {
13601 use super::*;
13602
13603 fn hello_body(role_json: &str) -> Vec<u8> {
13604 format!(
13605 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
13606 )
13607 .into_bytes()
13608 }
13609
13610 fn manifest_from(body: &[u8]) -> ModuleManifest {
13611 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
13612 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
13613 .expect("manifest parses")
13614 }
13615
13616 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
13617
13618 #[test]
13619 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
13620 let body = hello_body(&format!(
13621 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
13622 ));
13623 let manifest = manifest_from(&body);
13624 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
13627 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
13628 }
13629
13630 #[test]
13631 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
13632 let body = hello_body(&format!(
13633 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
13634 ));
13635 let manifest = manifest_from(&body);
13636 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
13637 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
13638 }
13639
13640 #[test]
13641 fn non_management_roles_are_never_reported() {
13642 let body = hello_body(
13643 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
13644 );
13645 let manifest = manifest_from(&body);
13646 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
13647 }
13648}