1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 path::{Path, PathBuf},
5 sync::{Arc, Mutex, RwLock},
6 time::{Duration, Instant as StdInstant},
7};
8
9use serde::{Deserialize, Serialize};
10use subc_control::{
11 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
12 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
13 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
14 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
15 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
16 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
17 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
18};
19use subc_protocol::{
20 error_codes,
21 manifest::{
22 validate_hello_capability_grammar, validate_hello_event_declarations,
23 validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
24 Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
25 },
26 scope::{
27 ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
28 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_DESCRIBE_OP, SCOPE_SYNC_OP,
29 },
30 session::{
31 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
32 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
33 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
34 ROLE_VERSIONS_FIELD,
35 },
36 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
37 Priority, RouteTarget, PROTOCOL_VERSION,
38};
39use tokio::time::{timeout_at, Instant};
40use tracing::{debug, info, warn};
41
42use crate::{
43 capability_requirements::{
44 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
45 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
46 RuntimeModule,
47 },
48 daemon_config::RestartRequiredSection,
49 forwarding::{
50 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
51 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
52 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
53 },
54 observability::{
55 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
56 },
57 provenance::{
58 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
59 },
60 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
61 router::{RouteCtx, RouterError},
62 scopes::{BoundScope, HelloLaunchNonces, ScopeTable},
63 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
64 stderr_tail::{CaptureState, TailEntry},
65 supervise::{
66 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
67 SupervisorHandle, SwapHelloAdmission,
68 },
69 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
70};
71
72pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
78
79const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
80const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
81const CAP_PING_PONG: &str = "ping_pong_v1";
82const CAP_SESSION_ATTACH: &str = "session_attach_v1";
83const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
84
85const SUBC_CONTROL_OPS: &[&str] = &[
86 ops::SERVER_DESCRIBE,
87 ops::CATALOG_LIST,
88 ops::ROUTE_OPEN,
89 ops::ROUTE_POLL,
90 ops::ROUTE_CLOSING,
91 ops::ROUTE_CLOSED,
92 ops::SUPERVISOR_LIST,
93 ops::SUPERVISOR_RESTART,
94 ops::SUPERVISOR_SWAP,
95 ops::SUPERVISOR_RELOAD,
96 ops::SUPERVISOR_RESCAN,
97 ops::SUPERVISOR_RELEASE_RESERVED,
98 ops::SUPERVISOR_SET_ENABLED,
99 ops::SUPERVISOR_HEALTH_PROBE,
100 ops::SUPERVISOR_HEALTH,
101 ops::SUPERVISOR_STDERR_TAIL,
102 ops::SUPERVISOR_TERMINALS,
103 ops::SUPERVISOR_ROUTES,
104 ops::SUPERVISOR_PROVENANCE,
105 ops::SUPERVISOR_SPAWN_SNAPSHOT,
106 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
107];
108
109const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
110 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
111 "supervisor.live_roots",
112 SCOPE_SYNC_OP,
113 SCOPE_DESCRIBE_OP,
114 "operator.confirm",
115];
116
117const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
121
122const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
123
124pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
132
133pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
148
149pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
167
168const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
169const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
170
171fn reload_verdict(
172 configured: &Path,
173 spawned_from: Option<&Path>,
174 image: subc_control::RunningImageAgreement,
175) -> PendingReloadVerdict {
176 let path = match spawned_from {
177 Some(spawned_from) if configured == spawned_from => ReloadPathAgreement::Match,
178 Some(spawned_from) => ReloadPathAgreement::Mismatch {
179 configured: configured.to_path_buf(),
180 spawned_from: spawned_from.to_path_buf(),
181 },
182 None => ReloadPathAgreement::Unavailable {
183 reason: if matches!(
184 image,
185 subc_control::RunningImageAgreement::Unavailable {
186 reason: subc_control::RunningImageUnavailableReason::NotRunning
187 }
188 ) {
189 ReloadPathUnavailableReason::NotRunning
190 } else {
191 ReloadPathUnavailableReason::SpawnedPathUnavailable
192 },
193 },
194 };
195 PendingReloadVerdict { path, image }
196}
197
198#[derive(Clone)]
199struct DaemonProvenanceFacts {
200 build: DaemonBuildProvenance,
201 pid: Option<u32>,
202 started_at_ms: Option<u64>,
203 start_clock: Option<crate::clock::StartClock>,
204 executable_path: Option<PathBuf>,
205 executable_identity: Option<SpawnedFileIdentity>,
206 process_start_time: Option<u64>,
207 probe: ExecutableIdentityProbe,
208}
209
210impl Default for DaemonProvenanceFacts {
211 fn default() -> Self {
212 Self {
213 build: DaemonBuildProvenance {
214 build_git_sha: None,
215 build_lock_digest: None,
216 },
217 pid: None,
218 started_at_ms: None,
219 start_clock: None,
220 executable_path: None,
221 executable_identity: None,
222 process_start_time: None,
223 probe: ExecutableIdentityProbe::default(),
224 }
225 }
226}
227
228#[derive(Debug, Clone)]
229struct SupervisorRescanContext {
230 supervisor: Supervisor,
231 config_path: PathBuf,
232 configured_port: Option<u16>,
233 storage_config: Option<crate::daemon_config::StorageConfig>,
234 admission_facts_carrier_module_id: Option<String>,
235 admission_facts_targets: Option<Vec<String>>,
236 scope_authority_owners: Vec<String>,
237}
238
239const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
260 "reloading",
261 "supervisor_not_live",
262 "registration_not_active",
263 "no_forwarding_connection",
264 "relay_send_failed",
265];
266
267#[derive(Clone)]
269pub struct ControlHandler {
270 registry: Arc<Registry>,
271 forwarding: Arc<ForwardingTable>,
272 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
273 supervisor: SupervisorHandle,
274 subc_capabilities: Arc<[String]>,
275 route_bind_relay_timeout: Duration,
279 route_bind_relay_timeouts: BTreeMap<String, Duration>,
283 route_bind_breakers: RouteBindBreakers,
286 route_bind_concurrency: RouteBindConcurrency,
289 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
294 route_bind_breaker_threshold: u32,
296 route_bind_breaker_cooldown: Duration,
298 health_probe_timeout: Duration,
299 storage_config: Option<crate::daemon_config::StorageConfig>,
302 machine_id: Option<crate::machine_id::MachineId>,
306 admission_facts_carrier_module_id: Option<String>,
307 admission_facts_targets: Option<Vec<String>>,
308 scopes: Arc<RwLock<ScopeTable>>,
312 scope_authority_owners: Vec<String>,
315 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
318 rescan: Option<SupervisorRescanContext>,
319 connected_clients: ConnectedClients,
320 counters: DaemonCounters,
321 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
322 daemon_provenance: DaemonProvenanceFacts,
323 #[cfg(test)]
324 control_dispatch_delay: Option<Duration>,
325 #[cfg(test)]
326 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
327}
328
329impl fmt::Debug for ControlHandler {
330 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
331 f.debug_struct("ControlHandler")
332 .field("registry", &self.registry)
333 .field("forwarding", &self.forwarding)
334 .field("process_liveness", &self.process_liveness.is_some())
335 .field("supervisor", &self.supervisor)
336 .field("subc_capabilities", &self.subc_capabilities)
337 .finish()
338 }
339}
340
341struct RouteOpenRequest {
342 target: RouteTarget,
343 identity: BindIdentity,
344 consumer_identity: Option<ConsumerIdentity>,
345 consumer_capabilities: Option<Vec<String>>,
346 role_versions: Option<BTreeMap<String, String>>,
347 admission_facts: Option<serde_json::Value>,
348 scope: Option<ScopeSelector>,
349}
350
351struct RouteBindReservationGuard {
352 forwarding: Arc<ForwardingTable>,
353 endpoint: ModuleEndpointId,
354 relay_corr: u64,
355 armed: bool,
356}
357
358struct ModuleControlRpcGuard {
359 forwarding: Arc<ForwardingTable>,
360 endpoint: ModuleEndpointId,
361 corr: u64,
362 armed: bool,
363}
364
365impl ModuleControlRpcGuard {
366 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
367 Self {
368 forwarding,
369 endpoint,
370 corr,
371 armed: true,
372 }
373 }
374
375 fn disarm(&mut self) {
376 self.armed = false;
377 }
378}
379
380impl Drop for ModuleControlRpcGuard {
381 fn drop(&mut self) {
382 if self.armed {
383 let _ = self
384 .forwarding
385 .cancel_module_control_rpc(self.endpoint, self.corr);
386 }
387 }
388}
389
390impl RouteBindReservationGuard {
391 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
392 Self {
393 forwarding,
394 endpoint,
395 relay_corr,
396 armed: true,
397 }
398 }
399
400 fn release_and_disarm(&mut self) {
401 if !self.armed {
402 return;
403 }
404 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
405 self.endpoint,
406 self.relay_corr,
407 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
408 ) {
409 send_goodbye_target_best_effort(
410 &self.forwarding.counters(),
411 &target,
412 "canceled route.bind",
413 );
414 }
415 self.armed = false;
416 }
417
418 fn disarm(&mut self) {
419 self.armed = false;
420 }
421}
422
423impl Drop for RouteBindReservationGuard {
424 fn drop(&mut self) {
425 self.release_and_disarm();
426 }
427}
428
429#[derive(Debug, Clone, Default)]
459pub(crate) struct RouteBindBreakers {
460 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
461}
462
463#[derive(Debug, Clone, Default)]
464pub(crate) struct RouteBindConcurrency {
465 modules: Arc<Mutex<HashMap<String, usize>>>,
466}
467
468struct RouteBindConcurrencyGuard {
469 concurrency: RouteBindConcurrency,
470 module_id: String,
471}
472
473impl RouteBindConcurrency {
474 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
477 let mut modules = self
478 .modules
479 .lock()
480 .expect("route.bind concurrency mutex poisoned");
481 let in_flight = modules.entry(module_id.to_string()).or_default();
482 if *in_flight >= limit {
483 return Err(*in_flight);
484 }
485 *in_flight += 1;
486 Ok(RouteBindConcurrencyGuard {
487 concurrency: self.clone(),
488 module_id: module_id.to_string(),
489 })
490 }
491}
492
493impl Drop for RouteBindConcurrencyGuard {
494 fn drop(&mut self) {
495 let mut modules = self
496 .concurrency
497 .modules
498 .lock()
499 .expect("route.bind concurrency mutex poisoned");
500 let remove = {
501 let in_flight = modules
502 .get_mut(&self.module_id)
503 .expect("admitted route.bind has a concurrency entry");
504 *in_flight -= 1;
505 *in_flight == 0
506 };
507 if remove {
508 modules.remove(&self.module_id);
509 }
510 }
511}
512
513#[derive(Debug, Default)]
514struct ModuleBreakerState {
515 consecutive_timeouts: u32,
517 cooldown_until: Option<Instant>,
520 probe_in_flight: Option<Arc<()>>,
525}
526
527enum RouteBindAdmission<'a> {
529 Admitted {
530 guard: RouteBindBreakerGuard<'a>,
531 probe: bool,
534 },
535 Refused {
536 consecutive_timeouts: u32,
537 retry_in: Duration,
541 probe_in_flight: bool,
542 },
543}
544
545struct RouteBindBreakerGuard<'a> {
552 breakers: RouteBindBreakers,
553 module_id: &'a str,
554 probe_token: Option<Arc<()>>,
555 settled: bool,
556}
557
558impl RouteBindBreakerGuard<'_> {
559 fn record_accepted(&mut self) -> bool {
563 self.settled = true;
564 self.breakers.record_accepted(self.module_id)
565 }
566
567 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
570 self.settled = true;
571 self.breakers.record_timeout(
572 self.module_id,
573 self.probe_token.as_ref(),
574 threshold,
575 cooldown,
576 )
577 }
578
579 fn record_inconclusive(&mut self) {
588 self.settled = true;
589 self.breakers
590 .record_inconclusive(self.module_id, self.probe_token.as_ref());
591 }
592}
593
594impl Drop for RouteBindBreakerGuard<'_> {
595 fn drop(&mut self) {
596 if !self.settled {
597 self.breakers
598 .record_inconclusive(self.module_id, self.probe_token.as_ref());
599 }
600 }
601}
602
603struct BreakerOpened {
607 consecutive_timeouts: u32,
608 reopened_after_probe: bool,
611}
612
613impl RouteBindBreakers {
614 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
615 self.modules
616 .lock()
617 .expect("route.bind breaker mutex poisoned")
618 }
619
620 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
623 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
624 probe: probe_token.is_some(),
625 guard: RouteBindBreakerGuard {
626 breakers: self.clone(),
627 module_id,
628 probe_token,
629 settled: false,
630 },
631 };
632
633 let mut modules = self.lock();
634 let Some(state) = modules.get_mut(module_id) else {
635 return admitted(None);
636 };
637 let Some(cooldown_until) = state.cooldown_until else {
638 return admitted(None);
639 };
640 if state.probe_in_flight.is_some() {
641 return RouteBindAdmission::Refused {
642 consecutive_timeouts: state.consecutive_timeouts,
643 retry_in: Duration::ZERO,
644 probe_in_flight: true,
645 };
646 }
647 let now = Instant::now();
648 if now < cooldown_until {
649 return RouteBindAdmission::Refused {
650 consecutive_timeouts: state.consecutive_timeouts,
651 retry_in: cooldown_until - now,
652 probe_in_flight: false,
653 };
654 }
655 let token = Arc::new(());
656 state.probe_in_flight = Some(Arc::clone(&token));
657 admitted(Some(token))
658 }
659
660 fn record_accepted(&self, module_id: &str) -> bool {
661 self.lock()
662 .remove(module_id)
663 .is_some_and(|state| state.cooldown_until.is_some())
664 }
665
666 fn record_timeout(
667 &self,
668 module_id: &str,
669 probe_token: Option<&Arc<()>>,
670 threshold: u32,
671 cooldown: Duration,
672 ) -> Option<BreakerOpened> {
673 let mut modules = self.lock();
674 let state = modules.entry(module_id.to_string()).or_default();
675 let was_open = state.cooldown_until.is_some();
676 let was_probe = Self::owns_probe(state, probe_token);
677 if was_probe {
678 state.probe_in_flight = None;
679 }
680 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
681 if state.consecutive_timeouts < threshold {
682 return None;
683 }
684 state.cooldown_until = Some(Instant::now() + cooldown);
685 Some(BreakerOpened {
686 consecutive_timeouts: state.consecutive_timeouts,
687 reopened_after_probe: was_open && was_probe,
688 })
689 }
690
691 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
692 state
696 .probe_in_flight
697 .as_ref()
698 .zip(token)
699 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
700 }
701
702 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
703 if let Some(state) = self.lock().get_mut(module_id) {
704 if Self::owns_probe(state, probe_token) {
705 state.probe_in_flight = None;
706 }
707 }
708 }
709
710 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
725 self.lock()
726 .remove(module_id)
727 .map(|state| state.consecutive_timeouts)
728 .filter(|discarded| *discarded > 0)
729 }
730
731 fn open_snapshot(&self) -> Option<serde_json::Value> {
738 let now = Instant::now();
739 let modules = self.lock();
740 let open = modules
741 .iter()
742 .filter_map(|(module_id, state)| {
743 let cooldown_until = state.cooldown_until?;
744 Some((
745 module_id.clone(),
746 serde_json::json!({
747 "consecutive_timeouts": state.consecutive_timeouts,
748 "cooldown_remaining_ms":
749 cooldown_until.saturating_duration_since(now).as_millis() as u64,
750 "probe_in_flight": state.probe_in_flight.is_some(),
751 }),
752 ))
753 })
754 .collect::<serde_json::Map<String, serde_json::Value>>();
755 (!open.is_empty()).then_some(serde_json::Value::Object(open))
756 }
757}
758
759impl ControlHandler {
760 pub fn new(registry: Arc<Registry>) -> Self {
761 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
762 }
763
764 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
765 let counters = forwarding.counters();
766 let route_bind_breakers = forwarding.route_bind_breakers();
770 let route_bind_concurrency = forwarding.route_bind_concurrency();
771 let route_outages = forwarding.route_outages();
772 Self {
773 registry,
774 forwarding,
775 process_liveness: None,
776 supervisor: SupervisorHandle::new(),
777 subc_capabilities: Arc::from([
778 CAP_MANIFEST_REGISTRATION.to_string(),
779 CAP_CHANNEL_LIFECYCLE.to_string(),
780 CAP_PING_PONG.to_string(),
781 CAP_SESSION_ATTACH.to_string(),
782 CAP_ADMISSION_FACTS_RELAY.to_string(),
783 CAP_SCOPES_V1.to_string(),
784 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
785 ]),
786 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
787 route_bind_relay_timeouts: BTreeMap::new(),
788 route_bind_breakers,
789 route_bind_concurrency,
790 route_outages,
791 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
792 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
793 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
794 storage_config: None,
795 machine_id: None,
796 admission_facts_carrier_module_id: None,
797 admission_facts_targets: None,
798 scopes: Arc::new(RwLock::new(ScopeTable::new(
799 crate::daemon_config::default_scope_authority_owners(),
800 ))),
801 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
802 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
803 rescan: None,
804 connected_clients: ConnectedClients::new(),
805 counters,
806 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
807 daemon_provenance: DaemonProvenanceFacts::default(),
808 #[cfg(test)]
809 control_dispatch_delay: None,
810 #[cfg(test)]
811 provenance_probe_override: None,
812 }
813 }
814
815 pub fn with_storage_config(
818 mut self,
819 storage_config: Option<crate::daemon_config::StorageConfig>,
820 ) -> Self {
821 self.storage_config = storage_config;
822 self
823 }
824
825 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
828 self.machine_id = machine_id;
829 self
830 }
831
832 pub fn with_admission_facts_config(
836 mut self,
837 carrier_module_id: Option<String>,
838 targets: Option<Vec<String>>,
839 ) -> Self {
840 self.admission_facts_carrier_module_id = carrier_module_id;
841 self.admission_facts_targets = targets;
842 self
843 }
844
845 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
849 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
850 self.scope_authority_owners = owners;
851 self
852 }
853
854 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
857 self.route_bind_relay_timeout = timeout;
858 self
859 }
860
861 pub fn with_route_bind_relay_timeouts(
867 mut self,
868 timeouts: impl IntoIterator<Item = (String, Duration)>,
869 ) -> Self {
870 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
871 self
872 }
873
874 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
880 self.route_bind_relay_timeouts
881 .get(module_id)
882 .copied()
883 .unwrap_or(self.route_bind_relay_timeout)
884 }
885
886 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
894 self.route_bind_breaker_threshold = threshold.max(1);
895 self.route_bind_breaker_cooldown = cooldown;
896 self
897 }
898
899 #[cfg(test)]
900 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
901 self.health_probe_timeout = timeout;
902 self
903 }
904
905 #[cfg(test)]
906 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
907 self.control_dispatch_delay = Some(delay);
908 self
909 }
910
911 pub fn with_process_liveness(
912 mut self,
913 process_liveness: Arc<dyn ModuleProcessLiveness>,
914 ) -> Self {
915 self.process_liveness = Some(process_liveness);
916 self
917 }
918
919 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
920 self.supervisor = supervisor;
921 self
922 }
923
924 pub fn with_daemon_provenance(
925 mut self,
926 pid: u32,
927 started_at_ms: u64,
928 executable_path: Option<PathBuf>,
929 build_git_sha: Option<String>,
930 build_lock_digest: Option<String>,
931 ) -> Self {
932 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
933 let process_start_time = process_start_time(pid);
934 self.daemon_provenance = DaemonProvenanceFacts {
935 build: DaemonBuildProvenance {
936 build_git_sha,
937 build_lock_digest,
938 },
939 pid: Some(pid),
940 started_at_ms: Some(started_at_ms),
941 start_clock: None,
942 executable_path,
943 executable_identity,
944 process_start_time,
945 probe: ExecutableIdentityProbe::default(),
946 };
947 self
948 }
949
950 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
951 self.daemon_provenance.start_clock = Some(clock);
952 self
953 }
954
955 #[cfg(test)]
956 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
957 self.provenance_probe_override = Some(result);
958 self
959 }
960
961 pub fn with_capability_config(
965 self,
966 modules: impl IntoIterator<Item = (String, bool)>,
967 reserved_capabilities: BTreeMap<String, String>,
968 ) -> Self {
969 self.capability_evaluator
970 .configure(modules, reserved_capabilities);
971 self
972 }
973
974 pub fn with_supervisor_rescan(
975 mut self,
976 supervisor: Supervisor,
977 config_path: impl Into<PathBuf>,
978 configured_port: Option<u16>,
979 ) -> Self {
980 self.rescan = Some(SupervisorRescanContext {
981 supervisor,
982 config_path: config_path.into(),
983 configured_port,
984 storage_config: self.storage_config.clone(),
985 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
986 admission_facts_targets: self.admission_facts_targets.clone(),
987 scope_authority_owners: self.scope_authority_owners.clone(),
988 });
989 self
990 }
991
992 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
993 self.connected_clients = connected_clients;
994 self
995 }
996
997 pub fn forwarding(&self) -> Arc<ForwardingTable> {
998 Arc::clone(&self.forwarding)
999 }
1000
1001 pub(crate) fn counters(&self) -> DaemonCounters {
1002 self.counters.clone()
1003 }
1004
1005 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1008 tokio::spawn(async move {
1009 loop {
1010 self.capability_evaluator
1011 .wait_for_change_or_deadline()
1012 .await;
1013 self.refresh_capability_requirements();
1014 }
1015 });
1016 }
1017
1018 fn runtime_capability_snapshot(
1019 &self,
1020 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1021 let runtime = self
1022 .supervisor
1023 .list()
1024 .into_iter()
1025 .map(|module| {
1026 let status = module.status().map_err(|err| {
1027 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1028 })?;
1029 Ok(RuntimeModule {
1030 module_id: status.module_id,
1031 state: status.state,
1032 enabled: status.enabled,
1033 })
1034 })
1035 .collect::<Result<Vec<_>, RouterError>>()?;
1036 let (_, registrations) = self.registry.list_modules().map_err(|err| {
1037 RouterError::backend(
1038 0,
1039 0,
1040 format!("failed to list capability registrations: {err}"),
1041 )
1042 })?;
1043 let registrations = registrations
1044 .into_iter()
1045 .map(|registration| RegisteredModule {
1046 module_id: registration.manifest.module_id,
1047 module_version: registration.manifest.module_version,
1048 capabilities: registration.manifest.capabilities,
1049 })
1050 .collect();
1051 Ok((runtime, registrations))
1052 }
1053
1054 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1061 let cached_registration = RegisteredModule {
1062 module_id: registration.manifest.module_id.clone(),
1063 module_version: registration.manifest.module_version.clone(),
1064 capabilities: registration.manifest.capabilities.clone(),
1065 };
1066 if self.capability_evaluator.record_hello(&cached_registration) {
1067 warn!(
1068 module_id = %cached_registration.module_id,
1069 "capability claims drifted from the cached manifest"
1070 );
1071 }
1072 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1073 self.enforce_capability_denies();
1074 }
1075 self.refresh_capability_requirements();
1076 }
1077
1078 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1082 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1083 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1084 self.supervisor.set_swap_promotion_observer(observer);
1085 }
1086
1087 pub fn refresh_capability_requirements(&self) {
1088 match self.runtime_capability_snapshot() {
1089 Ok((runtime, registrations)) => {
1090 log_requirement_events(
1091 self.capability_evaluator
1092 .evaluate_now(&runtime, ®istrations),
1093 );
1094 }
1095 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1096 }
1097 }
1098
1099 fn enforce_capability_denies(&self) {
1103 let (_, registrations) = match self.registry.list_modules() {
1104 Ok(snapshot) => snapshot,
1105 Err(err) => {
1106 warn!(error = %err, "failed to read registrations for capability deny census");
1107 return;
1108 }
1109 };
1110 let manifests = registrations
1111 .into_iter()
1112 .map(|registration| {
1113 (
1114 registration.manifest.module_id.clone(),
1115 registration.manifest,
1116 )
1117 })
1118 .collect::<BTreeMap<_, _>>();
1119 let census = match self.forwarding.route_census(None) {
1120 Ok(census) => census,
1121 Err(err) => {
1122 warn!(error = %err, "failed to read route census for capability deny enforcement");
1123 return;
1124 }
1125 };
1126
1127 for (target_module_id, routes) in census {
1128 let Some(target_manifest) = manifests.get(&target_module_id) else {
1129 continue;
1130 };
1131 let mut closed_routes = Vec::new();
1132 let mut module_goodbyes = Vec::new();
1133 for route in routes {
1134 let Principal::Reserved {
1135 module_id: opening_module_id,
1136 } = &route.principal
1137 else {
1138 continue;
1139 };
1140 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1141 continue;
1142 };
1143 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1144 continue;
1145 };
1146
1147 match self.forwarding.release_client_route(
1148 route.goodbye_target.connection_id,
1149 route.goodbye_target.channel,
1150 route.goodbye_target.epoch,
1151 ) {
1152 Ok(RouteRelease::Removed(module_goodbye)) => {
1153 warn!(
1154 opening_module_id,
1155 target_module_id,
1156 capability,
1157 "force-closing route because an attested capability deny edge now matches"
1158 );
1159 closed_routes.push(route);
1160 module_goodbyes.push(module_goodbye);
1161 }
1162 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1163 Err(err) => warn!(
1164 opening_module_id,
1165 target_module_id,
1166 capability,
1167 error = %err,
1168 "failed to force-close capability-denied route"
1169 ),
1170 }
1171 }
1172
1173 if closed_routes.is_empty() {
1174 continue;
1175 }
1176 send_route_control_pushes(
1177 &self.forwarding,
1178 closed_routes,
1179 ClientControlPush::RouteClosed {
1180 module_id: target_module_id,
1181 channels: Vec::new(),
1182 reason: RouteCloseReason::CapabilityDenied,
1183 drained: false,
1184 abandoned: 0,
1185 excluded_subscriptions: 0,
1186 terminal: Some(false),
1187 },
1188 );
1189 self.emit_route_goodbyes(module_goodbyes);
1190 }
1191 }
1192
1193 fn not_ready_reason(
1199 &self,
1200 registration: &crate::registry::ModuleRegistration,
1201 ) -> Option<NotReadyReason> {
1202 if !registration.ready {
1203 return Some(NotReadyReason {
1204 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1205 capability: None,
1206 });
1207 }
1208 self.first_unprovided_required_capability(registration)
1209 .map(|capability| NotReadyReason {
1210 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1211 capability: Some(capability),
1212 })
1213 }
1214
1215 fn first_unprovided_required_capability(
1233 &self,
1234 registration: &crate::registry::ModuleRegistration,
1235 ) -> Option<String> {
1236 let required = registration
1237 .manifest
1238 .capabilities
1239 .iter()
1240 .flat_map(|declarations| declarations.requires.iter())
1241 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1242 .map(|requirement| requirement.capability.as_str())
1243 .collect::<BTreeSet<_>>();
1244 if required.is_empty() {
1245 return None;
1246 }
1247 let module_id = registration.manifest.module_id.as_str();
1248 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1249 if required
1250 .iter()
1251 .any(|capability| verdict(capability).is_none())
1252 {
1253 self.refresh_capability_requirements();
1254 }
1255 required
1256 .into_iter()
1257 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1258 .map(str::to_string)
1259 }
1260
1261 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1262 self.capability_evaluator
1263 .statuses()
1264 .into_iter()
1265 .map(capability_requirement_status)
1266 .collect()
1267 }
1268
1269 fn deregister_connection(
1276 &self,
1277 connection_id: ConnectionId,
1278 reason: RegistrationEndReason,
1279 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1280 self.registry
1281 .deregister_connection_with_reason(connection_id, reason)
1282 }
1283
1284 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1285 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1286 return None;
1287 }
1288 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1289 parse_client_control_request(&frame.body)
1290 else {
1291 return None;
1292 };
1293 Some(target_module_id(&target).to_string())
1294 }
1295
1296 pub(crate) fn route_open_capacity_refusal(
1297 &self,
1298 ctx: &RouteCtx,
1299 frame: &Frame,
1300 target_module_id: &str,
1301 in_flight: usize,
1302 limit: usize,
1303 ) -> Result<Frame, RouterError> {
1304 self.route_open_admission_refusal_frame(
1305 ctx,
1306 frame,
1307 target_module_id,
1308 "open_admission_full",
1309 (in_flight, limit),
1310 format!(
1311 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1312 ),
1313 )
1314 }
1315
1316 fn route_open_target_capacity_refusal(
1317 &self,
1318 ctx: &RouteCtx,
1319 frame: &Frame,
1320 target_module_id: &str,
1321 in_flight: usize,
1322 ) -> Result<Frame, RouterError> {
1323 self.route_open_admission_refusal_frame(
1324 ctx,
1325 frame,
1326 target_module_id,
1327 "target_binds_full",
1328 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1329 format!(
1330 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1331 ),
1332 )
1333 }
1334
1335 fn route_open_admission_refusal_frame(
1343 &self,
1344 ctx: &RouteCtx,
1345 frame: &Frame,
1346 target_module_id: &str,
1347 reason: &'static str,
1348 (in_flight, limit): (usize, usize),
1349 message: impl Into<String>,
1350 ) -> Result<Frame, RouterError> {
1351 let code = error_codes::TARGET_UNAVAILABLE;
1352 self.counters.increment_route_open_refused(code);
1353 info!(
1354 target: "control",
1355 code,
1356 reason,
1357 module_id = ?target_module_id,
1358 connection_id = ctx.connection_id.get(),
1359 in_flight,
1360 limit,
1361 "route.open refused"
1362 );
1363 control_error_frame(frame, code, message.into())
1364 }
1365
1366 #[cfg(test)]
1372 pub fn handle_control(
1373 &self,
1374 connection_id: ConnectionId,
1375 frame: Frame,
1376 ) -> Result<Vec<Frame>, RouterError> {
1377 match frame.header.ty {
1378 FrameType::Ping => Ok(vec![pong(&frame)?]),
1379 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1380 FrameType::Goodbye => self.handle_goodbye(connection_id),
1381 ty => Ok(vec![control_error_frame(
1382 &frame,
1383 "unsupported_control_frame",
1384 format!("unsupported channel-0 frame {ty:?}"),
1385 )?]),
1386 }
1387 }
1388
1389 pub async fn handle_control_frame(
1390 &self,
1391 ctx: &RouteCtx,
1392 frame: Frame,
1393 ) -> Result<Vec<Frame>, RouterError> {
1394 self.handle_control_frame_timed(ctx, frame, None).await
1395 }
1396
1397 pub(crate) async fn handle_control_frame_timed(
1398 &self,
1399 ctx: &RouteCtx,
1400 frame: Frame,
1401 dispatch_started_at: Option<StdInstant>,
1402 ) -> Result<Vec<Frame>, RouterError> {
1403 match frame.header.ty {
1404 FrameType::Ping => Ok(vec![pong(&frame)?]),
1405 FrameType::Hello => {
1406 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1407 }
1408 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1409 FrameType::Cancel => {
1410 if self
1413 .forwarding
1414 .operator_confirms()
1415 .cancel(ctx.connection_id, frame.header.corr)
1416 || self
1417 .supervisor
1418 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1419 {
1420 Ok(Vec::new())
1421 } else {
1422 Ok(vec![control_error_frame(
1423 &frame,
1424 "unknown_subscription",
1425 "no supervisor spawn subscription has this correlation id",
1426 )?])
1427 }
1428 }
1429 FrameType::Request => {
1430 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1433 if op
1434 .as_ref()
1435 .is_some_and(|probe| probe.op == "operator.confirm")
1436 {
1437 return self.handle_operator_confirm(ctx, frame);
1438 }
1439 if self
1440 .forwarding
1441 .module_endpoint_for_connection(ctx.connection_id)
1442 .map_err(RouterError::Forwarding)?
1443 .is_some()
1444 {
1445 if !is_known_module_request_op(&frame.body) {
1446 return Ok(vec![control_error_frame(
1447 &frame,
1448 "unsupported_control_frame",
1449 "module-originated channel-0 REQUEST is not supported",
1450 )?]);
1451 }
1452 let request = match parse_module_control_request_from_module(&frame.body) {
1453 Ok(request) => request,
1454 Err((err, ControlRequestBodyError::UnknownOp)) => {
1455 return Ok(vec![control_error_frame(
1456 &frame,
1457 "unsupported_control_frame",
1458 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1459 )?])
1460 }
1461 Err((err, ControlRequestBodyError::InvalidBody)) => {
1462 return Ok(vec![control_error_frame(
1463 &frame,
1464 "invalid_control_body",
1465 format!("malformed module control body: {err}"),
1466 )?])
1467 }
1468 };
1469 let op = module_control_request_op(&request);
1470 let corr = frame.header.corr;
1471 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1472 let result =
1473 self.handle_module_control_request(ctx.connection_id, frame, request);
1474 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1475 return result;
1476 }
1477
1478 if is_known_module_request_op(&frame.body) {
1479 return Ok(vec![control_error_frame(
1480 &frame,
1481 "not_registered",
1482 "catalog.update requires an active module registration owned by this connection",
1483 )?]);
1484 }
1485
1486 let request = match parse_client_control_request(&frame.body) {
1487 Ok(request) => request,
1488 Err((err, ControlRequestBodyError::UnknownOp)) => {
1489 return Ok(vec![control_error_frame(
1490 &frame,
1491 "unknown_control_op",
1492 format!("unknown client control op: {err}"),
1493 )?])
1494 }
1495 Err((err, ControlRequestBodyError::InvalidBody)) => {
1496 return Ok(vec![control_error_frame(
1497 &frame,
1498 "invalid_control_body",
1499 format!("malformed client control body: {err}"),
1500 )?])
1501 }
1502 };
1503 let op = client_control_request_op(&request);
1504 let corr = frame.header.corr;
1505 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1506 #[cfg(test)]
1507 if let Some(delay) = self.control_dispatch_delay {
1508 tokio::time::sleep(delay).await;
1509 }
1510 let result = self
1511 .handle_client_control_request(ctx, frame, request)
1512 .await;
1513 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1514 result
1515 }
1516 FrameType::Push => {
1517 let Some(endpoint) = self
1518 .forwarding
1519 .module_endpoint_for_connection(ctx.connection_id)
1520 .map_err(RouterError::Forwarding)?
1521 else {
1522 return Ok(vec![control_error_frame(
1523 &frame,
1524 "unsupported_control_frame",
1525 "client-originated channel-0 PUSH is not supported",
1526 )?]);
1527 };
1528 self.handle_status_update(endpoint, frame)
1529 }
1530 FrameType::Response | FrameType::Error
1531 if self
1532 .forwarding
1533 .module_endpoint_for_connection(ctx.connection_id)
1534 .map_err(RouterError::Forwarding)?
1535 .is_some() =>
1536 {
1537 self.handle_module_relay_response(ctx.connection_id, frame)
1538 }
1539 ty => Ok(vec![control_error_frame(
1540 &frame,
1541 "unsupported_control_frame",
1542 format!("unsupported channel-0 frame {ty:?}"),
1543 )?]),
1544 }
1545 }
1546
1547 pub fn cleanup_connection(
1548 &self,
1549 connection_id: ConnectionId,
1550 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1551 self.cleanup_connection_with_end_reason(
1552 connection_id,
1553 RegistrationEndReason::ConnectionClosed,
1554 )
1555 }
1556
1557 fn cleanup_connection_with_end_reason(
1558 &self,
1559 connection_id: ConnectionId,
1560 requested_reason: RegistrationEndReason,
1561 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1562 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
1563 self.registry
1564 .get_module_by_connection(connection_id)?
1565 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
1566 .and_then(|module| module.registration_end_reason().ok().flatten())
1567 .unwrap_or(requested_reason)
1568 } else {
1569 requested_reason
1570 };
1571 let crash_closed = self
1572 .registry
1573 .get_module_by_connection(connection_id)?
1574 .and_then(|registration| {
1575 self.forwarding
1576 .module_endpoint_for_connection(connection_id)
1577 .ok()
1578 .flatten()
1579 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1580 .map(|routes| (registration.manifest.module_id, routes))
1581 });
1582 let crash_closed = crash_closed.map(|(module_id, routes)| {
1583 let terminal = match self.supervisor.get(&module_id) {
1584 None => false,
1585 Some(module) => match module.will_recover_after_connection_loss() {
1586 Ok(will_recover) => !will_recover,
1587 Err(err) => {
1588 warn!(
1589 %module_id,
1590 error = %err,
1591 "failed to read crash recovery verdict; reporting non-terminal conservatively"
1592 );
1593 false
1594 }
1595 },
1596 };
1597 let reason = match self.forwarding.is_daemon_draining() {
1601 Ok(true) => RouteCloseReason::Restart,
1602 Ok(false) => RouteCloseReason::Crash,
1603 Err(err) => {
1604 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
1605 RouteCloseReason::Crash
1606 }
1607 };
1608 (module_id, routes, reason, terminal)
1609 });
1610 let registrations = self.deregister_connection(connection_id, end_reason);
1611 let cleanup = if crash_closed.is_some() {
1612 self.forwarding.cleanup_connection_counted(connection_id)
1613 } else {
1614 self.forwarding
1620 .cleanup_connection(connection_id)
1621 .map(|released| crate::forwarding::ConnectionCleanup {
1622 released,
1623 abandoned_relays: 0,
1624 })
1625 };
1626 if let Some((module_id, routes, reason, terminal)) = crash_closed {
1631 let abandoned = cleanup
1632 .as_ref()
1633 .map_or(0, |cleanup| cleanup.abandoned_relays);
1634 send_route_control_pushes(
1635 &self.forwarding,
1636 routes,
1637 ClientControlPush::RouteClosed {
1638 module_id,
1639 channels: Vec::new(),
1640 reason,
1641 drained: false,
1642 abandoned,
1643 excluded_subscriptions: 0,
1644 terminal: Some(terminal),
1645 },
1646 );
1647 }
1648 if let Ok(cleanup) = cleanup {
1649 self.emit_route_goodbyes(cleanup.released);
1650 }
1651 if matches!(®istrations, Ok(r) if !r.is_empty()) {
1655 crate::supervise::notify_registration_release();
1656 self.capability_evaluator.wake_deadline_loop();
1657 self.refresh_capability_requirements();
1658 }
1659 self.supervisor.remove_spawn_subscribers(connection_id);
1660 self.hello_launch_nonces
1663 .lock()
1664 .unwrap_or_else(|poisoned| poisoned.into_inner())
1665 .forget(connection_id);
1666 self.scopes
1667 .write()
1668 .unwrap_or_else(|poisoned| poisoned.into_inner())
1669 .release_connection(connection_id);
1670 registrations
1671 }
1672
1673 pub(crate) fn handle_route_goodbye(
1674 &self,
1675 connection_id: ConnectionId,
1676 route_channel: u16,
1677 route_epoch: u32,
1678 ) -> Result<bool, RouterError> {
1679 debug!(
1680 connection_id = connection_id.get(),
1681 route_channel, route_epoch, "handling route GOODBYE"
1682 );
1683 let RouteRelease::Removed(released_route) = self
1684 .forwarding
1685 .release_client_route(connection_id, route_channel, route_epoch)
1686 .map_err(RouterError::Forwarding)?
1687 else {
1688 return Ok(false);
1689 };
1690 self.emit_route_goodbyes(vec![released_route]);
1691 Ok(true)
1692 }
1693
1694 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
1695 for released in released_routes {
1696 let frame = match Frame::build_with_version(
1697 released.negotiated_ver,
1698 FrameType::Goodbye,
1699 control_flags(),
1700 released.channel,
1701 released.epoch,
1702 0,
1703 Vec::new(),
1704 ) {
1705 Ok(frame) => frame,
1706 Err(err) => {
1707 warn!(
1708 route_channel = released.channel,
1709 error = %err,
1710 "failed to build route GOODBYE frame"
1711 );
1712 continue;
1713 }
1714 };
1715 if !released.close_on_delivery_failure() {
1716 crate::forwarding::send_module_route_goodbye(
1717 &self.counters,
1718 &released.sink,
1719 frame,
1720 released.module_id.as_deref(),
1721 "client route released",
1722 );
1723 continue;
1724 }
1725 if let Err(err) = released.sink.try_send(frame) {
1726 warn!(
1727 target_connection_id = released.connection_id.get(),
1728 route_channel = released.channel,
1729 error = %err,
1730 "route GOODBYE was not delivered to client; closing target connection"
1731 );
1732 if self
1733 .forwarding
1734 .escalate_client_delivery_failure(
1735 released.connection_id,
1736 released.channel,
1737 released.epoch,
1738 CloseReason::new(
1739 "route_goodbye_delivery_failed",
1740 format!(
1741 "failed to enqueue route GOODBYE for channel {}: {err}",
1742 released.channel
1743 ),
1744 ),
1745 crate::forwarding::UndeliveredFrame {
1746 module_id: released.module_id.as_deref(),
1747 sink: &released.sink,
1748 },
1749 )
1750 .unwrap_or(false)
1751 {
1752 self.counters.increment_goodbye_relay_client_failed();
1753 }
1754 }
1755 }
1756 }
1757
1758 fn send_abandoned_route_bind_goodbye(
1770 &self,
1771 module_sink: &crate::FrameSink,
1772 negotiated_ver: u8,
1773 module_channel: u16,
1774 module_epoch: u32,
1775 ) {
1776 let frame = match Frame::build_with_version(
1777 negotiated_ver,
1778 FrameType::Goodbye,
1779 control_flags(),
1780 module_channel,
1781 module_epoch,
1782 0,
1783 Vec::new(),
1784 ) {
1785 Ok(frame) => frame,
1786 Err(err) => {
1787 warn!(
1788 route_channel = module_channel,
1789 error = %err,
1790 "failed to build GOODBYE for abandoned route.bind"
1791 );
1792 return;
1793 }
1794 };
1795 crate::forwarding::send_module_route_goodbye(
1796 &self.counters,
1797 module_sink,
1798 frame,
1799 None,
1800 "abandoned route.bind",
1801 );
1802 }
1803
1804 fn handle_hello(
1805 &self,
1806 connection_id: ConnectionId,
1807 sink: Option<crate::FrameSink>,
1808 frame: Frame,
1809 ) -> Result<Vec<Frame>, RouterError> {
1810 debug!(
1811 connection_id = connection_id.get(),
1812 corr = frame.header.corr,
1813 "handling HELLO"
1814 );
1815 if self
1819 .registry
1820 .get_module_by_connection(connection_id)
1821 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
1822 .is_some()
1823 {
1824 return Ok(vec![control_error_frame(
1825 &frame,
1826 "invalid_hello",
1827 "connection is already registered as a module",
1828 )?]);
1829 }
1830 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
1831 Ok(value) => value,
1832 Err(err) => {
1833 return Ok(vec![control_error_frame(
1834 &frame,
1835 "invalid_hello",
1836 format!("malformed HELLO body: {err}"),
1837 )?])
1838 }
1839 };
1840 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
1841 return Ok(vec![control_error_frame(
1842 &frame,
1843 "invalid_capability_grammar",
1844 err.to_string(),
1845 )?]);
1846 }
1847 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
1848 return Ok(vec![control_error_frame(
1849 &frame,
1850 "invalid_manifest",
1851 err.to_string(),
1852 )?]);
1853 }
1854 if let Err(err) = validate_hello_event_declarations(&hello_value) {
1855 return Ok(vec![control_error_frame(
1856 &frame,
1857 "invalid_event_declaration",
1858 err.to_string(),
1859 )?]);
1860 }
1861 if let Some(provenance) = hello_value
1862 .get("manifest")
1863 .and_then(|manifest| manifest.get("provenance"))
1864 {
1865 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
1866 return Ok(vec![control_error_frame(
1867 &frame,
1868 "invalid_manifest",
1869 format!("malformed manifest provenance: {err}"),
1870 )?]);
1871 }
1872 }
1873 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
1874 Ok(hello) => hello,
1875 Err(err) => {
1876 return Ok(vec![control_error_frame(
1877 &frame,
1878 "invalid_hello",
1879 format!("malformed HELLO body: {err}"),
1880 )?])
1881 }
1882 };
1883
1884 if hello.protocol_ver != hello.manifest.protocol_ver {
1885 return Ok(vec![control_error_frame(
1886 &frame,
1887 "invalid_manifest",
1888 format!(
1889 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
1890 hello.protocol_ver, hello.manifest.protocol_ver
1891 ),
1892 )?]);
1893 }
1894
1895 if hello.manifest.module_id.trim().is_empty() {
1896 return Ok(vec![control_error_frame(
1897 &frame,
1898 "invalid_manifest",
1899 "manifest module_id must not be empty",
1900 )?]);
1901 }
1902
1903 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
1904 Ok(negotiated_ver) => negotiated_ver,
1905 Err(message) => {
1906 return Ok(vec![control_error_frame(
1907 &frame,
1908 "version_unsupported",
1909 message,
1910 )?])
1911 }
1912 };
1913
1914 let swap_admission = self
1922 .supervisor
1923 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
1924 if swap_admission == SwapHelloAdmission::Refused {
1925 warn!(
1926 module_id = %hello.manifest.module_id,
1927 connection_id = connection_id.get(),
1928 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
1929 );
1930 return Ok(vec![control_error_frame(
1931 &frame,
1932 "swap_token_invalid",
1933 format!(
1934 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
1935 hello.manifest.module_id
1936 ),
1937 )?]);
1938 }
1939 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
1940
1941 if let Some(rejection) = (!swap_candidate)
1949 .then(|| {
1950 self.supervisor.reserved_hello_rejection(
1951 &hello.manifest.module_id,
1952 hello.launch_nonce.as_deref(),
1953 )
1954 })
1955 .flatten()
1956 {
1957 let message = match rejection {
1958 ReservedHelloRejection::Exact { module_id } => format!(
1959 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
1960 ),
1961 ReservedHelloRejection::Prefix {
1962 prefix,
1963 owner_module_id,
1964 } => format!(
1965 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
1966 hello.manifest.module_id
1967 ),
1968 };
1969 return Ok(vec![control_error_frame(
1970 &frame,
1971 "reserved_module",
1972 message,
1973 )?]);
1974 }
1975
1976 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
1977 &hello.manifest.module_id,
1978 hello.manifest.capabilities.as_ref(),
1979 );
1980 if let Some(refusal) = reserved_capability_refusals.first() {
1981 let capability = refusal.capability.clone();
1982 let bound_module = refusal.claimants[0].clone();
1983 log_duplicate_claim_events(reserved_capability_refusals);
1984 return Ok(vec![control_error_frame(
1985 &frame,
1986 "reserved_capability",
1987 format!(
1988 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
1989 capability, bound_module, hello.manifest.module_id
1990 ),
1991 )?]);
1992 }
1993
1994 if self
1997 .forwarding
1998 .connection_has_client_routes(connection_id)
1999 .map_err(RouterError::Forwarding)?
2000 {
2001 return Ok(vec![control_error_frame(
2002 &frame,
2003 "invalid_hello",
2004 "connection has open client routes and cannot also register as a module",
2005 )?]);
2006 }
2007
2008 self.hello_launch_nonces
2013 .lock()
2014 .unwrap_or_else(|poisoned| poisoned.into_inner())
2015 .record(connection_id, hello.launch_nonce.as_deref());
2016 let control_ops = effective_module_control_ops(hello.control_ops);
2017 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2020 if swap_candidate {
2021 return self.register_swap_candidate(
2022 connection_id,
2023 sink,
2024 &frame,
2025 hello.manifest,
2026 negotiated_ver,
2027 control_ops,
2028 hello_ack,
2029 );
2030 }
2031 let registration = match self.registry.register_with_control_ops(
2032 hello.manifest,
2033 negotiated_ver,
2034 connection_id,
2035 control_ops,
2036 ) {
2037 Ok(registration) => registration,
2038 Err(RegistryError::DuplicateModuleId { module_id }) => {
2039 return Ok(vec![control_error_frame(
2040 &frame,
2041 "duplicate_module_id",
2042 format!(
2043 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2044 ),
2045 )?])
2046 }
2047 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2048 return Ok(vec![control_error_frame(
2049 &frame,
2050 "invalid_module_id",
2051 err.to_string(),
2052 )?])
2053 }
2054 Err(err) => {
2055 return Ok(vec![control_error_frame(
2056 &frame,
2057 "registry_error",
2058 err.to_string(),
2059 )?])
2060 }
2061 };
2062
2063 let reply = if let Some(sink) = sink {
2064 let concurrency = manifest_concurrency(®istration.manifest);
2080 if let Err(err) = self.forwarding.register_module_connection_acked(
2081 connection_id,
2082 registration.manifest.module_id.clone(),
2083 negotiated_ver,
2084 concurrency,
2085 sink,
2086 hello_ack,
2087 ) {
2088 if matches!(
2092 self.deregister_connection(
2093 connection_id,
2094 RegistrationEndReason::RegistrationFailed,
2095 ),
2096 Ok(r) if !r.is_empty()
2097 ) {
2098 crate::supervise::notify_registration_release();
2099 }
2100 return Ok(vec![control_error_frame(
2101 &frame,
2102 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2103 "invalid_hello"
2104 } else {
2105 forwarding_error_code(&err)
2106 },
2107 err.to_string(),
2108 )?]);
2109 }
2110 Vec::new()
2111 } else {
2112 vec![hello_ack]
2115 };
2116
2117 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2126 info!(
2127 module_id = %registration.manifest.module_id,
2128 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2129 );
2130 }
2131
2132 self.apply_registration_capabilities(®istration);
2133
2134 info!(
2135 module_id = %registration.manifest.module_id,
2136 module_version = %registration.manifest.module_version,
2137 negotiated_ver,
2138 routable_provider = manifest_provides_routable_role(®istration.manifest),
2139 connection_id = connection_id.get(),
2140 "module registered"
2141 );
2142
2143 Ok(reply)
2144 }
2145
2146 #[allow(clippy::too_many_arguments)]
2156 fn register_swap_candidate(
2157 &self,
2158 connection_id: ConnectionId,
2159 sink: Option<crate::FrameSink>,
2160 frame: &Frame,
2161 manifest: ModuleManifest,
2162 negotiated_ver: u8,
2163 control_ops: Vec<String>,
2164 hello_ack: Frame,
2165 ) -> Result<Vec<Frame>, RouterError> {
2166 let module_id = manifest.module_id.clone();
2167 let registration = match self.registry.register_candidate_with_control_ops(
2168 manifest,
2169 negotiated_ver,
2170 connection_id,
2171 control_ops,
2172 ) {
2173 Ok(registration) => registration,
2174 Err(RegistryError::DuplicateModuleId { module_id }) => {
2175 return Ok(vec![control_error_frame(
2176 frame,
2177 "duplicate_module_id",
2178 format!(
2179 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2180 ),
2181 )?])
2182 }
2183 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2184 return Ok(vec![control_error_frame(
2185 frame,
2186 "invalid_module_id",
2187 err.to_string(),
2188 )?])
2189 }
2190 Err(err) => {
2191 return Ok(vec![control_error_frame(
2192 frame,
2193 "registry_error",
2194 err.to_string(),
2195 )?])
2196 }
2197 };
2198 let reply = if let Some(sink) = sink {
2199 let concurrency = manifest_concurrency(®istration.manifest);
2203 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2204 connection_id,
2205 module_id.clone(),
2206 negotiated_ver,
2207 concurrency,
2208 sink,
2209 hello_ack,
2210 ) {
2211 if matches!(
2212 self.deregister_connection(
2213 connection_id,
2214 RegistrationEndReason::RegistrationFailed,
2215 ),
2216 Ok(r) if !r.is_empty()
2217 ) {
2218 crate::supervise::notify_registration_release();
2219 }
2220 return Ok(vec![control_error_frame(
2221 frame,
2222 forwarding_error_code(&err),
2223 err.to_string(),
2224 )?]);
2225 }
2226 Vec::new()
2227 } else {
2228 vec![hello_ack]
2229 };
2230 self.supervisor.mark_swap_candidate_admitted(&module_id);
2231 info!(
2232 module_id = %module_id,
2233 module_version = %registration.manifest.module_version,
2234 negotiated_ver,
2235 ready = registration.ready,
2236 connection_id = connection_id.get(),
2237 "swap candidate registered; not routable until cutover"
2238 );
2239 Ok(reply)
2240 }
2241
2242 fn build_hello_ack(
2243 &self,
2244 frame: &Frame,
2245 negotiated_ver: u8,
2246 module_id: &str,
2247 ) -> Result<Frame, RouterError> {
2248 let ack = ModuleHelloAckBody {
2249 negotiated_ver,
2250 subc_ops: module_subc_ops(),
2251 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2252 storage: self
2253 .storage_config
2254 .as_ref()
2255 .map(|cfg| cfg.descriptor_for(module_id)),
2256 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2257 };
2258 let body = serde_json::to_vec(&ack).map_err(|err| {
2259 RouterError::backend(
2260 0,
2261 frame.header.corr,
2262 format!("failed to encode HELLO_ACK: {err}"),
2263 )
2264 })?;
2265
2266 Frame::build_with_version(
2267 negotiated_ver,
2268 FrameType::HelloAck,
2269 control_flags(),
2270 0,
2271 0,
2272 frame.header.corr,
2273 body,
2274 )
2275 .map_err(RouterError::FrameBuild)
2276 }
2277
2278 async fn handle_client_control_request(
2279 &self,
2280 ctx: &RouteCtx,
2281 frame: Frame,
2282 request: ClientControlRequest,
2283 ) -> Result<Vec<Frame>, RouterError> {
2284 match request {
2285 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2286 ClientControlRequest::CatalogList { module_id } => {
2287 self.handle_catalog_list(frame, module_id)
2288 }
2289 ClientControlRequest::RouteOpen {
2290 target,
2291 identity,
2292 consumer_identity,
2293 consumer_capabilities,
2294 role_versions,
2295 admission_facts,
2296 scope,
2297 } => {
2298 self.handle_route_open(
2299 ctx,
2300 frame,
2301 RouteOpenRequest {
2302 target,
2303 identity,
2304 consumer_identity,
2305 consumer_capabilities,
2306 role_versions,
2307 admission_facts,
2308 scope,
2309 },
2310 )
2311 .await
2312 }
2313 ClientControlRequest::RoutePoll {
2314 route_channel,
2315 route_epoch,
2316 kind,
2317 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2318 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2319 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2320 self.handle_supervisor_spawn_snapshot(frame)
2321 }
2322 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2323 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2324 }
2325 ClientControlRequest::SupervisorRestart {
2326 module_id,
2327 drain_timeout_ms,
2328 } => {
2329 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms)
2330 .await
2331 }
2332 ClientControlRequest::SupervisorSwap {
2333 module_id,
2334 ready_timeout_ms,
2335 } => {
2336 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms)
2337 .await
2338 }
2339 ClientControlRequest::SupervisorReload { module_id } => {
2340 self.handle_supervisor_reload(frame, module_id).await
2341 }
2342 ClientControlRequest::SupervisorRescan { preview } => {
2343 self.handle_supervisor_rescan(frame, preview).await
2344 }
2345 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2346 self.handle_supervisor_release_reserved(frame, module_id)
2347 .await
2348 }
2349 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2350 self.handle_supervisor_set_enabled(frame, module_id, enabled)
2351 .await
2352 }
2353 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2354 self.handle_supervisor_health_probe(frame, module_id).await
2355 }
2356 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2357 ClientControlRequest::SupervisorRoutes { module_id } => {
2358 self.handle_supervisor_routes(frame, module_id)
2359 }
2360 ClientControlRequest::SupervisorProvenance { module_id } => {
2361 self.handle_supervisor_provenance(frame, module_id).await
2362 }
2363 ClientControlRequest::SupervisorStderrTail {
2364 module_id,
2365 max_lines,
2366 max_bytes,
2367 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2368 ClientControlRequest::SupervisorTerminals { module_id } => {
2369 self.handle_supervisor_terminals(frame, module_id).await
2370 }
2371 }
2372 }
2373
2374 fn handle_module_control_request(
2375 &self,
2376 connection_id: ConnectionId,
2377 frame: Frame,
2378 request: ModuleControlRequestFromModule,
2379 ) -> Result<Vec<Frame>, RouterError> {
2380 match request {
2381 ModuleControlRequestFromModule::CatalogUpdate {
2382 provides,
2383 capabilities,
2384 ready,
2385 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2386 ModuleControlRequestFromModule::LiveRoots {} => {
2387 let registered = self
2388 .registry
2389 .get_module_by_connection(connection_id)
2390 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2391 let Some(registration) = registered else {
2392 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2393 };
2394 let response = self
2395 .forwarding
2396 .live_roots(®istration.manifest.module_id)
2397 .map_err(RouterError::Forwarding)?;
2398 Ok(vec![control_response_body_frame(
2399 &frame,
2400 &response,
2401 "ModuleControlResponseToModule::LiveRoots",
2402 )?])
2403 }
2404 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
2405 self.handle_scope_sync(connection_id, frame, generation, scopes)
2406 }
2407 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
2408 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
2409 }
2410 }
2411 }
2412
2413 fn handle_operator_confirm(
2414 &self,
2415 ctx: &RouteCtx,
2416 frame: Frame,
2417 ) -> Result<Vec<Frame>, RouterError> {
2418 use crate::operator_confirm::{audit, Outcome};
2419 let registration = self
2420 .registry
2421 .get_module_by_connection(ctx.connection_id)
2422 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2423 let Some(registration) = registration else {
2424 let outcome = Outcome::refusal("not_registered");
2425 audit("", "", "", outcome, Duration::ZERO, Duration::ZERO, false);
2426 return Ok(vec![outcome.frame(&frame)]);
2427 };
2428 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
2429 Ok(request) => request,
2430 Err(_) => {
2431 let outcome = Outcome::refusal("invalid_control_body");
2432 audit("", "", "", outcome, Duration::ZERO, Duration::ZERO, false);
2433 return Ok(vec![outcome.frame(&frame)]);
2434 }
2435 };
2436 let module_id = registration.manifest.module_id;
2437 let nonce = self
2441 .hello_launch_nonces
2442 .lock()
2443 .unwrap_or_else(|p| p.into_inner())
2444 .nonce(ctx.connection_id)
2445 .map(str::to_owned);
2446 let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
2447 self.supervisor
2448 .spawned_consumer_authorized(&module_id, nonce)
2449 });
2450 let confirms = self.forwarding.operator_confirms();
2451 self.forwarding
2452 .with_operator_route(
2453 ctx.connection_id,
2454 request.route_channel,
2455 request.route_epoch,
2456 |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
2457 )
2458 .map_err(RouterError::Forwarding)
2459 }
2460
2461 fn handle_scope_sync(
2465 &self,
2466 connection_id: ConnectionId,
2467 frame: Frame,
2468 generation: u64,
2469 scopes: Vec<ScopeRecord>,
2470 ) -> Result<Vec<Frame>, RouterError> {
2471 let Some(registration) = self
2472 .registry
2473 .get_module_by_connection(connection_id)
2474 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2475 else {
2476 return Ok(vec![control_error_frame(
2477 &frame,
2478 "not_registered",
2479 "scope.sync requires an active module registration owned by this connection",
2480 )?]);
2481 };
2482 let owner = registration.manifest.module_id;
2483 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
2484 let is_current_launch = |connection: ConnectionId| {
2485 self.hello_launch_nonces
2486 .lock()
2487 .unwrap_or_else(|poisoned| poisoned.into_inner())
2488 .presented(connection, current_nonce.as_deref())
2489 };
2490 let mut table = self
2495 .scopes
2496 .write()
2497 .unwrap_or_else(|poisoned| poisoned.into_inner());
2498 let outcome = table.sync(&owner, connection_id, is_current_launch, generation, scopes);
2499 let drained = match &outcome {
2500 Ok(applied) => self
2501 .forwarding
2502 .publish_scope_changes(&applied.tag_changes)
2503 .map_err(RouterError::Forwarding)?,
2504 Err(_) => Vec::new(),
2505 };
2506 drop(table);
2507 match outcome {
2508 Ok(applied) => {
2509 let counts = ScopeOutcomeCounts::of(&applied.results);
2510 info!(
2511 owner = %owner,
2512 generation,
2513 records = applied.results.len(),
2514 created = counts.created,
2515 replaced = counts.replaced,
2516 updated = counts.updated,
2517 unchanged = counts.unchanged,
2518 refused = counts.refused,
2519 ended = applied.ended.len(),
2520 tag_changes = applied.tag_changes.len(),
2521 routes_closed = drained.len(),
2522 "scope sync accepted"
2523 );
2524 for refused in applied
2530 .results
2531 .iter()
2532 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
2533 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
2534 {
2535 warn!(
2536 owner = %owner,
2537 generation,
2538 scope_ref = %refused.scope_ref,
2539 scope_epoch = refused.scope_epoch,
2540 code = refused.code.as_deref().unwrap_or(""),
2541 "scope record refused"
2542 );
2543 }
2544 self.close_scope_drained_routes(drained);
2545 let response = ModuleControlResponseToModule::ScopeSync {
2546 generation,
2547 results: applied.results,
2548 ended: applied.ended,
2549 };
2550 Ok(vec![control_response_body_frame(
2551 &frame,
2552 &response,
2553 "ModuleControlResponseToModule::ScopeSync",
2554 )?])
2555 }
2556 Err(refusal) => {
2557 info!(
2558 owner = %owner,
2559 generation,
2560 code = refusal.code,
2561 "scope sync refused"
2562 );
2563 Ok(vec![control_error_frame(
2564 &frame,
2565 refusal.code,
2566 refusal.message,
2567 )?])
2568 }
2569 }
2570 }
2571
2572 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
2578 if drained.is_empty() {
2579 return;
2580 }
2581 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
2582 BTreeMap::new();
2583 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
2584 for route in drained {
2585 warn!(
2586 module_id = %route.module_id,
2587 reason = ?route.reason,
2588 client_connection_id = route.client.connection_id.get(),
2589 route_channel = route.client.channel,
2590 "closing route because its scope changed"
2591 );
2592 pushes
2593 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
2594 .or_insert_with(|| (route.reason, Vec::new()))
2595 .1
2596 .push(EndpointRoute {
2597 goodbye_target: route.client.clone(),
2598 principal: Principal::Unverified,
2599 bound_at: Instant::now(),
2600 draining: false,
2601 drain_reason: None,
2602 });
2603 goodbyes.push(route.module);
2604 goodbyes.push(route.client);
2605 }
2606 for ((module_id, _), (reason, routes)) in pushes {
2607 send_route_control_pushes(
2608 &self.forwarding,
2609 routes,
2610 ClientControlPush::RouteClosed {
2611 module_id,
2612 channels: Vec::new(),
2613 reason,
2614 drained: false,
2615 abandoned: 0,
2616 excluded_subscriptions: 0,
2617 terminal: Some(false),
2618 },
2619 );
2620 }
2621 self.emit_route_goodbyes(goodbyes);
2622 }
2623
2624 fn handle_scope_describe(
2627 &self,
2628 connection_id: ConnectionId,
2629 frame: Frame,
2630 owner: Principal,
2631 scope_ref: String,
2632 ) -> Result<Vec<Frame>, RouterError> {
2633 let registered = self
2634 .registry
2635 .get_module_by_connection(connection_id)
2636 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2637 if registered.is_none() {
2638 return Ok(vec![control_error_frame(
2639 &frame,
2640 "not_registered",
2641 "scope.describe requires an active module registration owned by this connection",
2642 )?]);
2643 }
2644 let description = self
2645 .scopes
2646 .read()
2647 .unwrap_or_else(|poisoned| poisoned.into_inner())
2648 .describe(&owner, &scope_ref);
2649 let owner_configured = match &owner {
2650 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
2655 _ => false,
2656 };
2657 let response = ModuleControlResponseToModule::ScopeDescribe {
2658 status: description.status,
2659 scope_epoch: description.scope_epoch,
2660 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
2661 owner_synced: description.owner_synced,
2662 owner_configured,
2663 scope: description.stamp,
2664 };
2665 Ok(vec![control_response_body_frame(
2666 &frame,
2667 &response,
2668 "ModuleControlResponseToModule::ScopeDescribe",
2669 )?])
2670 }
2671
2672 fn handle_catalog_update(
2673 &self,
2674 connection_id: ConnectionId,
2675 frame: Frame,
2676 provides: Vec<ProviderRole>,
2677 capabilities: Option<CapabilityDeclarations>,
2678 ready: Option<bool>,
2679 ) -> Result<Vec<Frame>, RouterError> {
2680 self.refresh_capability_requirements();
2681 let Some(registration) = self
2682 .registry
2683 .get_module_by_connection(connection_id)
2684 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2685 else {
2686 return Ok(vec![control_error_frame(
2687 &frame,
2688 "not_registered",
2689 "catalog.update requires an active module registration owned by this connection",
2690 )?]);
2691 };
2692
2693 if let Some(message) =
2694 catalog_update_frozen_field_message(®istration.manifest, &provides)
2695 {
2696 return Ok(vec![control_error_frame(
2697 &frame,
2698 "catalog_update_frozen_field",
2699 message,
2700 )?]);
2701 }
2702
2703 let mut candidate = registration.manifest.clone();
2704 candidate.provides = provides.clone();
2705 candidate.capabilities = capabilities
2706 .clone()
2707 .or_else(|| registration.manifest.capabilities.clone());
2708 if let Err(err) = candidate.validate_capability_grammar() {
2709 return Ok(vec![control_error_frame(
2710 &frame,
2711 "invalid_capability_grammar",
2712 err.to_string(),
2713 )?]);
2714 }
2715
2716 let mut conflicts = self
2719 .capability_evaluator
2720 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
2721 if let Some(conflict) = conflicts.first() {
2722 let message = format!(
2723 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2724 conflict.capability, conflict.claimants[0], candidate.module_id
2725 );
2726 for conflict in &mut conflicts {
2727 conflict.source = DuplicateClaimSource::CatalogUpdate;
2728 }
2729 log_duplicate_claim_events(conflicts);
2730 return Ok(vec![control_error_frame(
2731 &frame,
2732 "reserved_capability",
2733 message,
2734 )?]);
2735 }
2736
2737 let updated = self
2738 .registry
2739 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
2740 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2741 if updated.is_none() {
2742 return Ok(vec![control_error_frame(
2743 &frame,
2744 "not_registered",
2745 "catalog.update requires an active module registration owned by this connection",
2746 )?]);
2747 }
2748 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
2749 log_duplicate_claim_events(
2750 self.capability_evaluator
2751 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
2752 );
2753 }
2754 if capability_census_trigger(
2755 registration.manifest.capabilities.as_ref(),
2756 updated
2757 .as_ref()
2758 .and_then(|entry| entry.manifest.capabilities.as_ref()),
2759 ) {
2760 self.enforce_capability_denies();
2761 }
2762 self.refresh_capability_requirements();
2763
2764 let response = ModuleControlResponseToModule::CatalogUpdate {};
2765 control_response_body_frame(
2766 &frame,
2767 &response,
2768 "ModuleControlResponseToModule::CatalogUpdate",
2769 )
2770 .map(|frame| vec![frame])
2771 }
2772
2773 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
2774 self.refresh_capability_requirements();
2775 let mut counters = self.counters.snapshot();
2781 if let (Ok((connections_with_routes, max)), Some(obj)) = (
2782 self.forwarding.client_route_concentration(),
2783 counters.as_object_mut(),
2784 ) {
2785 obj.insert(
2786 "client_connections_with_routes".into(),
2787 connections_with_routes.into(),
2788 );
2789 obj.insert("max_routes_on_one_connection".into(), max.into());
2790 }
2791 if let (Some(open_breakers), Some(obj)) = (
2797 self.route_bind_breakers.open_snapshot(),
2798 counters.as_object_mut(),
2799 ) {
2800 obj.insert("route_bind_breakers_open".into(), open_breakers);
2801 }
2802 let response = ClientControlResponse::ServerDescribe {
2803 protocol_ver: PROTOCOL_VERSION,
2804 subc_ops: subc_ops(),
2805 capabilities: self.subc_capabilities.as_ref().to_vec(),
2806 connected_clients: self.connected_clients.count(),
2807 counters: Some(counters),
2808 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
2809 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
2810 capability_requirements: self.capability_requirement_statuses(),
2811 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2812 };
2813 Ok(vec![control_response_body_frame(
2814 &frame,
2815 &response,
2816 "ClientControlResponse::ServerDescribe",
2817 )?])
2818 }
2819
2820 fn handle_catalog_list(
2821 &self,
2822 frame: Frame,
2823 module_id: Option<String>,
2824 ) -> Result<Vec<Frame>, RouterError> {
2825 let (generation, modules) = self.registry.list_modules().map_err(|err| {
2826 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
2827 })?;
2828 let entries = modules
2829 .into_iter()
2830 .filter(|registration| {
2831 module_id
2832 .as_deref()
2833 .map(|wanted| registration.manifest.module_id == wanted)
2834 .unwrap_or(true)
2835 })
2836 .map(|registration| {
2837 let not_ready = self.not_ready_reason(®istration);
2838 let roles = registration.manifest.provides;
2839 CatalogEntry {
2840 module_id: registration.manifest.module_id,
2841 ready: not_ready.is_none(),
2842 not_ready,
2843 module_version: Some(registration.manifest.module_version),
2844 roles,
2845 control_ops: registration.control_ops,
2846 capabilities: registration.manifest.capabilities,
2847 self_signals: registration.manifest.self_signals,
2848 }
2849 })
2850 .collect();
2851 let response = ClientControlResponse::CatalogList {
2852 generation,
2853 modules: entries,
2854 subc_ops: subc_ops(),
2855 };
2856 Ok(vec![control_response_body_frame(
2857 &frame,
2858 &response,
2859 "ClientControlResponse::CatalogList",
2860 )?])
2861 }
2862
2863 fn route_open_principal(
2864 &self,
2865 frame: &Frame,
2866 consumer_identity: Option<ConsumerIdentity>,
2867 ) -> Result<Result<Principal, Frame>, RouterError> {
2868 let Some(consumer_identity) = consumer_identity else {
2869 return Ok(Ok(Principal::Direct));
2870 };
2871
2872 if self.supervisor.spawned_consumer_authorized(
2873 &consumer_identity.module_id,
2874 &consumer_identity.launch_nonce,
2875 ) {
2876 return Ok(Ok(Principal::Reserved {
2877 module_id: consumer_identity.module_id,
2878 }));
2879 }
2880
2881 Ok(Err(control_error_frame(
2882 frame,
2883 "bad_consumer_identity",
2884 format!(
2885 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
2886 consumer_identity.module_id
2887 ),
2888 )?))
2889 }
2890
2891 fn route_open_refusal_frame(
2898 &self,
2899 ctx: &RouteCtx,
2900 frame: &Frame,
2901 module_id: &str,
2902 reason: &'static str,
2903 code: &'static str,
2904 message: impl Into<String>,
2905 ) -> Result<Frame, RouterError> {
2906 self.observe_route_open_refusal(ctx, module_id, reason, code);
2907 control_error_frame(frame, code, message.into())
2908 }
2909
2910 fn route_open_breaker_refusal_frame(
2928 &self,
2929 ctx: &RouteCtx,
2930 frame: &Frame,
2931 module_id: &str,
2932 consecutive_timeouts: u32,
2933 retry_in: Duration,
2934 probe_in_flight: bool,
2935 ) -> Result<Frame, RouterError> {
2936 self.counters
2937 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
2938 debug!(
2939 target: "control",
2940 code = "module_timeout",
2941 module_id = ?module_id,
2942 connection_id = ctx.connection_id.get(),
2943 consecutive_timeouts,
2944 retry_in_ms = retry_in.as_millis() as u64,
2945 probe_in_flight,
2946 "route.open refused by open bind-relay breaker"
2947 );
2948 let detail = if probe_in_flight {
2955 "one test route is already being tried; retry once it settles".to_string()
2956 } else {
2957 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
2958 };
2959 control_error_frame(
2960 frame,
2961 "module_timeout",
2962 format!(
2963 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
2964 timed out in a row); {detail}; its established routes are unaffected"
2965 ),
2966 )
2967 }
2968
2969 fn observe_route_open_refusal(
2982 &self,
2983 ctx: &RouteCtx,
2984 module_id: &str,
2985 reason: &'static str,
2986 code: &'static str,
2987 ) {
2988 self.counters.increment_route_open_refused(code);
2989 info!(
2990 target: "control",
2991 code,
2992 reason,
2993 module_id = ?module_id,
2994 connection_id = ctx.connection_id.get(),
2995 "route.open refused"
2996 );
2997 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
2998 self.route_outages.record_not_serving(module_id, reason);
2999 }
3000 }
3001
3002 fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
3047 self.route_outages.record_accepted(module_id);
3048 self.counters.increment_route_open_accepted(principal);
3049 info!(
3050 target: "control",
3051 principal,
3052 module_id,
3053 connection_id = ctx.connection_id.get(),
3054 "route.open accepted"
3055 );
3056 }
3057
3058 fn supervised_absent_route_open_refusal_frame(
3059 &self,
3060 ctx: &RouteCtx,
3061 frame: &Frame,
3062 module_id: &str,
3063 code: &'static str,
3064 status: &crate::supervise::ModuleStatus,
3065 ) -> Result<Frame, RouterError> {
3066 self.counters.increment_route_open_refused(code);
3067 info!(
3068 target: "control",
3069 code,
3070 reason = "supervised_not_registered",
3071 module_id = ?module_id,
3072 connection_id = ctx.connection_id.get(),
3073 state = %status.state,
3074 enabled = status.enabled,
3075 live = status.live,
3076 "route.open refused"
3077 );
3078 self.route_outages
3082 .record_not_serving(module_id, "supervised_not_registered");
3083 control_error_frame(
3084 frame,
3085 code,
3086 format!(
3087 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
3088 status.state, status.enabled, status.live
3089 ),
3090 )
3091 }
3092
3093 async fn handle_route_open(
3094 &self,
3095 ctx: &RouteCtx,
3096 frame: Frame,
3097 request: RouteOpenRequest,
3098 ) -> Result<Vec<Frame>, RouterError> {
3099 let RouteOpenRequest {
3100 target,
3101 mut identity,
3102 consumer_identity,
3103 consumer_capabilities,
3104 role_versions,
3105 admission_facts,
3106 scope,
3107 } = request;
3108 let target_module_id = target_module_id(&target).to_string();
3109 if self
3110 .registry
3111 .get_module_by_connection(ctx.connection_id)
3112 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3113 .is_some()
3114 {
3115 return Ok(vec![control_error_frame(
3116 &frame,
3117 "invalid_request",
3118 "module connections cannot open client routes",
3119 )?]);
3120 }
3121 debug!(
3122 connection_id = ctx.connection_id.get(),
3123 corr = frame.header.corr,
3124 module_id = %target_module_id,
3125 "handling route.open"
3126 );
3127
3128 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
3134 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
3135 self.observe_route_open_refusal(
3136 ctx,
3137 &target_module_id,
3138 "invalid_role_versions",
3139 error_codes::INVALID_REQUEST,
3140 );
3141 return Ok(vec![control_error_body_frame(
3142 &frame,
3143 ErrorBody {
3144 code: error_codes::INVALID_REQUEST.to_string(),
3145 message: error.to_string(),
3146 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
3147 },
3148 )?]);
3149 }
3150
3151 let Some(registration) = self
3170 .registry
3171 .get_module(&target_module_id)
3172 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3173 else {
3174 if let Some((status, warming)) =
3175 self.supervisor_status(&target_module_id, frame.header.corr)?
3176 {
3177 if status.protocol == ModuleProtocol::None {
3184 return Ok(vec![self.route_open_refusal_frame(
3185 ctx,
3186 &frame,
3187 &target_module_id,
3188 "protocol_none",
3189 error_codes::MODULE_NO_PROTOCOL,
3190 format!(
3191 "module_id '{target_module_id}' is declared protocol: none; \
3192 it speaks no subc wire and serves no routes"
3193 ),
3194 )?]);
3195 }
3196 let code = if warming {
3197 "module_warming"
3198 } else {
3199 "target_unavailable"
3200 };
3201 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
3202 ctx,
3203 &frame,
3204 &target_module_id,
3205 code,
3206 &status,
3207 )?]);
3208 }
3209 if let Some(removed_ago_ms) =
3210 self.supervisor.removal_tombstone_age_ms(&target_module_id)
3211 {
3212 return Ok(vec![self.route_open_refusal_frame(
3213 ctx,
3214 &frame,
3215 &target_module_id,
3216 "removed",
3217 error_codes::MODULE_REMOVED,
3218 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
3219 )?]);
3220 }
3221 return Ok(vec![self.route_open_refusal_frame(
3222 ctx,
3223 &frame,
3224 &target_module_id,
3225 "not_registered",
3226 error_codes::UNKNOWN_MODULE,
3227 format!("module_id '{target_module_id}' is not registered"),
3228 )?]);
3229 };
3230
3231 if !registration.ready {
3235 self.counters
3236 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
3237 info!(
3238 target: "control",
3239 code = error_codes::MODULE_WARMING,
3240 module_id = ?target_module_id,
3241 connection_id = ctx.connection_id.get(),
3242 reason = "declared_not_ready",
3243 "route.open refused"
3244 );
3245 self.route_outages
3248 .record_not_serving(&target_module_id, "declared_not_ready");
3249 return Ok(vec![control_error_body_frame(
3250 &frame,
3251 ErrorBody {
3252 code: error_codes::MODULE_WARMING.to_string(),
3253 message: format!(
3254 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
3255 ),
3256 detail: Some(serde_json::json!({
3257 "reason": "declared_not_ready"
3258 })),
3259 },
3260 )?]);
3261 }
3262
3263 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
3282 self.counters
3283 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
3284 info!(
3285 target: "control",
3286 code = error_codes::MODULE_WARMING,
3287 module_id = ?target_module_id,
3288 connection_id = ctx.connection_id.get(),
3289 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3290 capability = %capability,
3291 "route.open refused"
3292 );
3293 return Ok(vec![control_error_body_frame(
3294 &frame,
3295 ErrorBody {
3296 code: error_codes::MODULE_WARMING.to_string(),
3297 message: format!(
3298 "module_id '{target_module_id}' requires capability '{capability}', \
3299 which no registered module provides; retry"
3300 ),
3301 detail: Some(serde_json::json!({
3302 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3303 "capability": capability,
3304 })),
3305 },
3306 )?]);
3307 }
3308
3309 if !target_has_required_role(&target, ®istration.manifest.provides) {
3310 return Ok(vec![self.route_open_refusal_frame(
3311 ctx,
3312 &frame,
3313 &target_module_id,
3314 "role_not_provided",
3315 "target_unavailable",
3316 format!("module_id '{target_module_id}' does not provide the requested target"),
3317 )?]);
3318 }
3319
3320 if registration.state != ChannelState::Active {
3321 return Ok(vec![self.route_open_refusal_frame(
3322 ctx,
3323 &frame,
3324 &target_module_id,
3325 "registration_not_active",
3326 "target_unavailable",
3327 format!("module_id '{target_module_id}' is not active"),
3328 )?]);
3329 }
3330
3331 if self
3332 .forwarding
3333 .module_is_draining(&target_module_id)
3334 .map_err(RouterError::Forwarding)?
3335 {
3336 return Ok(vec![self.route_open_refusal_frame(
3337 ctx,
3338 &frame,
3339 &target_module_id,
3340 "reloading",
3341 "module_reloading",
3342 format!("module_id '{target_module_id}' is reloading"),
3343 )?]);
3344 }
3345
3346 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
3347 process_liveness.process_live(&target_module_id) == Some(false)
3348 }) {
3349 if process_liveness.process_replacing(&target_module_id) {
3356 return Ok(vec![self.route_open_refusal_frame(
3357 ctx,
3358 &frame,
3359 &target_module_id,
3360 "reloading",
3361 "module_reloading",
3362 format!("module_id '{target_module_id}' is reloading"),
3363 )?]);
3364 }
3365 return Ok(vec![self.route_open_refusal_frame(
3366 ctx,
3367 &frame,
3368 &target_module_id,
3369 "supervisor_not_live",
3370 "target_unavailable",
3371 format!("module_id '{target_module_id}' is not live"),
3372 )?]);
3373 }
3374
3375 if !self
3376 .forwarding
3377 .has_live_module_connection(&target_module_id)
3378 .map_err(RouterError::Forwarding)?
3379 {
3380 return Ok(vec![self.route_open_refusal_frame(
3381 ctx,
3382 &frame,
3383 &target_module_id,
3384 "no_forwarding_connection",
3385 "target_unavailable",
3386 format!("module_id '{target_module_id}' has no live forwarding connection"),
3387 )?]);
3388 }
3389
3390 if let Some(error) =
3391 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
3392 {
3393 self.observe_route_open_refusal(
3394 ctx,
3395 &target_module_id,
3396 "op_not_allowed",
3397 "op_not_allowed",
3398 );
3399 return Ok(vec![error]);
3400 }
3401
3402 let principal = match self.route_open_principal(&frame, consumer_identity)? {
3403 Ok(principal) => principal,
3404 Err(error) => {
3405 self.observe_route_open_refusal(
3406 ctx,
3407 &target_module_id,
3408 "bad_consumer_identity",
3409 "bad_consumer_identity",
3410 );
3411 return Ok(vec![error]);
3412 }
3413 };
3414
3415 if let Principal::Reserved {
3419 module_id: opening_module_id,
3420 } = &principal
3421 {
3422 if let Some(opening_registration) = self
3423 .registry
3424 .get_module(opening_module_id)
3425 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3426 {
3427 if let Some(capability) =
3428 denied_capability(&opening_registration.manifest, ®istration.manifest)
3429 {
3430 warn!(
3431 opening_module_id,
3432 target_module_id,
3433 capability,
3434 "refusing route.open because an attested capability deny edge matches"
3435 );
3436 return Ok(vec![self.route_open_refusal_frame(
3437 ctx,
3438 &frame,
3439 &target_module_id,
3440 "capability_deny_edge",
3441 "capability_forbidden",
3442 format!(
3443 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
3444 ),
3445 )?]);
3446 }
3447 }
3448 }
3449
3450 if admission_facts.is_some() {
3451 let carrier_matches = matches!(
3452 &principal,
3453 Principal::Reserved { module_id }
3454 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
3455 );
3456 if !carrier_matches {
3457 return Ok(vec![self.route_open_refusal_frame(
3458 ctx,
3459 &frame,
3460 &target_module_id,
3461 "admission_facts_carrier_not_permitted",
3462 "admission_facts_not_permitted",
3463 "admission facts may only be carried by the configured reserved module",
3464 )?]);
3465 }
3466
3467 let target_allowed = self
3468 .admission_facts_targets
3469 .as_ref()
3470 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
3471 if !target_allowed {
3472 return Ok(vec![self.route_open_refusal_frame(
3473 ctx,
3474 &frame,
3475 &target_module_id,
3476 "admission_facts_target_not_listed",
3477 "admission_facts_target_not_allowed",
3478 format!(
3479 "admission facts are not permitted for target module_id '{target_module_id}'"
3480 ),
3481 )?]);
3482 }
3483
3484 }
3488
3489 let (bound_scope, scope_stamp) = match scope {
3495 None => (None, None),
3496 Some(selector) => {
3497 let owner_configured = match &selector.owner {
3498 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3504 _ => false,
3505 };
3506 let admitted = self
3507 .scopes
3508 .read()
3509 .unwrap_or_else(|poisoned| poisoned.into_inner())
3510 .admit(&principal, &target_module_id, &selector, owner_configured)
3511 .and_then(|admission| {
3512 crate::scopes::check_target_flow_support(
3513 &admission.stamp,
3514 &target_module_id,
3515 registration.manifest.capabilities.as_ref(),
3516 )?;
3517 Ok(admission)
3518 });
3519 match admitted {
3520 Ok(admission) => (
3521 Some(BoundScope {
3522 owner: admission.owner,
3523 scope_ref: admission.stamp.scope_ref.clone(),
3524 tag: admission.tag,
3525 }),
3526 Some(admission.stamp),
3527 ),
3528 Err(refusal) => {
3529 return Ok(vec![self.route_open_refusal_frame(
3530 ctx,
3531 &frame,
3532 &target_module_id,
3533 refusal.code,
3534 refusal.code,
3535 refusal.message,
3536 )?]);
3537 }
3538 }
3539 }
3540 };
3541
3542 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
3564 Ok(project_root) => project_root,
3565 Err(err) => {
3566 return Ok(vec![control_error_frame(
3567 &frame,
3568 "invalid_project_root",
3569 err.to_string(),
3570 )?])
3571 }
3572 };
3573 identity.project_root = project_root.as_path().to_path_buf();
3574
3575 let _concurrency_guard = match self
3586 .route_bind_concurrency
3587 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
3588 {
3589 Ok(guard) => guard,
3590 Err(in_flight) => {
3591 return Ok(vec![self.route_open_target_capacity_refusal(
3592 ctx,
3593 &frame,
3594 &target_module_id,
3595 in_flight,
3596 )?]);
3597 }
3598 };
3599
3600 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
3603 RouteBindAdmission::Admitted { guard, probe } => {
3604 if probe {
3605 info!(
3606 module_id = %target_module_id,
3607 connection_id = ctx.connection_id.get(),
3608 "route.bind breaker half-open: admitting one probe"
3609 );
3610 }
3611 guard
3612 }
3613 RouteBindAdmission::Refused {
3614 consecutive_timeouts,
3615 retry_in,
3616 probe_in_flight,
3617 } => {
3618 return Ok(vec![self.route_open_breaker_refusal_frame(
3619 ctx,
3620 &frame,
3621 &target_module_id,
3622 consecutive_timeouts,
3623 retry_in,
3624 probe_in_flight,
3625 )?]);
3626 }
3627 };
3628
3629 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
3634 let relay_deadline = Instant::now() + route_bind_relay_timeout;
3635 let pending = match self
3636 .forwarding
3637 .begin_route_bind_relay_for(
3638 ctx.connection_id,
3639 ctx.egress.clone(),
3640 response_version(&frame),
3641 frame.header.corr,
3642 &target_module_id,
3643 principal.clone(),
3644 bound_scope,
3645 Some(project_root),
3646 relay_deadline,
3647 )
3648 .await
3649 {
3650 Ok(pending) => pending,
3651 Err(err) => {
3652 return Ok(vec![self.route_open_refusal_frame(
3653 ctx,
3654 &frame,
3655 &target_module_id,
3656 "relay_reservation_failed",
3657 forwarding_error_code(&err),
3658 err.to_string(),
3659 )?])
3660 }
3661 };
3662 let crate::forwarding::PendingRouteBindRelay {
3663 endpoint,
3664 module_sink,
3665 negotiated_ver,
3666 client_channel,
3667 client_epoch,
3668 module_channel,
3669 module_epoch,
3670 corr: relay_corr,
3671 receiver,
3672 } = pending;
3673 let mut reservation =
3674 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
3675
3676 if let Some(stamp) = scope_stamp
3681 .as_ref()
3682 .filter(|stamp| stamp.attributes.flow_id.is_some())
3683 {
3684 let relay_registration = self
3685 .registry
3686 .get_module_by_connection(endpoint.connection_id)
3687 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3688 if let Err(refusal) = crate::scopes::check_target_flow_support(
3689 stamp,
3690 &target_module_id,
3691 relay_registration
3692 .as_ref()
3693 .and_then(|registration| registration.manifest.capabilities.as_ref()),
3694 ) {
3695 reservation.release_and_disarm();
3696 return Ok(vec![self.route_open_refusal_frame(
3697 ctx,
3698 &frame,
3699 &target_module_id,
3700 refusal.code,
3701 refusal.code,
3702 refusal.message,
3703 )?]);
3704 }
3705 }
3706
3707 debug!(
3708 connection_id = ctx.connection_id.get(),
3709 client_channel,
3710 client_epoch,
3711 module_channel,
3712 module_epoch,
3713 "reserved route handle pair"
3714 );
3715 let principal_label = match &principal {
3718 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
3719 Principal::Direct => "direct".to_string(),
3720 other => format!("{other:?}"),
3721 };
3722 let relay = ModuleControlRequest::RouteBind {
3723 route_channel: module_channel,
3724 epoch: module_epoch,
3725 target,
3726 identity,
3727 principal: Some(principal),
3728 consumer_capabilities,
3729 role_versions,
3730 admission_facts,
3731 scope: scope_stamp,
3732 };
3733 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
3734 RouterError::backend(
3735 0,
3736 frame.header.corr,
3737 format!("failed to encode route.bind request: {err}"),
3738 )
3739 })?;
3740 let relay_frame = Frame::build_with_version(
3741 negotiated_ver,
3742 FrameType::Request,
3743 control_flags(),
3744 0,
3745 0,
3746 relay_corr,
3747 relay_body,
3748 )
3749 .map_err(RouterError::FrameBuild)?;
3750
3751 if let Err(err) = module_sink.send(relay_frame).await {
3752 reservation.release_and_disarm();
3753 return Ok(vec![self.route_open_refusal_frame(
3754 ctx,
3755 &frame,
3756 &target_module_id,
3757 "relay_send_failed",
3758 "target_unavailable",
3759 err.to_string(),
3760 )?]);
3761 }
3762
3763 if !self
3764 .forwarding
3765 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
3766 .map_err(RouterError::Forwarding)?
3767 {
3768 self.send_abandoned_route_bind_goodbye(
3769 &module_sink,
3770 negotiated_ver,
3771 module_channel,
3772 module_epoch,
3773 );
3774 }
3775
3776 match timeout_at(relay_deadline, receiver).await {
3777 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
3778 reservation.disarm();
3779 if breaker.record_accepted() {
3780 info!(
3781 module_id = %target_module_id,
3782 "route.bind breaker closed: the probe was accepted"
3783 );
3784 }
3785 self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
3786 Ok(Vec::new())
3787 }
3788 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
3789 reservation.release_and_disarm();
3790 breaker.record_inconclusive();
3794 let scope_code = match body.code.as_str() {
3798 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
3799 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
3800 _ => None,
3801 };
3802 if let Some(code) = scope_code {
3803 self.observe_route_open_refusal(
3804 ctx,
3805 &target_module_id,
3806 "scope_changed_before_commit",
3807 code,
3808 );
3809 return Ok(vec![control_error_body_frame(&frame, body)?]);
3810 }
3811 self.counters
3812 .increment_route_open_refused("module_rejected");
3813 info!(
3814 target: "control",
3815 code = "module_rejected",
3816 module_code = ?body.code,
3817 module_id = ?target_module_id,
3818 connection_id = ctx.connection_id.get(),
3819 "route.open refused"
3820 );
3821 Ok(vec![control_error_body_frame(&frame, body)?])
3822 }
3823 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
3824 reservation.release_and_disarm();
3825 breaker.record_inconclusive();
3826 tracing::warn!(
3833 module_id = %target_module_id,
3834 "route.bind relay abandoned: {message}"
3835 );
3836 Ok(vec![self.route_open_refusal_frame(
3837 ctx,
3838 &frame,
3839 &target_module_id,
3840 "relay_abandoned",
3841 "target_unavailable",
3842 message,
3843 )?])
3844 }
3845 Ok(Err(_)) => {
3846 reservation.release_and_disarm();
3847 breaker.record_inconclusive();
3848 Ok(vec![self.route_open_refusal_frame(
3849 ctx,
3850 &frame,
3851 &target_module_id,
3852 "relay_waiter_canceled",
3853 "target_unavailable",
3854 "route.bind relay waiter was canceled before the module responded",
3855 )?])
3856 }
3857 Err(_) => {
3858 reservation.release_and_disarm();
3859 if let Some(opened) = breaker.record_timeout(
3863 self.route_bind_breaker_threshold,
3864 self.route_bind_breaker_cooldown,
3865 ) {
3866 warn!(
3867 module_id = %target_module_id,
3868 consecutive_timeouts = opened.consecutive_timeouts,
3869 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
3870 reopened_after_probe = opened.reopened_after_probe,
3871 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
3872 );
3873 }
3874 tracing::warn!(
3882 module_id = %target_module_id,
3883 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
3884 "route.bind relay timed out: module did not ack within budget"
3885 );
3886 Ok(vec![self.route_open_refusal_frame(
3887 ctx,
3888 &frame,
3889 &target_module_id,
3890 "relay_timed_out",
3891 "module_timeout",
3892 format!(
3893 "module_id '{target_module_id}' did not answer route.bind within {:?}",
3894 route_bind_relay_timeout
3895 ),
3896 )?])
3897 }
3898 }
3899 }
3900
3901 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3902 let response = ClientControlResponse::SupervisorSpawnSnapshot {
3903 snapshot: self.supervisor.spawn_snapshot(),
3904 };
3905 Ok(vec![control_response_body_frame(
3906 &frame,
3907 &response,
3908 "ClientControlResponse::SupervisorSpawnSnapshot",
3909 )?])
3910 }
3911
3912 fn handle_supervisor_spawn_subscribe(
3913 &self,
3914 ctx: &RouteCtx,
3915 frame: Frame,
3916 since: Option<SpawnCursor>,
3917 ) -> Result<Vec<Frame>, RouterError> {
3918 match self.supervisor.subscribe_spawns(
3919 ctx.connection_id,
3920 frame.header.corr,
3921 response_version(&frame),
3922 since,
3923 ctx.egress.clone(),
3924 ) {
3925 Ok(()) => Ok(Vec::new()),
3926 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
3927 Ok(vec![control_error_body_frame(
3928 &frame,
3929 ErrorBody {
3930 code: "spawn_cursor_incarnation_mismatch".to_string(),
3931 message: "spawn cursor belongs to a different daemon incarnation"
3932 .to_string(),
3933 detail: Some(serde_json::json!({
3934 "current_daemon_incarnation": current
3935 })),
3936 },
3937 )?])
3938 }
3939 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
3940 &frame,
3941 ErrorBody {
3942 code: "spawn_cursor_too_old".to_string(),
3943 message: "spawn cursor predates the retained event ring".to_string(),
3944 detail: Some(serde_json::json!({
3945 "oldest_retained_cursor": oldest
3946 })),
3947 },
3948 )?]),
3949 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
3950 0,
3951 frame.header.corr,
3952 format!("failed to open supervisor spawn subscription: {error}"),
3953 )),
3954 }
3955 }
3956
3957 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3958 let generation = self
3959 .registry
3960 .generation()
3961 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3962 let mut modules = Vec::new();
3963 for module in self.supervisor.list() {
3964 let status = module.status_for_control("list").map_err(|err| {
3965 RouterError::backend(
3966 0,
3967 frame.header.corr,
3968 format!("failed to read supervisor status: {err}"),
3969 )
3970 })?;
3971 let (configured, _) = module.configuration().map_err(|err| {
3972 RouterError::backend(
3973 0,
3974 frame.header.corr,
3975 format!("failed to read module configuration: {err}"),
3976 )
3977 })?;
3978 let image = module.running_image_agreement().await;
3980 let resources = Some(module.child_resource_usage());
3983 let pending_reload = Some(reload_verdict(
3984 &configured.program,
3985 status.spawned_from.as_deref(),
3986 image,
3987 ));
3988 modules.push(SupervisorEntry {
3989 launch_nonce_env: Some(!cfg!(unix)),
3992 module_id: status.module_id,
3993 state: status.state.to_string(),
3994 enabled: status.enabled,
3995 live: status.live,
3996 protocol: status.protocol,
3997 health: status.health.status,
3998 pending_reload,
3999 last_probe_ms: status.health.last_probe_ms,
4000 last_exit_code: status.last_exit.as_ref().and_then(|e| e.code),
4001 last_exit_signal: status.last_exit.as_ref().and_then(|e| e.signal),
4002 last_exit_ms: status.last_exit.as_ref().map(|e| e.at_ms),
4003 last_exit_kind: status.last_exit.as_ref().map(|e| e.kind.into()),
4004 restart_count: Some(status.restart_count),
4005 max_restarts: Some(status.max_restarts),
4006 lifetime_restarts: Some(status.lifetime_restarts),
4007 spawn_generation: Some(status.spawn_generation),
4008 restart_window_secs: Some(status.restart_window.as_secs()),
4009 drain_timeout_ms: Some(status.drain_timeout.as_millis() as u64),
4010 restart_backoff_ms: Some(status.restart_backoff.as_millis() as u64),
4011 restart_max_backoff_ms: Some(status.restart_max_backoff.as_millis() as u64),
4012 resources,
4013 });
4014 }
4015 let response = ClientControlResponse::SupervisorList {
4016 generation,
4017 modules,
4018 };
4019 Ok(vec![control_response_body_frame(
4020 &frame,
4021 &response,
4022 "ClientControlResponse::SupervisorList",
4023 )?])
4024 }
4025
4026 fn handle_supervisor_stderr_tail(
4027 &self,
4028 frame: Frame,
4029 module_id: String,
4030 max_lines: Option<u32>,
4031 max_bytes: Option<u32>,
4032 ) -> Result<Vec<Frame>, RouterError> {
4033 let Some(module) = self.supervisor.get(&module_id) else {
4034 return Ok(vec![control_error_frame(
4035 &frame,
4036 "unknown_module",
4037 format!("module_id '{module_id}' is not supervised"),
4038 )?]);
4039 };
4040
4041 let snapshot = module.stderr_tail(
4042 max_lines.map(|value| value as usize),
4043 max_bytes.map(|value| value as usize),
4044 );
4045
4046 let response = ClientControlResponse::SupervisorStderrTail {
4047 module_id,
4048 tail: StderrTail {
4049 capture: match snapshot.capture {
4050 CaptureState::Captured => StderrCaptureState::Captured,
4051 CaptureState::Incomplete { reason } => {
4052 StderrCaptureState::Incomplete { reason }
4053 }
4054 CaptureState::NotCaptured { reason } => {
4055 StderrCaptureState::NotCaptured { reason }
4056 }
4057 },
4058 entries: snapshot
4059 .entries
4060 .into_iter()
4061 .map(|entry| match entry {
4062 TailEntry::Line {
4063 text,
4064 truncated,
4065 at_ms,
4066 } => StderrTailEntry::Line {
4067 text,
4068 truncated,
4069 at_ms,
4070 },
4071 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
4072 })
4073 .collect(),
4074 dropped_lines: snapshot.dropped_lines,
4075 },
4076 };
4077 Ok(vec![control_response_body_frame(
4078 &frame,
4079 &response,
4080 "ClientControlResponse::SupervisorStderrTail",
4081 )?])
4082 }
4083
4084 async fn handle_supervisor_terminals(
4085 &self,
4086 frame: Frame,
4087 module_id: String,
4088 ) -> Result<Vec<Frame>, RouterError> {
4089 let Some(module) = self.supervisor.get(&module_id) else {
4090 return Ok(vec![control_error_frame(
4091 &frame,
4092 "unknown_module",
4093 format!("module_id '{module_id}' is not supervised"),
4094 )?]);
4095 };
4096
4097 let terminals = module
4100 .read_durable_terminal_history()
4101 .await
4102 .map_err(|error| {
4103 RouterError::backend(
4104 0,
4105 frame.header.corr,
4106 format!("failed to read terminal history: {error}"),
4107 )
4108 })?;
4109 let response = ClientControlResponse::SupervisorTerminals {
4110 module_id,
4111 terminals,
4112 };
4113 Ok(vec![control_response_body_frame(
4114 &frame,
4115 &response,
4116 "ClientControlResponse::SupervisorTerminals",
4117 )?])
4118 }
4119
4120 fn handle_supervisor_routes(
4121 &self,
4122 frame: Frame,
4123 module_id: Option<String>,
4124 ) -> Result<Vec<Frame>, RouterError> {
4125 let modules = self
4126 .forwarding
4127 .route_census(module_id.as_deref())
4128 .map_err(RouterError::Forwarding)?
4129 .into_iter()
4130 .map(|(module_id, routes)| SupervisorRouteModule {
4131 module_id,
4132 routes: routes
4133 .into_iter()
4134 .map(|route| SupervisorRoute {
4135 consumer: match route.principal {
4136 Principal::Reserved { module_id } => {
4137 SupervisorRouteConsumer::Reserved { module_id }
4138 }
4139 Principal::Direct | Principal::Unverified => {
4140 SupervisorRouteConsumer::Direct {
4141 connection_id: route.goodbye_target.connection_id.get(),
4142 }
4143 }
4144 },
4145 age_ms: Instant::now()
4146 .saturating_duration_since(route.bound_at)
4147 .as_millis()
4148 .try_into()
4149 .unwrap_or(u64::MAX),
4150 draining: route.draining,
4151 drain_reason: route.drain_reason,
4152 })
4153 .collect(),
4154 })
4155 .collect();
4156 let response = ClientControlResponse::SupervisorRoutes { modules };
4157 Ok(vec![control_response_body_frame(
4158 &frame,
4159 &response,
4160 "ClientControlResponse::SupervisorRoutes",
4161 )?])
4162 }
4163
4164 async fn handle_supervisor_provenance(
4165 &self,
4166 frame: Frame,
4167 module_id: Option<String>,
4168 ) -> Result<Vec<Frame>, RouterError> {
4169 let mut selected = if let Some(module_id) = module_id {
4170 let Some(module) = self.supervisor.get(&module_id) else {
4171 return Ok(vec![control_error_frame(
4172 &frame,
4173 "unknown_module",
4174 format!("module_id '{module_id}' is not supervised"),
4175 )?]);
4176 };
4177 vec![module]
4178 } else {
4179 self.supervisor.list()
4180 };
4181
4182 let mut modules = Vec::with_capacity(selected.len());
4183 for module in selected.drain(..) {
4184 let (status, observed_image) = module
4185 .status_and_running_image_agreement()
4186 .await
4187 .map_err(|err| {
4188 RouterError::backend(
4189 0,
4190 frame.header.corr,
4191 format!("failed to read supervisor status: {err}"),
4192 )
4193 })?;
4194 let module_declared = self
4195 .registry
4196 .get_module(&status.module_id)
4197 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4198 .and_then(|registration| registration.manifest.provenance)
4199 .map(|build| ModuleDeclaredProvenance::Reported { build })
4200 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
4201 #[cfg(test)]
4202 let running_image = match &self.provenance_probe_override {
4203 Some(result) => result.clone(),
4204 None => observed_image,
4205 };
4206 #[cfg(not(test))]
4207 let running_image = observed_image;
4208 modules.push(SupervisorModuleProvenance {
4209 module_id: status.module_id,
4210 module_declared,
4211 daemon_observed: SupervisorObservedProcess {
4212 pid: status.pid,
4213 spawned_at_ms: status.spawned_at_ms,
4214 spawned_from: status.spawned_from,
4215 running_image,
4216 },
4217 });
4218 }
4219 let daemon = SupervisorDaemonProvenance {
4220 daemon_build: self.daemon_provenance.build.clone(),
4221 daemon_observed: DaemonObservedProcess {
4222 pid: self.daemon_provenance.pid,
4223 started_at_ms: self
4224 .daemon_provenance
4225 .start_clock
4226 .map(|clock| clock.started_at_ms())
4227 .or(self.daemon_provenance.started_at_ms),
4228 running_image: self
4229 .daemon_provenance
4230 .probe
4231 .observe(
4232 self.daemon_provenance.pid,
4233 self.daemon_provenance.executable_path.as_deref(),
4234 self.daemon_provenance.executable_identity,
4235 self.daemon_provenance.process_start_time,
4236 )
4237 .await,
4238 },
4239 };
4240 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
4241 Ok(vec![control_response_body_frame(
4242 &frame,
4243 &response,
4244 "ClientControlResponse::SupervisorProvenance",
4245 )?])
4246 }
4247
4248 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4249 self.refresh_capability_requirements();
4250 let generation = self
4251 .registry
4252 .generation()
4253 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4254 let modules = self
4255 .supervisor
4256 .list()
4257 .into_iter()
4258 .map(|module| {
4259 let status = module.status_for_control("health").map_err(|err| {
4260 RouterError::backend(
4261 0,
4262 frame.header.corr,
4263 format!("failed to read supervisor health: {err}"),
4264 )
4265 })?;
4266 let module_id = status.module_id;
4267 let capability_detail = self
4268 .capability_evaluator
4269 .required_problem_detail(&module_id);
4270 Ok(SupervisorHealthEntry {
4271 module_id,
4272 status: status.health.status,
4273 detail: append_capability_problem_detail(
4274 status.health.detail,
4275 capability_detail,
4276 ),
4277 metrics: status.health.metrics,
4278 consecutive_failures: status.health.consecutive_failures,
4279 late_answer_count: status.health.late_answer_count,
4280 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
4281 last_action: status.health.last_action,
4282 last_action_ms: status.health.last_action_ms,
4283 last_probe_ms: status.health.last_probe_ms,
4284 })
4285 })
4286 .collect::<Result<Vec<_>, RouterError>>()?;
4287 let response = ClientControlResponse::SupervisorHealth {
4288 generation,
4289 modules,
4290 };
4291 Ok(vec![control_response_body_frame(
4292 &frame,
4293 &response,
4294 "ClientControlResponse::SupervisorHealth",
4295 )?])
4296 }
4297
4298 async fn handle_supervisor_restart(
4299 &self,
4300 frame: Frame,
4301 module_id: String,
4302 drain_timeout_ms: Option<u64>,
4303 ) -> Result<Vec<Frame>, RouterError> {
4304 let operation_lock = self.supervisor.operation_lock();
4305 let _operation_guard = operation_lock.lock().await;
4306 let Some(module) = self.supervisor.get(&module_id) else {
4307 return Ok(vec![control_error_frame(
4308 &frame,
4309 "unknown_module",
4310 format!("module_id '{module_id}' is not supervised"),
4311 )?]);
4312 };
4313
4314 self.route_outages.mark_operator_action(&module_id);
4315 if let Err(err) = module.restart(drain_timeout_ms).await {
4316 self.route_outages
4317 .operator_action_ended_unrefused(&module_id);
4318 let (code, message) = match err {
4319 crate::supervise::SuperviseError::Disabled { .. } => {
4320 ("module_disabled", err.to_string())
4321 }
4322 crate::supervise::SuperviseError::SwapInProgress { .. } => {
4323 ("swap_in_progress", err.to_string())
4324 }
4325 _ => (
4326 "target_unavailable",
4327 format!("failed to restart module_id '{module_id}': {err}"),
4328 ),
4329 };
4330 return Ok(vec![control_error_frame(&frame, code, message)?]);
4331 }
4332
4333 let response = ClientControlResponse::SupervisorAck {
4334 module_id,
4335 applied: true,
4336 };
4337 Ok(vec![control_response_body_frame(
4338 &frame,
4339 &response,
4340 "ClientControlResponse::SupervisorAck",
4341 )?])
4342 }
4343
4344 async fn handle_supervisor_swap(
4348 &self,
4349 frame: Frame,
4350 module_id: String,
4351 ready_timeout_ms: Option<u64>,
4352 ) -> Result<Vec<Frame>, RouterError> {
4353 let module = {
4360 let operation_lock = self.supervisor.operation_lock();
4361 let _operation_guard = operation_lock.lock().await;
4362 self.supervisor.get(&module_id)
4363 };
4364 let Some(module) = module else {
4365 return Ok(vec![control_error_frame(
4366 &frame,
4367 "unknown_module",
4368 format!("module_id '{module_id}' is not supervised"),
4369 )?]);
4370 };
4371
4372 self.route_outages.mark_operator_action(&module_id);
4373 if let Err(err) = module
4374 .swap(ready_timeout_ms.map(Duration::from_millis))
4375 .await
4376 {
4377 self.route_outages
4378 .operator_action_ended_unrefused(&module_id);
4379 use crate::supervise::SuperviseError;
4380 let message = err.to_string();
4381 let error = match err {
4382 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
4383 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
4384 code: "swap_refused".to_string(),
4385 message,
4386 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
4387 },
4388 SuperviseError::SwapFailed {
4389 arm,
4390 candidate_exit,
4391 ..
4392 } => ErrorBody {
4393 code: "swap_failed".to_string(),
4394 message,
4395 detail: Some(serde_json::json!({
4396 "arm": arm.as_str(),
4397 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
4398 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
4399 })),
4400 },
4401 _ => ErrorBody::new(
4402 "target_unavailable",
4403 format!("failed to swap module_id '{module_id}': {message}"),
4404 ),
4405 };
4406 return Ok(vec![control_error_body_frame(&frame, error)?]);
4407 }
4408 self.route_outages
4412 .operator_action_ended_unrefused(&module_id);
4413
4414 let response = ClientControlResponse::SupervisorAck {
4415 module_id,
4416 applied: true,
4417 };
4418 Ok(vec![control_response_body_frame(
4419 &frame,
4420 &response,
4421 "ClientControlResponse::SupervisorAck",
4422 )?])
4423 }
4424
4425 async fn handle_supervisor_reload(
4426 &self,
4427 frame: Frame,
4428 module_id: String,
4429 ) -> Result<Vec<Frame>, RouterError> {
4430 let operation_lock = self.supervisor.operation_lock();
4431 let _operation_guard = operation_lock.lock().await;
4432 let Some(module) = self.supervisor.get(&module_id) else {
4433 return Ok(vec![control_error_frame(
4434 &frame,
4435 "unknown_module",
4436 format!("module_id '{module_id}' is not supervised"),
4437 )?]);
4438 };
4439
4440 self.route_outages.mark_operator_action(&module_id);
4441 if let Err(err) = module.reload().await {
4442 self.route_outages
4443 .operator_action_ended_unrefused(&module_id);
4444 let (code, message) = match err {
4445 crate::supervise::SuperviseError::Disabled { .. } => {
4446 ("module_disabled", err.to_string())
4447 }
4448 crate::supervise::SuperviseError::SwapInProgress { .. } => {
4449 ("swap_in_progress", err.to_string())
4450 }
4451 _ => (
4452 "reload_failed",
4453 format!("failed to reload module_id '{module_id}': {err}"),
4454 ),
4455 };
4456 return Ok(vec![control_error_frame(&frame, code, message)?]);
4457 }
4458
4459 let response = ClientControlResponse::SupervisorAck {
4460 module_id,
4461 applied: true,
4462 };
4463 Ok(vec![control_response_body_frame(
4464 &frame,
4465 &response,
4466 "ClientControlResponse::SupervisorAck",
4467 )?])
4468 }
4469
4470 async fn handle_supervisor_rescan(
4471 &self,
4472 frame: Frame,
4473 preview: bool,
4474 ) -> Result<Vec<Frame>, RouterError> {
4475 let Some(context) = self.rescan.clone() else {
4476 return Ok(vec![control_error_frame(
4477 &frame,
4478 "rescan_unavailable",
4479 "the daemon was not started with a reloadable config path".to_string(),
4480 )?]);
4481 };
4482
4483 let operation_lock = self.supervisor.operation_lock();
4484 let _operation_guard = operation_lock.lock().await;
4485 let loaded = match crate::daemon_config::load(&context.config_path) {
4486 Ok(config) => config,
4487 Err(err) => {
4488 return Ok(vec![control_error_frame(
4489 &frame,
4490 "invalid_daemon_config",
4491 format!("supervisor rescan rejected daemon config: {err}"),
4492 )?])
4493 }
4494 };
4495 let Some(config) = loaded else {
4503 return Ok(vec![control_error_frame(
4504 &frame,
4505 "invalid_daemon_config",
4506 format!(
4507 "daemon config not found at {}; refusing to rescan (an absent config would \
4508 retire every supervised module)",
4509 context.config_path.display()
4510 ),
4511 )?]);
4512 };
4513 let (
4514 configured_port,
4515 storage_config,
4516 admission_facts_carrier_module_id,
4517 admission_facts_targets,
4518 scope_authority_owners,
4519 modules,
4520 reserved_capabilities,
4521 ) = (
4522 config.port,
4523 config.storage,
4524 config.admission_facts_carrier_module_id,
4525 config.admission_facts_targets,
4526 config.scope_authority_owners,
4527 config.modules,
4528 config.reserved_capabilities,
4529 );
4530
4531 let mut restart_required = Vec::new();
4540 for section in RestartRequiredSection::ALL {
4541 let changed = match section {
4542 RestartRequiredSection::Port => configured_port != context.configured_port,
4543 RestartRequiredSection::Storage => storage_config != context.storage_config,
4544 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
4545 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
4546 }
4547 RestartRequiredSection::AdmissionFactsTargets => {
4548 admission_facts_targets != context.admission_facts_targets
4549 }
4550 RestartRequiredSection::ScopeAuthorityOwners => {
4551 scope_authority_owners != context.scope_authority_owners
4552 }
4553 };
4554 if changed {
4555 restart_required.push(section.label().to_string());
4556 }
4557 }
4558 if !restart_required.is_empty() {
4559 warn!(
4560 config_path = %context.config_path.display(),
4561 sections = %restart_required.join(", "),
4562 "daemon config changed outside the modules section; restart the daemon to apply those changes"
4563 );
4564 }
4565
4566 for configured in &modules {
4567 if let Err(err) = validate_spec(&configured.module_spec()) {
4568 return Ok(vec![control_error_frame(
4569 &frame,
4570 "invalid_daemon_config",
4571 format!("supervisor rescan rejected daemon config: {err}"),
4572 )?]);
4573 }
4574 }
4575
4576 let configured_capabilities = modules
4577 .iter()
4578 .map(|module| (module.module_id.clone(), module.enabled))
4579 .collect::<Vec<_>>();
4580 let preview_capability_warnings = if preview {
4581 let (_, registrations) = self.runtime_capability_snapshot()?;
4582 let current_modules = self
4583 .supervisor
4584 .list()
4585 .into_iter()
4586 .map(|module| module.module_id().to_string())
4587 .collect::<BTreeSet<_>>();
4588 let resulting_modules = configured_capabilities.clone();
4589 let removed = current_modules
4590 .into_iter()
4591 .filter(|module_id| {
4592 !resulting_modules
4593 .iter()
4594 .any(|(configured_id, _)| configured_id == module_id)
4595 })
4596 .collect::<Vec<_>>();
4597 self.capability_evaluator.preview_removal_warnings(
4598 resulting_modules,
4599 &removed,
4600 ®istrations,
4601 )
4602 } else {
4603 Vec::new()
4604 };
4605 let result = match self
4606 .reconcile_supervised_modules(&context.supervisor, modules, preview)
4607 .await
4608 {
4609 Ok(result) => result,
4610 Err(message) => {
4611 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
4612 }
4613 };
4614 if !preview {
4615 self.capability_evaluator
4616 .configure(configured_capabilities, reserved_capabilities);
4617 self.capability_evaluator.wake_deadline_loop();
4618 self.refresh_capability_requirements();
4619 }
4620 let mut result = result;
4621 result.restart_required = restart_required;
4622 result.capability_warnings = preview_capability_warnings;
4623 let response = ClientControlResponse::SupervisorRescan { result };
4624 Ok(vec![control_response_body_frame(
4625 &frame,
4626 &response,
4627 "ClientControlResponse::SupervisorRescan",
4628 )?])
4629 }
4630
4631 async fn handle_supervisor_release_reserved(
4632 &self,
4633 frame: Frame,
4634 module_id: String,
4635 ) -> Result<Vec<Frame>, RouterError> {
4636 let Some(context) = self.rescan.clone() else {
4637 return Ok(vec![control_error_frame(
4638 &frame,
4639 "release_unavailable",
4640 "reserved-id release requires a daemon started with a reloadable config path",
4641 )?]);
4642 };
4643 let operation_lock = self.supervisor.operation_lock();
4644 let _operation_guard = operation_lock.lock().await;
4645 let loaded = match crate::daemon_config::load(&context.config_path) {
4646 Ok(Some(config)) => config,
4647 Ok(None) => {
4648 return Ok(vec![control_error_frame(
4649 &frame,
4650 "invalid_daemon_config",
4651 format!(
4652 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
4653 context.config_path.display()
4654 ),
4655 )?])
4656 }
4657 Err(err) => {
4658 return Ok(vec![control_error_frame(
4659 &frame,
4660 "invalid_daemon_config",
4661 format!("unable to verify reserved-id release against daemon config: {err}"),
4662 )?])
4663 }
4664 };
4665 if loaded
4666 .modules
4667 .iter()
4668 .any(|configured| configured.module_id == module_id)
4669 {
4670 return Ok(vec![control_error_frame(
4671 &frame,
4672 "reserved_module_configured",
4673 format!(
4674 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
4675 ),
4676 )?]);
4677 }
4678 if !self.supervisor.release_retained_reserved_gate(&module_id) {
4679 return Ok(vec![control_error_frame(
4680 &frame,
4681 "reserved_gate_not_retained",
4682 format!(
4683 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
4684 ),
4685 )?]);
4686 }
4687
4688 let response = ClientControlResponse::SupervisorAck {
4689 module_id,
4690 applied: true,
4691 };
4692 Ok(vec![control_response_body_frame(
4693 &frame,
4694 &response,
4695 "ClientControlResponse::SupervisorAck",
4696 )?])
4697 }
4698
4699 async fn reconcile_supervised_modules(
4708 &self,
4709 supervisor: &Supervisor,
4710 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
4711 preview: bool,
4712 ) -> Result<SupervisorRescanResult, String> {
4713 let mut current = BTreeMap::new();
4714 for module in self.supervisor.list() {
4715 let (spec, health) = module.configuration().map_err(|err| {
4716 format!(
4717 "failed to read configuration for module_id '{}': {err}",
4718 module.module_id()
4719 )
4720 })?;
4721 let enabled = module
4722 .status()
4723 .map_err(|err| {
4724 format!(
4725 "failed to read status for module_id '{}': {err}",
4726 module.module_id()
4727 )
4728 })?
4729 .enabled;
4730 current.insert(
4731 module.module_id().to_string(),
4732 (module, spec, health, enabled),
4733 );
4734 }
4735 let configured = configured_modules
4736 .into_iter()
4737 .map(|module| (module.module_id.clone(), module))
4738 .collect::<BTreeMap<_, _>>();
4739
4740 let added = configured
4741 .keys()
4742 .filter(|module_id| !current.contains_key(*module_id))
4743 .cloned()
4744 .collect::<Vec<_>>();
4745 let removed = current
4746 .keys()
4747 .filter(|module_id| !configured.contains_key(*module_id))
4748 .cloned()
4749 .collect::<Vec<_>>();
4750 let mut changed_pending_reload = Vec::new();
4751 let mut configuration_changes = BTreeSet::new();
4752 let mut enabled_changes = BTreeSet::new();
4753 let mut unchanged = 0_u32;
4754
4755 for (module_id, configured_module) in &configured {
4756 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
4757 else {
4758 continue;
4759 };
4760 let launch_changed = *current_spec != configured_module.module_spec();
4764 let configuration_changed =
4765 launch_changed || *current_health != configured_module.health;
4766 let enabled_changed = *current_enabled != configured_module.enabled;
4767 if configuration_changed {
4768 configuration_changes.insert(module_id.clone());
4769 }
4770 if launch_changed {
4771 changed_pending_reload.push(module_id.clone());
4772 }
4773 if enabled_changed {
4774 enabled_changes.insert(module_id.clone());
4775 }
4776 if !configuration_changed && !enabled_changed {
4777 unchanged = unchanged.saturating_add(1);
4778 }
4779 }
4780
4781 if preview {
4787 return Ok(SupervisorRescanResult {
4788 added,
4789 removed,
4790 changed_pending_reload,
4791 enabled_changes: enabled_changes.iter().cloned().collect(),
4792 unchanged,
4793 preview: true,
4794 restart_required: Vec::new(),
4798 capability_warnings: Vec::new(),
4799 });
4800 }
4801
4802 for module_id in &removed {
4803 let module = ¤t
4804 .get(module_id)
4805 .expect("removed module came from current supervisor state")
4806 .0;
4807 module.retire().await.map_err(|err| {
4808 format!("failed to retire module_id '{module_id}' during rescan: {err}")
4809 })?;
4810 self.supervisor.record_rescan_removal(module_id);
4844 self.supervisor.retire(module_id);
4845 self.route_outages.forget(module_id);
4846 }
4847
4848 for module_id in configured.keys() {
4849 let Some((module, _, _, _)) = current.get(module_id) else {
4850 continue;
4851 };
4852 let configured_module = configured
4853 .get(module_id)
4854 .expect("configured module id came from configured map");
4855 if configuration_changes.contains(module_id) {
4856 module
4857 .update_configuration(
4858 configured_module.module_spec(),
4859 configured_module.health.clone(),
4860 configured_module.drain_timeout_ms,
4861 )
4862 .await
4863 .map_err(|err| {
4864 format!(
4865 "failed to update module_id '{module_id}' configuration during rescan: {err}"
4866 )
4867 })?;
4868 }
4869 if enabled_changes.contains(module_id) {
4870 self.route_outages.mark_operator_action(module_id);
4873 module
4874 .set_enabled(configured_module.enabled)
4875 .await
4876 .map_err(|err| {
4877 self.route_outages.operator_action_ended_unrefused(module_id);
4878 format!(
4879 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
4880 configured_module.enabled
4881 )
4882 })?;
4883 }
4884 }
4885
4886 for module_id in &added {
4887 let configured_module = configured
4888 .get(module_id)
4889 .expect("added module id came from configured map");
4890 supervisor
4891 .supervise_configured_with_health(
4892 configured_module.module_spec(),
4893 configured_module.enabled,
4894 configured_module.health.clone(),
4895 configured_module.drain_timeout_ms,
4896 configured_module.restart,
4897 )
4898 .map_err(|err| {
4899 format!("failed to add module_id '{module_id}' during rescan: {err}")
4900 })?;
4901 }
4902
4903 Ok(SupervisorRescanResult {
4904 added,
4905 removed,
4906 changed_pending_reload,
4907 enabled_changes: enabled_changes.iter().cloned().collect(),
4908 unchanged,
4909 preview: false,
4910 restart_required: Vec::new(),
4913 capability_warnings: Vec::new(),
4914 })
4915 }
4916
4917 async fn handle_supervisor_set_enabled(
4918 &self,
4919 frame: Frame,
4920 module_id: String,
4921 enabled: bool,
4922 ) -> Result<Vec<Frame>, RouterError> {
4923 let operation_lock = self.supervisor.operation_lock();
4924 let _operation_guard = operation_lock.lock().await;
4925 let Some(module) = self.supervisor.get(&module_id) else {
4926 return Ok(vec![control_error_frame(
4927 &frame,
4928 "unknown_module",
4929 format!("module_id '{module_id}' is not supervised"),
4930 )?]);
4931 };
4932
4933 self.route_outages.mark_operator_action(&module_id);
4936 let applied = match module.set_enabled(enabled).await {
4937 Ok(applied) => applied,
4938 Err(err) => {
4939 self.route_outages
4940 .operator_action_ended_unrefused(&module_id);
4941 return Ok(vec![control_error_frame(
4942 &frame,
4943 "target_unavailable",
4944 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
4945 )?]);
4946 }
4947 };
4948 if !applied {
4949 self.route_outages
4952 .operator_action_ended_unrefused(&module_id);
4953 }
4954
4955 self.capability_evaluator.wake_deadline_loop();
4956 self.refresh_capability_requirements();
4957 let response = ClientControlResponse::SupervisorAck { module_id, applied };
4958 Ok(vec![control_response_body_frame(
4959 &frame,
4960 &response,
4961 "ClientControlResponse::SupervisorAck",
4962 )?])
4963 }
4964
4965 async fn handle_supervisor_health_probe(
4966 &self,
4967 frame: Frame,
4968 module_id: String,
4969 ) -> Result<Vec<Frame>, RouterError> {
4970 self.refresh_capability_requirements();
4971 let Some(registration) = self
4972 .registry
4973 .get_module(&module_id)
4974 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4975 else {
4976 return Ok(vec![control_error_frame(
4977 &frame,
4978 "unknown_module",
4979 format!("module_id '{module_id}' is not registered"),
4980 )?]);
4981 };
4982
4983 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
4996 {
4997 return Ok(vec![control_error_frame(
4998 &frame,
4999 "health_not_advertised",
5000 format!("module_id '{module_id}' did not advertise health.check"),
5001 )?]);
5002 }
5003
5004 let deadline = Instant::now() + self.health_probe_timeout;
5005 let pending = match self.forwarding.begin_module_control_rpc_for(
5006 &module_id,
5007 MODULE_CONTROL_OP_HEALTH_CHECK,
5008 deadline,
5009 ) {
5010 Ok(pending) => pending,
5011 Err(err) => {
5012 return Ok(vec![control_error_frame(
5013 &frame,
5014 forwarding_error_code(&err),
5015 err.to_string(),
5016 )?])
5017 }
5018 };
5019
5020 let PendingModuleControlRpc {
5021 endpoint,
5022 module_sink,
5023 negotiated_ver,
5024 corr: probe_corr,
5025 receiver,
5026 } = pending;
5027 let mut guard =
5028 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
5029 let probe_body =
5030 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5031 RouterError::backend(
5032 0,
5033 frame.header.corr,
5034 format!("failed to encode health.check request: {err}"),
5035 )
5036 })?;
5037 let probe_frame = Frame::build_with_version(
5038 negotiated_ver,
5039 FrameType::Request,
5040 control_flags(),
5041 0,
5042 0,
5043 probe_corr,
5044 probe_body,
5045 )
5046 .map_err(RouterError::FrameBuild)?;
5047
5048 if let Err(err) = module_sink.send(probe_frame).await {
5049 return Ok(vec![control_error_frame(
5050 &frame,
5051 "target_unavailable",
5052 err.to_string(),
5053 )?]);
5054 }
5055
5056 match timeout_at(deadline, receiver).await {
5057 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
5058 guard.disarm();
5059 let Some(report) = response.health_report() else {
5060 return Ok(vec![control_error_frame(
5061 &frame,
5062 "invalid_control_body",
5063 "health.check RPC returned a non-health response",
5064 )?]);
5065 };
5066 let HealthReport {
5071 status,
5072 detail,
5073 metrics,
5074 } = report;
5075 let capability_detail = self
5076 .capability_evaluator
5077 .required_problem_detail(&module_id);
5078 let response = ClientControlResponse::SupervisorHealthProbe {
5079 module_id,
5080 status,
5081 detail: append_capability_problem_detail(detail, capability_detail),
5082 metrics,
5083 };
5084 Ok(vec![control_response_body_frame(
5085 &frame,
5086 &response,
5087 "ClientControlResponse::SupervisorHealthProbe",
5088 )?])
5089 }
5090 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
5091 guard.disarm();
5092 Ok(vec![control_error_body_frame(&frame, body)?])
5093 }
5094 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5095 guard.disarm();
5096 Ok(vec![control_error_frame(
5097 &frame,
5098 "target_unavailable",
5099 message,
5100 )?])
5101 }
5102 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5103 guard.disarm();
5104 Ok(vec![control_error_frame(
5105 &frame,
5106 "invalid_control_body",
5107 message,
5108 )?])
5109 }
5110 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5111 guard.disarm();
5112 Ok(vec![control_error_frame(
5113 &frame,
5114 "invalid_control_body",
5115 format!("expected module-control op '{expected}', got '{actual}'"),
5116 )?])
5117 }
5118 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
5119 guard.disarm();
5120 Ok(vec![control_error_frame(
5121 &frame,
5122 "module_timeout",
5123 format!(
5124 "module_id '{module_id}' answered health.check after {:?}",
5125 self.health_probe_timeout
5126 ),
5127 )?])
5128 }
5129 Ok(Err(_)) => Ok(vec![control_error_frame(
5130 &frame,
5131 "target_unavailable",
5132 "health.check waiter was canceled before the module responded",
5133 )?]),
5134 Err(_) => Ok(vec![control_error_frame(
5135 &frame,
5136 "module_timeout",
5137 format!(
5138 "module_id '{module_id}' did not answer health.check within {:?}",
5139 self.health_probe_timeout
5140 ),
5141 )?]),
5142 }
5143 }
5144
5145 fn supervisor_status(
5146 &self,
5147 module_id: &str,
5148 corr: u64,
5149 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
5150 self.supervisor
5151 .get(module_id)
5152 .map(|module| {
5153 let warming = module.is_warming_for_control("status").map_err(|err| {
5154 RouterError::backend(
5155 0,
5156 corr,
5157 format!(
5158 "failed to read supervisor warming state for module_id '{module_id}': {err}"
5159 ),
5160 )
5161 })?;
5162 module.status_for_control("status").map_err(|err| {
5163 RouterError::backend(
5164 0,
5165 corr,
5166 format!(
5167 "failed to read supervisor status for module_id '{module_id}': {err}"
5168 ),
5169 )
5170 }).map(|status| (status, warming))
5171 })
5172 .transpose()
5173 }
5174
5175 fn guard_module_control_op(
5176 &self,
5177 frame: &Frame,
5178 module_id: &str,
5179 op: &str,
5180 ) -> Result<Option<Frame>, RouterError> {
5181 if self.module_grants_op(module_id, op, frame.header.corr)? {
5182 return Ok(None);
5183 }
5184
5185 Ok(Some(control_error_frame(
5186 frame,
5187 "op_not_allowed",
5188 format!("module_id '{module_id}' did not grant control op '{op}'"),
5189 )?))
5190 }
5191
5192 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
5193 let Some(registration) = self
5194 .registry
5195 .get_module(module_id)
5196 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
5197 else {
5198 return Ok(false);
5199 };
5200 Ok(module_registration_grants_op(®istration.control_ops, op))
5201 }
5202
5203 fn handle_status_update(
5204 &self,
5205 endpoint: ModuleEndpointId,
5206 frame: Frame,
5207 ) -> Result<Vec<Frame>, RouterError> {
5208 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
5209 Ok(update) => update,
5210 Err(err) => {
5211 if is_known_module_push_op(&frame.body) {
5216 return Ok(vec![control_error_frame(
5217 &frame,
5218 "invalid_control_body",
5219 format!("malformed module control push body: {err}"),
5220 )?]);
5221 }
5222 return Ok(Vec::new());
5223 }
5224 };
5225
5226 match update {
5227 ModuleControlPush::RouteStatus {
5228 route_channel,
5229 route_epoch,
5230 status,
5231 } => {
5232 self.forwarding
5233 .cache_status(endpoint, route_channel, route_epoch, status)
5234 .map_err(RouterError::Forwarding)?;
5235 }
5236 }
5237 Ok(Vec::new())
5238 }
5239
5240 fn handle_route_poll(
5241 &self,
5242 ctx: &RouteCtx,
5243 frame: Frame,
5244 route_channel: u16,
5245 route_epoch: u32,
5246 kind: PollKind,
5247 ) -> Result<Vec<Frame>, RouterError> {
5248 let snapshot = self
5249 .forwarding
5250 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
5251 .map_err(RouterError::Forwarding)?;
5252 let response = match (kind, snapshot) {
5253 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
5254 ClientControlResponse::RoutePoll {
5255 route_channel,
5256 route_epoch,
5257 status,
5258 live: None,
5259 }
5260 }
5261 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
5262 route_channel,
5263 route_epoch,
5264 status: None,
5265 live: None,
5266 },
5267 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
5268 let live = self
5294 .process_liveness
5295 .as_ref()
5296 .and_then(|source| source.process_live(&module_id))
5297 .unwrap_or(true);
5298 ClientControlResponse::RoutePoll {
5299 route_channel,
5300 route_epoch,
5301 status: None,
5302 live: Some(live),
5303 }
5304 }
5305 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
5306 route_channel,
5307 route_epoch,
5308 status: None,
5309 live: Some(false),
5310 },
5311 };
5312
5313 Ok(vec![control_response_body_frame(
5314 &frame,
5315 &response,
5316 "ClientControlResponse::RoutePoll",
5317 )?])
5318 }
5319
5320 pub(crate) fn observe_module_control_completion(
5321 &self,
5322 completion: ModuleControlRpcCompletion,
5323 ) -> bool {
5324 match completion {
5325 ModuleControlRpcCompletion::Unknown => false,
5326 ModuleControlRpcCompletion::Settled => true,
5327 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
5328 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
5329 info!(
5330 module_id = %module_id,
5331 latency_ms,
5332 "late health.check answer proves the module is alive"
5333 );
5334 match self
5335 .supervisor
5336 .record_late_health_answer(&module_id, latency_ms)
5337 {
5338 Ok(true) => {}
5339 Ok(false) => debug!(
5340 module_id = %module_id,
5341 latency_ms,
5342 "late health.check answer has no active supervisor snapshot"
5343 ),
5344 Err(err) => warn!(
5345 module_id = %module_id,
5346 latency_ms,
5347 error = %err,
5348 "failed to record late health.check answer"
5349 ),
5350 }
5351 true
5352 }
5353 }
5354 }
5355
5356 fn refuse_to_end_module_connection_for_a_client(
5377 &self,
5378 module_connection_id: ConnectionId,
5379 corr: u64,
5380 err: ForwardingError,
5381 ) -> Result<(), RouterError> {
5382 if let ForwardingError::ConnectionClosing { connection_id } = err {
5383 if connection_id != module_connection_id {
5384 warn!(
5385 module_connection_id = module_connection_id.get(),
5386 client_connection_id = connection_id.get(),
5387 corr,
5388 "dropping a route.bind response for a closing client; the module connection keeps serving"
5389 );
5390 return Ok(());
5391 }
5392 }
5393 Err(RouterError::Forwarding(err))
5394 }
5395
5396 fn handle_module_relay_response(
5397 &self,
5398 connection_id: ConnectionId,
5399 frame: Frame,
5400 ) -> Result<Vec<Frame>, RouterError> {
5401 let mut secondary_error = None;
5402 let outcome = match frame.header.ty {
5403 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
5404 Ok(probe) if probe.op == "route.bind" => {
5405 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
5406 Ok(ModuleControlResponse::RouteBindAck {}) => {
5407 RouteBindRelayOutcome::Accepted
5408 }
5409 Ok(other) => {
5410 let message =
5411 format!("route.bind response carried unexpected body: {other:?}");
5412 secondary_error = Some(control_error_frame(
5413 &frame,
5414 "invalid_control_body",
5415 message.clone(),
5416 )?);
5417 RouteBindRelayOutcome::ModuleGone(message)
5418 }
5419 Err(err) => {
5420 let message = format!("malformed route.bind response body: {err}");
5421 secondary_error = Some(control_error_frame(
5422 &frame,
5423 "invalid_control_body",
5424 message.clone(),
5425 )?);
5426 RouteBindRelayOutcome::ModuleGone(message)
5427 }
5428 }
5429 }
5430 Ok(probe) => {
5431 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
5432 {
5433 Ok(response) => ModuleControlRpcOutcome::Response(response),
5434 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
5435 "malformed {} response body: {err}",
5436 probe.op
5437 )),
5438 };
5439 let completion = self
5440 .forwarding
5441 .complete_module_control_rpc(
5442 connection_id,
5443 frame.header.corr,
5444 Some(&probe.op),
5445 outcome,
5446 )
5447 .map_err(RouterError::Forwarding)?;
5448 if !self.observe_module_control_completion(completion) {
5449 debug!(
5450 connection_id = connection_id.get(),
5451 corr = frame.header.corr,
5452 op = %probe.op,
5453 "dropping late or unknown module-control RPC response"
5454 );
5455 }
5456 return Ok(Vec::new());
5457 }
5458 Err(err) => {
5459 if let Some(expected_op) = self
5460 .forwarding
5461 .pending_module_control_op(connection_id, frame.header.corr)
5462 .map_err(RouterError::Forwarding)?
5463 {
5464 let completion = self
5465 .forwarding
5466 .complete_module_control_rpc(
5467 connection_id,
5468 frame.header.corr,
5469 None,
5470 ModuleControlRpcOutcome::MalformedResponse(format!(
5471 "malformed {expected_op} response body: {err}"
5472 )),
5473 )
5474 .map_err(RouterError::Forwarding)?;
5475 if !self.observe_module_control_completion(completion) {
5476 debug!(
5477 connection_id = connection_id.get(),
5478 corr = frame.header.corr,
5479 "dropping late malformed module-control RPC response"
5480 );
5481 }
5482 return Ok(Vec::new());
5483 }
5484 let message = format!("malformed route.bind response body: {err}");
5485 secondary_error = Some(control_error_frame(
5486 &frame,
5487 "invalid_control_body",
5488 message.clone(),
5489 )?);
5490 RouteBindRelayOutcome::ModuleGone(message)
5491 }
5492 },
5493 FrameType::Error => {
5494 if self
5495 .forwarding
5496 .pending_module_control_op(connection_id, frame.header.corr)
5497 .map_err(RouterError::Forwarding)?
5498 .is_some()
5499 {
5500 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
5501 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
5502 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
5503 "malformed module-control ERROR body: {err}"
5504 )),
5505 };
5506 let completion = self
5507 .forwarding
5508 .complete_module_control_rpc(
5509 connection_id,
5510 frame.header.corr,
5511 None,
5512 outcome,
5513 )
5514 .map_err(RouterError::Forwarding)?;
5515 if !self.observe_module_control_completion(completion) {
5516 debug!(
5517 connection_id = connection_id.get(),
5518 corr = frame.header.corr,
5519 "dropping late or unknown module-control RPC error"
5520 );
5521 }
5522 return Ok(Vec::new());
5523 }
5524 match serde_json::from_slice::<ErrorBody>(&frame.body) {
5525 Ok(body) => RouteBindRelayOutcome::Rejected(body),
5526 Err(err) => {
5527 let message = format!("malformed route.bind ERROR body: {err}");
5528 secondary_error = Some(control_error_frame(
5529 &frame,
5530 "invalid_control_body",
5531 message.clone(),
5532 )?);
5533 RouteBindRelayOutcome::ModuleGone(message)
5534 }
5535 }
5536 }
5537 ty => {
5538 return Ok(vec![control_error_frame(
5539 &frame,
5540 "unsupported_control_frame",
5541 format!("unsupported module channel-0 frame {ty:?}"),
5542 )?])
5543 }
5544 };
5545
5546 let settled =
5547 self.forwarding
5548 .complete_pending_relay(connection_id, frame.header.corr, outcome);
5549 let completion = match settled {
5550 Ok(completion) => completion,
5551 Err(err) => {
5552 self.refuse_to_end_module_connection_for_a_client(
5553 connection_id,
5554 frame.header.corr,
5555 err,
5556 )?;
5557 return Ok(secondary_error.into_iter().collect());
5558 }
5559 };
5560 if let Some(target) = completion.abandoned.as_ref() {
5561 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
5562 }
5563 if !completion.settled {
5564 debug!(
5565 connection_id = connection_id.get(),
5566 corr = frame.header.corr,
5567 frame_type = ?frame.header.ty,
5568 "dropping late or unknown route.bind relay response"
5569 );
5570 }
5571 Ok(secondary_error.into_iter().collect())
5572 }
5573
5574 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
5575 debug!(connection_id = connection_id.get(), "handling GOODBYE");
5576 self.cleanup_connection_with_end_reason(
5580 connection_id,
5581 RegistrationEndReason::ExplicitGoodbye,
5582 )
5583 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
5584 Ok(Vec::new())
5585 }
5586}
5587
5588impl Default for ControlHandler {
5589 fn default() -> Self {
5590 Self::new(Arc::new(Registry::default()))
5591 }
5592}
5593
5594impl crate::supervise::SwapPromotionObserver for ControlHandler {
5595 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
5596 self.apply_registration_capabilities(registration);
5597 }
5598}
5599
5600fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
5601 CapabilityRequirementStatus {
5602 consumer: status.consumer,
5603 capability: status.capability,
5604 need: match status.need {
5605 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
5606 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
5607 },
5608 verdict: status.verdict.as_str().to_string(),
5609 episode_seq: status.episode_seq,
5610 config_satisfiable: status.config_satisfiable,
5611 runtime_available: status.runtime_available,
5612 detail: status.detail,
5613 }
5614}
5615
5616fn append_capability_problem_detail(
5617 detail: Option<String>,
5618 capability_detail: Option<String>,
5619) -> Option<String> {
5620 match (detail, capability_detail) {
5621 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
5622 (Some(detail), None) => Some(detail),
5623 (None, Some(capability_detail)) => Some(capability_detail),
5624 (None, None) => None,
5625 }
5626}
5627
5628fn subc_ops() -> Vec<String> {
5629 SUBC_CONTROL_OPS
5630 .iter()
5631 .map(|op| (*op).to_string())
5632 .collect()
5633}
5634
5635fn module_subc_ops() -> Vec<String> {
5636 SUBC_CONTROL_OPS
5637 .iter()
5638 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
5639 .map(|op| (*op).to_string())
5640 .collect()
5641}
5642
5643#[cfg(test)]
5644fn module_baseline_control_ops() -> Vec<String> {
5645 MODULE_BASELINE_CONTROL_OPS
5646 .iter()
5647 .map(|op| (*op).to_string())
5648 .collect()
5649}
5650
5651fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
5652 let mut seen = HashSet::new();
5653 let mut effective = Vec::new();
5654 for op in MODULE_BASELINE_CONTROL_OPS {
5655 if seen.insert((*op).to_string()) {
5656 effective.push((*op).to_string());
5657 }
5658 }
5659 for op in declared.unwrap_or_default() {
5660 if seen.insert(op.clone()) {
5661 effective.push(op);
5662 }
5663 }
5664 effective
5665}
5666
5667fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
5668 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
5669}
5670
5671fn target_module_id(target: &RouteTarget) -> &str {
5672 match target {
5673 RouteTarget::ToolProvider { module_id }
5674 | RouteTarget::ManagementSurface { module_id }
5675 | RouteTarget::InternalService { module_id, .. } => module_id,
5676 }
5677}
5678
5679fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
5680 roles.iter().any(|role| match (target, role) {
5681 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
5682 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
5683 (
5684 RouteTarget::InternalService { service_id, .. },
5685 ProviderRole::InternalService {
5686 service_id: provided,
5687 ..
5688 },
5689 ) => service_id == provided,
5690 _ => false,
5691 })
5692}
5693
5694fn is_routable_role(role: &ProviderRole) -> bool {
5695 matches!(
5696 role,
5697 ProviderRole::ToolProvider { .. }
5698 | ProviderRole::ManagementSurface { .. }
5699 | ProviderRole::InternalService { .. }
5700 )
5701}
5702
5703#[derive(Debug, Clone, Copy, PartialEq, Eq)]
5704enum ControlRequestBodyError {
5705 UnknownOp,
5706 InvalidBody,
5707}
5708
5709#[derive(Debug, Deserialize)]
5710struct ControlOpProbe {
5711 op: String,
5712}
5713
5714const MODULE_PUSH_OPS: &[&str] = &["route.status"];
5717
5718fn is_known_module_push_op(body: &[u8]) -> bool {
5719 serde_json::from_slice::<ControlOpProbe>(body)
5720 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
5721 .unwrap_or(false)
5722}
5723
5724fn is_known_module_request_op(body: &[u8]) -> bool {
5725 serde_json::from_slice::<ControlOpProbe>(body)
5726 .map(|probe| is_module_to_subc_op(&probe.op))
5727 .unwrap_or(false)
5728}
5729
5730fn is_module_to_subc_op(op: &str) -> bool {
5731 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
5732}
5733
5734fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
5735 debug!(
5736 op = %op,
5737 connection_id = connection_id.get(),
5738 corr,
5739 "control dispatch"
5740 );
5741}
5742
5743fn log_slow_control_dispatch(
5744 dispatch_started_at: Option<StdInstant>,
5745 op: &'static str,
5746 connection_id: ConnectionId,
5747 corr: u64,
5748) {
5749 let Some(dispatch_started_at) = dispatch_started_at else {
5750 return;
5751 };
5752 let elapsed = dispatch_started_at.elapsed();
5753 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
5754 warn!(
5755 op = %op,
5756 connection_id = connection_id.get(),
5757 corr,
5758 elapsed_ms = elapsed.as_millis() as u64,
5759 "slow control dispatch"
5760 );
5761 }
5762}
5763
5764fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
5765 match request {
5766 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
5767 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
5768 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
5769 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
5770 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
5771 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
5772 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
5773 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
5774 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
5775 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
5776 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
5777 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
5778 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
5779 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
5780 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
5781 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
5782 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
5783 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
5784 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
5785 }
5786}
5787
5788fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
5789 match request {
5790 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
5791 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
5792 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
5793 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
5794 }
5795}
5796
5797fn parse_client_control_request(
5798 body: &[u8],
5799) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
5800 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
5801 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5802 Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
5803 ControlRequestBodyError::InvalidBody
5804 }
5805 Ok(_) => ControlRequestBodyError::UnknownOp,
5806 Err(_) => ControlRequestBodyError::InvalidBody,
5807 };
5808 (err, classification)
5809 })
5810}
5811
5812fn parse_module_control_request_from_module(
5813 body: &[u8],
5814) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
5815 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
5816 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5817 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
5818 Ok(_) => ControlRequestBodyError::UnknownOp,
5819 Err(_) => ControlRequestBodyError::InvalidBody,
5820 };
5821 (err, classification)
5822 })
5823}
5824
5825#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
5826enum ProviderRoleKind {
5827 ToolProvider,
5828 PipelineStage,
5829 ManagementSurface,
5830 InternalService,
5831}
5832
5833fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
5834 match role {
5835 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
5836 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
5837 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
5838 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
5839 }
5840}
5841
5842fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
5843 roles.iter().map(provider_role_kind).collect()
5844}
5845
5846const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
5849
5850#[derive(Debug, Default, PartialEq, Eq)]
5852struct ScopeOutcomeCounts {
5853 created: usize,
5854 replaced: usize,
5855 updated: usize,
5856 unchanged: usize,
5857 refused: usize,
5858}
5859
5860impl ScopeOutcomeCounts {
5861 fn of(results: &[ScopeRecordResult]) -> Self {
5862 let mut counts = Self::default();
5863 for result in results {
5864 let slot = match result.outcome {
5865 ScopeRecordOutcome::Created => &mut counts.created,
5866 ScopeRecordOutcome::Replaced => &mut counts.replaced,
5867 ScopeRecordOutcome::Updated => &mut counts.updated,
5868 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
5869 ScopeRecordOutcome::Refused => &mut counts.refused,
5870 };
5871 *slot += 1;
5872 }
5873 counts
5874 }
5875}
5876
5877#[cfg(test)]
5878mod scope_outcome_count_tests {
5879 use super::*;
5880
5881 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
5882 ScopeRecordResult {
5883 scope_ref: "r".to_string(),
5884 scope_epoch: 1,
5885 outcome,
5886 code: None,
5887 message: None,
5888 version: None,
5889 parent_state: None,
5890 }
5891 }
5892
5893 #[test]
5896 fn every_outcome_is_counted_in_its_own_field() {
5897 let results = [
5898 result(ScopeRecordOutcome::Created),
5899 result(ScopeRecordOutcome::Created),
5900 result(ScopeRecordOutcome::Replaced),
5901 result(ScopeRecordOutcome::Updated),
5902 result(ScopeRecordOutcome::Unchanged),
5903 result(ScopeRecordOutcome::Refused),
5904 result(ScopeRecordOutcome::Refused),
5905 result(ScopeRecordOutcome::Refused),
5906 ];
5907 assert_eq!(
5908 ScopeOutcomeCounts::of(&results),
5909 ScopeOutcomeCounts {
5910 created: 2,
5911 replaced: 1,
5912 updated: 1,
5913 unchanged: 1,
5914 refused: 3,
5915 }
5916 );
5917 }
5918}
5919
5920fn capability_census_trigger(
5924 old: Option<&CapabilityDeclarations>,
5925 new: Option<&CapabilityDeclarations>,
5926) -> bool {
5927 let old_provides = old
5928 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
5929 .unwrap_or_default();
5930 let old_denies = old
5931 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
5932 .unwrap_or_default();
5933 let new = new.cloned().unwrap_or(CapabilityDeclarations {
5934 provides: Vec::new(),
5935 requires: Vec::new(),
5936 must_never_reach: Vec::new(),
5937 });
5938
5939 new.provides
5940 .iter()
5941 .any(|capability| !old_provides.contains(capability))
5942 || new
5943 .must_never_reach
5944 .iter()
5945 .any(|capability| !old_denies.contains(capability))
5946}
5947
5948fn denied_capability<'a>(
5951 opening_manifest: &'a ModuleManifest,
5952 target_manifest: &ModuleManifest,
5953) -> Option<&'a str> {
5954 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
5955 let target_capabilities = target_manifest.capabilities.as_ref()?;
5956 opening_capabilities
5957 .must_never_reach
5958 .iter()
5959 .find(|denied| {
5960 target_capabilities
5961 .provides
5962 .iter()
5963 .any(|provided| provided == *denied)
5964 })
5965 .map(String::as_str)
5966}
5967
5968fn catalog_update_frozen_field_message(
5969 registered: &ModuleManifest,
5970 provides: &[ProviderRole],
5971) -> Option<String> {
5972 let old_has_provides = !registered.provides.is_empty();
5973 let new_has_provides = !provides.is_empty();
5974 if old_has_provides != new_has_provides {
5975 return Some(format!(
5976 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
5977 registered.module_id
5978 ));
5979 }
5980
5981 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
5982 return Some(format!(
5983 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
5984 registered.module_id
5985 ));
5986 }
5987
5988 let registered_concurrency = manifest_concurrency(registered);
5989 let mut candidate = registered.clone();
5990 candidate.provides = provides.to_vec();
5991 let candidate_concurrency = manifest_concurrency(&candidate);
5992 if candidate_concurrency != registered_concurrency {
5993 return Some(format!(
5994 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
5995 registered.module_id, registered_concurrency, candidate_concurrency
5996 ));
5997 }
5998
5999 None
6002}
6003
6004fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
6005 manifest.provides.iter().any(is_routable_role)
6006}
6007
6008fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
6014 manifest
6015 .provides
6016 .iter()
6017 .find_map(|provider| match provider {
6018 ProviderRole::ToolProvider { concurrency, .. }
6019 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
6020 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
6021 })
6022 .unwrap_or(Concurrency::ModuleManaged)
6023}
6024
6025fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
6032 let has_management_surface = manifest
6033 .provides
6034 .iter()
6035 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
6036 if !has_management_surface {
6037 return false;
6038 }
6039 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
6040 return false;
6041 };
6042 let Some(provides) = raw
6043 .get("manifest")
6044 .and_then(|manifest| manifest.get("provides"))
6045 .and_then(serde_json::Value::as_array)
6046 else {
6047 return false;
6048 };
6049 provides.iter().any(|role| {
6054 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
6055 && role.get("concurrency").is_none()
6056 })
6057}
6058
6059fn negotiate_version(peer_version: u8) -> Result<u8, String> {
6060 if peer_version != PROTOCOL_VERSION {
6061 return Err(format!(
6062 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
6063 ));
6064 }
6065 Ok(PROTOCOL_VERSION)
6066}
6067
6068fn pong(frame: &Frame) -> Result<Frame, RouterError> {
6069 Frame::build_with_version(
6070 response_version(frame),
6071 FrameType::Pong,
6072 frame.header.flags,
6073 0,
6074 0,
6075 frame.header.corr,
6076 Vec::new(),
6077 )
6078 .map_err(RouterError::FrameBuild)
6079}
6080
6081fn control_error_frame(
6082 frame: &Frame,
6083 code: &'static str,
6084 message: impl Into<String>,
6085) -> Result<Frame, RouterError> {
6086 control_error_body_frame(
6087 frame,
6088 ErrorBody {
6089 code: code.to_string(),
6090 message: message.into(),
6091 detail: None,
6092 },
6093 )
6094}
6095
6096fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
6097 let body = serde_json::to_vec(&error).map_err(|err| {
6098 RouterError::backend(
6099 0,
6100 frame.header.corr,
6101 format!("failed to encode control ERROR: {err}"),
6102 )
6103 })?;
6104
6105 Frame::build_with_version(
6106 response_version(frame),
6107 FrameType::Error,
6108 control_flags(),
6109 0,
6110 0,
6111 frame.header.corr,
6112 body,
6113 )
6114 .map_err(RouterError::FrameBuild)
6115}
6116
6117fn control_response_body_frame<T: Serialize>(
6118 frame: &Frame,
6119 reply: &T,
6120 label: &'static str,
6121) -> Result<Frame, RouterError> {
6122 let body = serde_json::to_vec(reply).map_err(|err| {
6123 RouterError::backend(
6124 0,
6125 frame.header.corr,
6126 format!("failed to encode {label}: {err}"),
6127 )
6128 })?;
6129
6130 Frame::build_with_version(
6131 response_version(frame),
6132 FrameType::Response,
6133 control_flags(),
6134 0,
6135 0,
6136 frame.header.corr,
6137 body,
6138 )
6139 .map_err(RouterError::FrameBuild)
6140}
6141
6142fn forwarding_error_code(err: &ForwardingError) -> &'static str {
6172 match err {
6173 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
6174 ForwardingError::NoModuleConnection => "target_unavailable",
6175 ForwardingError::ModuleReloading { .. } => "module_reloading",
6176 ForwardingError::ClientRouteChannelExhausted { .. }
6177 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
6178 ForwardingError::StaleModuleEndpoint
6179 | ForwardingError::UnknownReservation { .. }
6180 | ForwardingError::ConnectionClosing { .. }
6181 | ForwardingError::ClientEgressClosed { .. }
6182 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
6183 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
6186 ForwardingError::RelayCorrelationExhausted
6187 | ForwardingError::RouteOpenBuild(_)
6188 | ForwardingError::Poisoned => "forwarding_error",
6189 }
6190}
6191
6192fn response_version(frame: &Frame) -> u8 {
6193 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
6194 frame.header.ver
6195 } else {
6196 PROTOCOL_VERSION
6197 }
6198}
6199
6200fn control_flags() -> Flags {
6201 Flags::new(false, Priority::Passive, false)
6202}
6203
6204fn send_goodbye_target_best_effort(
6209 counters: &DaemonCounters,
6210 target: &GoodbyeTarget,
6211 context: &'static str,
6212) {
6213 let Ok(frame) = Frame::build_with_version(
6214 target.negotiated_ver,
6215 FrameType::Goodbye,
6216 control_flags(),
6217 target.channel,
6218 target.epoch,
6219 0,
6220 Vec::new(),
6221 ) else {
6222 return;
6223 };
6224 crate::forwarding::send_module_route_goodbye(
6225 counters,
6226 &target.sink,
6227 frame,
6228 target.module_id.as_deref(),
6229 context,
6230 );
6231}
6232
6233pub(crate) fn send_route_control_pushes(
6234 forwarding: &ForwardingTable,
6235 routes: Vec<EndpointRoute>,
6236 push: ClientControlPush,
6237) {
6238 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
6239 for route in routes {
6240 let target = route.goodbye_target;
6241 if let Some((existing, channels)) = targets
6242 .iter_mut()
6243 .find(|(existing, _)| existing.connection_id == target.connection_id)
6244 {
6245 debug_assert_eq!(
6246 existing.negotiated_ver, target.negotiated_ver,
6247 "one connection cannot negotiate multiple frame versions"
6248 );
6249 if !channels.contains(&target.channel) {
6250 channels.push(target.channel);
6251 }
6252 continue;
6253 }
6254 let channel = target.channel;
6255 targets.push((target, vec![channel]));
6256 }
6257 for (target, mut channels) in targets {
6258 channels.sort_unstable();
6259 let mut push = push.clone();
6260 match &mut push {
6261 ClientControlPush::RouteClosing {
6262 channels: covered, ..
6263 }
6264 | ClientControlPush::RouteClosed {
6265 channels: covered, ..
6266 } => *covered = channels,
6267 }
6268 let body = match serde_json::to_vec(&push) {
6269 Ok(body) => body,
6270 Err(err) => {
6271 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
6272 continue;
6273 }
6274 };
6275 let frame = match Frame::build_with_version(
6276 target.negotiated_ver,
6277 FrameType::Push,
6278 control_flags(),
6279 0,
6280 0,
6281 0,
6282 body.clone(),
6283 ) {
6284 Ok(frame) => frame,
6285 Err(err) => {
6286 warn!(
6287 route_channel = target.channel,
6288 error = %err,
6289 "failed to build route lifecycle control PUSH frame"
6290 );
6291 continue;
6292 }
6293 };
6294 if let Err(err) = target.sink.try_send(frame) {
6295 if target.close_on_delivery_failure() {
6296 warn!(
6297 target_connection_id = target.connection_id.get(),
6298 route_channel = target.channel,
6299 error = %err,
6300 "route lifecycle control PUSH was not delivered to client; closing target connection"
6301 );
6302 let _ = forwarding.escalate_client_delivery_failure(
6303 target.connection_id,
6304 target.channel,
6305 target.epoch,
6306 CloseReason::new(
6307 "route_lifecycle_push_delivery_failed",
6308 format!(
6309 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
6310 target.channel
6311 ),
6312 ),
6313 crate::forwarding::UndeliveredFrame {
6314 module_id: target.module_id.as_deref(),
6315 sink: &target.sink,
6316 },
6317 );
6318 }
6319 }
6320 }
6321}
6322
6323#[cfg(test)]
6324mod tests {
6325 #[cfg(unix)]
6326 #[tokio::test]
6327 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
6328 let dir = subc_test_support::TestTempDir::new("rescan-live-health");
6329 let path = dir.join("subc.jsonc");
6330 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
6331 "program":"/bin/sleep","args":["60"],"protocol":"none",
6332 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
6333 }}}).to_string()).unwrap();
6334 let mut configured = crate::daemon_config::load(&path)
6335 .unwrap()
6336 .unwrap()
6337 .modules
6338 .pop()
6339 .unwrap();
6340 let registry = std::sync::Arc::new(crate::Registry::default());
6341 let handle = crate::SupervisorHandle::new();
6342 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
6343 .with_handle(handle.clone());
6344 let module = supervisor
6345 .supervise_configured_with_health(
6346 configured.module_spec(),
6347 true,
6348 configured.health.clone(),
6349 None,
6350 configured.restart,
6351 )
6352 .unwrap();
6353 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
6354 let before = module.status().unwrap().pid;
6355 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
6356 configured.health.cadence = std::time::Duration::from_secs(3600);
6357 let health_only = handler
6358 .reconcile_supervised_modules(&supervisor, vec![configured.clone()], false)
6359 .await
6360 .unwrap();
6361 assert!(
6362 health_only.changed_pending_reload.is_empty(),
6363 "health policy is already applied live"
6364 );
6365 assert_eq!(module.status().unwrap().pid, before);
6366 assert_eq!(
6367 module.configuration().unwrap().1.http,
6368 configured.health.http
6369 );
6370 configured.args = vec!["61".into()];
6371 let launch = handler
6372 .reconcile_supervised_modules(&supervisor, vec![configured], false)
6373 .await
6374 .unwrap();
6375 assert_eq!(launch.changed_pending_reload, ["stock"]);
6376 assert_eq!(
6377 module.status().unwrap().pid,
6378 before,
6379 "a launch edit is stored until reload"
6380 );
6381 module.drain().await.unwrap();
6382 }
6383 use std::{
6384 collections::BTreeMap,
6385 fmt,
6386 path::PathBuf,
6387 sync::{Arc, Mutex},
6388 time::Duration,
6389 };
6390 use subc_test_support::TestTempDir;
6391
6392 use serde_json::{json, Value};
6393 use subc_protocol::{
6394 manifest::{
6395 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
6396 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
6397 },
6398 session::HealthStatus,
6399 FrameType,
6400 };
6401
6402 use super::*;
6403 use crate::{
6404 forwarding::{DataRoute, DataRouteState},
6405 registry::ChannelState,
6406 router::FrameSink,
6407 stderr_tail::DEFAULT_MAX_LINE_BYTES,
6408 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
6409 RouteCtx, Router,
6410 };
6411 use tokio::{
6412 sync::mpsc,
6413 time::{sleep, Instant},
6414 };
6415 use tracing::{
6416 field::{Field, Visit},
6417 Event, Subscriber,
6418 };
6419 use tracing_subscriber::{layer::Context, prelude::*, Layer};
6420
6421 fn fake_aft_stub_path() -> PathBuf {
6440 let mut path = std::env::current_exe().expect("current_exe available in tests");
6441 path.pop(); path.pop(); path.push(if cfg!(windows) {
6444 "fake-aft-stub.exe"
6445 } else {
6446 "fake-aft-stub"
6447 });
6448 assert!(
6449 path.exists(),
6450 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
6451 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
6452 path.display()
6453 );
6454 path
6455 }
6456
6457 fn client_retries(code: &str) -> bool {
6462 subc_protocol::error_codes::is_retryable_route_open(code)
6463 }
6464
6465 #[test]
6476 fn retryability_of_forwarding_codes_matches_the_failure() {
6477 let transient = [
6480 ForwardingError::NoModuleConnection,
6481 ForwardingError::ModuleReloading {
6482 module_id: "m".into(),
6483 },
6484 ForwardingError::StaleModuleEndpoint,
6485 ForwardingError::UnknownReservation {
6486 client_channel: 1,
6487 module_channel: 1,
6488 },
6489 ForwardingError::ConnectionClosing {
6490 connection_id: ConnectionId::new(1),
6491 },
6492 ForwardingError::ClientEgressClosed {
6493 connection_id: ConnectionId::new(1),
6494 },
6495 ForwardingError::ModuleEgressUnavailable {
6496 connection_id: ConnectionId::new(1),
6497 },
6498 ];
6499 for err in transient {
6500 let code = forwarding_error_code(&err);
6501 assert!(
6502 client_retries(code),
6503 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
6504 );
6505 }
6506
6507 let permanent = [
6512 ForwardingError::ConnectionRoleConflict {
6513 connection_id: ConnectionId::new(1),
6514 },
6515 ForwardingError::ClientRouteChannelExhausted {
6516 connection_id: ConnectionId::new(1),
6517 },
6518 ForwardingError::ModuleRouteChannelExhausted {
6519 endpoint: ModuleEndpointId {
6520 connection_id: ConnectionId::new(1),
6521 generation: 1,
6522 },
6523 },
6524 ForwardingError::RelayCorrelationExhausted,
6525 ForwardingError::RouteOpenBuild("x".into()),
6526 ForwardingError::Poisoned,
6527 ];
6528 for err in permanent {
6529 let code = forwarding_error_code(&err);
6530 assert!(
6531 !client_retries(code),
6532 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
6533 );
6534 }
6535 }
6536
6537 #[tokio::test]
6565 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
6566 let handler = ControlHandler::default();
6567 let frame =
6568 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
6569
6570 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
6574 assert_eq!(
6575 stamped,
6576 Principal::Direct,
6577 "a caller that proved nothing must not be stamped as a supervised module"
6578 );
6579
6580 let forged = handler
6585 .route_open_principal(
6586 &frame,
6587 Some(ConsumerIdentity {
6588 module_id: "aft".to_string(),
6589 launch_nonce: "not-a-real-nonce".to_string(),
6590 }),
6591 )
6592 .unwrap();
6593 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
6594 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
6595 }
6596
6597 #[test]
6614 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
6615 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
6616 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
6617 let ClientControlRequest::RouteOpen {
6618 consumer_identity, ..
6619 } = parsed
6620 else {
6621 panic!("route.open body must parse as RouteOpen");
6622 };
6623 assert_eq!(
6624 consumer_identity,
6625 Some(ConsumerIdentity {
6626 module_id: "aft".to_string(),
6627 launch_nonce: "n".to_string(),
6628 }),
6629 "the wire field name must reach the value route_open_principal reads"
6630 );
6631 }
6632
6633 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
6634 ModuleManifest::builder(module_id, "0.1.0")
6635 .protocol_ver(protocol_ver)
6636 .provides(vec![ProviderRole::ToolProvider {
6637 tools: vec![Tool {
6638 name: "read".to_string(),
6639 description: None,
6640 execution_mode: ExecutionMode::Pure,
6641 schema: json!({"type": "object"}),
6642 }],
6643 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
6644 concurrency: Concurrency::ModuleManaged,
6645 emits_push: true,
6646 sub_supervises: true,
6647 }])
6648 .build()
6649 }
6650
6651 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
6652 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
6653 }
6654
6655 fn hello_frame_with_control_ops(
6656 module_id: &str,
6657 protocol_ver: u8,
6658 corr: u64,
6659 control_ops: Option<Vec<String>>,
6660 ) -> Frame {
6661 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
6662 }
6663
6664 fn hello_frame_with_nonce(
6665 module_id: &str,
6666 protocol_ver: u8,
6667 corr: u64,
6668 launch_nonce: Option<&str>,
6669 ) -> Frame {
6670 hello_frame_full(
6671 module_id,
6672 protocol_ver,
6673 corr,
6674 None,
6675 launch_nonce.map(ToOwned::to_owned),
6676 )
6677 }
6678
6679 fn hello_frame_full(
6680 module_id: &str,
6681 protocol_ver: u8,
6682 corr: u64,
6683 control_ops: Option<Vec<String>>,
6684 launch_nonce: Option<String>,
6685 ) -> Frame {
6686 let body = serde_json::to_vec(&ModuleHelloBody {
6687 manifest: manifest(module_id, protocol_ver),
6688 protocol_ver,
6689 control_ops,
6690 launch_nonce,
6691 })
6692 .unwrap();
6693 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
6694 }
6695
6696 fn non_routable_hello_frame_with_control_ops(
6697 module_id: &str,
6698 corr: u64,
6699 control_ops: Option<Vec<String>>,
6700 ) -> Frame {
6701 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
6702 manifest.provides.clear();
6703 let body = serde_json::to_vec(&ModuleHelloBody {
6704 manifest,
6705 protocol_ver: PROTOCOL_VERSION,
6706 control_ops,
6707 launch_nonce: None,
6708 })
6709 .unwrap();
6710 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
6711 }
6712
6713 fn capability_grammar_hello_frame(
6714 capabilities: Value,
6715 runtime_computed: Option<Value>,
6716 corr: u64,
6717 ) -> Frame {
6718 let mut body = serde_json::to_value(ModuleHelloBody {
6719 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
6720 protocol_ver: PROTOCOL_VERSION,
6721 control_ops: None,
6722 launch_nonce: None,
6723 })
6724 .expect("HELLO body serializes");
6725 body["manifest"]["capabilities"] = capabilities;
6726 if let Some(runtime_computed) = runtime_computed {
6727 body["runtime_computed"] = runtime_computed;
6728 }
6729 Frame::build(
6730 FrameType::Hello,
6731 control_flags(),
6732 0,
6733 0,
6734 corr,
6735 serde_json::to_vec(&body).expect("HELLO body reserializes"),
6736 )
6737 .expect("HELLO frame builds")
6738 }
6739
6740 fn channel_request(channel: u16, corr: u64) -> Frame {
6741 Frame::build(
6742 FrameType::Request,
6743 Flags::new(true, Priority::Interactive, false),
6744 channel,
6745 0,
6746 corr,
6747 b"opaque".to_vec(),
6748 )
6749 .unwrap()
6750 }
6751
6752 fn route_ctx(
6753 connection_id: ConnectionId,
6754 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
6755 let (tx, rx) = mpsc::channel(8);
6756 (
6757 RouteCtx {
6758 connection_id,
6759 egress: FrameSink::new(tx),
6760 },
6761 rx,
6762 )
6763 }
6764
6765 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
6766 serde_json::from_slice(&frame.body).unwrap()
6767 }
6768
6769 async fn hello_via_sink(
6774 handler: &ControlHandler,
6775 ctx: &RouteCtx,
6776 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
6777 hello: Frame,
6778 ) -> Frame {
6779 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
6780 assert!(
6781 replies.is_empty(),
6782 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
6783 );
6784 let ack = rx
6785 .try_recv()
6786 .expect("HELLO_ACK is queued on the module sink")
6787 .frame;
6788 assert_eq!(ack.header.ty, FrameType::HelloAck);
6789 ack
6790 }
6791
6792 fn parse_error(frame: &Frame) -> Value {
6793 serde_json::from_slice(&frame.body).unwrap()
6794 }
6795
6796 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
6797 serde_json::from_slice(&frame.body).unwrap()
6798 }
6799
6800 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
6801 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
6802 route_channel,
6803 route_epoch: 0,
6804 kind,
6805 })
6806 .unwrap();
6807 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6808 }
6809
6810 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
6811 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
6812 module_id: module_id.to_string(),
6813 })
6814 .unwrap();
6815 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6816 }
6817
6818 fn route_open_frame(corr: u64, module_id: &str, project_root: TestTempDir) -> Frame {
6819 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
6820 }
6821
6822 fn route_open_frame_with_consumer_capabilities(
6823 corr: u64,
6824 module_id: &str,
6825 project_root: TestTempDir,
6826 consumer_capabilities: Option<Vec<String>>,
6827 ) -> Frame {
6828 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6829 target: RouteTarget::ToolProvider {
6830 module_id: module_id.to_string(),
6831 },
6832 identity: BindIdentity::new(
6833 project_root.path().to_path_buf(),
6834 "unit".to_string(),
6835 "session".to_string(),
6836 ),
6837 consumer_identity: None,
6838 consumer_capabilities,
6839 role_versions: None,
6840 admission_facts: None,
6841 scope: None,
6842 })
6843 .unwrap();
6844 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6845 }
6846
6847 fn route_open_frame_with_role_versions(
6848 corr: u64,
6849 module_id: &str,
6850 project_root: TestTempDir,
6851 role_versions: Option<BTreeMap<String, String>>,
6852 ) -> Frame {
6853 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6854 target: RouteTarget::ToolProvider {
6855 module_id: module_id.to_string(),
6856 },
6857 identity: BindIdentity::new(
6858 project_root.path().to_path_buf(),
6859 "unit".to_string(),
6860 format!("session-{corr}"),
6861 ),
6862 consumer_identity: None,
6863 consumer_capabilities: None,
6864 role_versions,
6865 admission_facts: None,
6866 scope: None,
6867 })
6868 .unwrap();
6869 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6870 }
6871
6872 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
6873 entries
6874 .iter()
6875 .map(|(role, version)| (role.to_string(), version.to_string()))
6876 .collect()
6877 }
6878
6879 fn route_open_frame_with_admission_facts(
6880 corr: u64,
6881 module_id: &str,
6882 project_root: TestTempDir,
6883 consumer_identity: Option<subc_control::ConsumerIdentity>,
6884 facts: Option<Value>,
6885 ) -> Frame {
6886 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6887 target: RouteTarget::ToolProvider {
6888 module_id: module_id.to_string(),
6889 },
6890 identity: BindIdentity::new(
6891 project_root.path().to_path_buf(),
6892 "unit".to_string(),
6893 format!("session-{corr}"),
6894 ),
6895 consumer_identity,
6896 consumer_capabilities: None,
6897 role_versions: None,
6898 admission_facts: facts,
6899 scope: None,
6900 })
6901 .unwrap();
6902 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6903 }
6904
6905 #[derive(Clone, Default)]
6906 struct EventCapture {
6907 events: Arc<Mutex<Vec<CapturedEvent>>>,
6908 }
6909
6910 #[derive(Clone, Debug)]
6911 struct CapturedEvent {
6912 target: String,
6913 level: tracing::Level,
6914 fields: BTreeMap<String, String>,
6915 }
6916
6917 impl EventCapture {
6918 fn events(&self) -> Vec<CapturedEvent> {
6919 self.events.lock().unwrap().clone()
6920 }
6921 }
6922
6923 impl<S> Layer<S> for EventCapture
6924 where
6925 S: Subscriber,
6926 {
6927 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
6928 let mut visitor = EventFieldVisitor::default();
6929 event.record(&mut visitor);
6930 self.events.lock().unwrap().push(CapturedEvent {
6931 target: event.metadata().target().to_string(),
6932 level: *event.metadata().level(),
6933 fields: visitor.fields,
6934 });
6935 }
6936 }
6937
6938 #[derive(Default)]
6939 struct EventFieldVisitor {
6940 fields: BTreeMap<String, String>,
6941 }
6942
6943 impl Visit for EventFieldVisitor {
6944 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
6945 self.fields
6946 .insert(field.name().to_string(), format!("{value:?}"));
6947 }
6948 }
6949
6950 fn health_response(corr: u64, status: HealthStatus) -> Frame {
6951 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
6952 status,
6953 detail: Some("warming".to_string()),
6954 metrics: Some(json!({"queue_depth": 3})),
6955 })
6956 .unwrap();
6957 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
6958 }
6959
6960 fn route_bind_ack(corr: u64) -> Frame {
6961 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
6962 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
6963 }
6964
6965 fn unique_project_root(label: &str) -> TestTempDir {
6966 TestTempDir::new(label)
6967 }
6968
6969 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
6970 match parse_route_poll(frame) {
6971 ClientControlResponse::RoutePoll {
6972 status: None,
6973 live: Some(live),
6974 ..
6975 } => assert_eq!(live, expected_live),
6976 other => panic!("unexpected route.poll response: {other:?}"),
6977 }
6978 }
6979
6980 fn bind_liveness_route(
6981 registry: &Registry,
6982 forwarding: &ForwardingTable,
6983 module_id: &str,
6984 ) -> (RouteCtx, u16, u32) {
6985 let module_connection = ConnectionId::new(101);
6986 let client_connection = ConnectionId::new(202);
6987 let registration = registry
6988 .register_with_control_ops(
6989 manifest(module_id, PROTOCOL_VERSION),
6990 PROTOCOL_VERSION,
6991 module_connection,
6992 module_baseline_control_ops(),
6993 )
6994 .unwrap();
6995 let (module_tx, _module_rx) = mpsc::channel(8);
6996 let endpoint = forwarding
6997 .register_module_connection(
6998 module_connection,
6999 module_id.to_string(),
7000 PROTOCOL_VERSION,
7001 manifest_concurrency(®istration.manifest),
7002 FrameSink::new(module_tx),
7003 )
7004 .unwrap();
7005 let (client_ctx, _client_rx) = route_ctx(client_connection);
7006 let pending = forwarding
7007 .begin_route_bind_relay_for_test(
7008 client_connection,
7009 client_ctx.egress.clone(),
7010 1,
7011 module_id,
7012 )
7013 .unwrap();
7014 assert_eq!(pending.endpoint, endpoint);
7015 let route_channel = pending.client_channel;
7016 let route_epoch = pending.client_epoch;
7017 forwarding
7018 .complete_pending_relay(
7019 module_connection,
7020 pending.corr,
7021 RouteBindRelayOutcome::Accepted,
7022 )
7023 .unwrap();
7024 (client_ctx, route_channel, route_epoch)
7025 }
7026
7027 struct FakeProcessLiveness {
7028 live: Option<bool>,
7029 }
7030
7031 impl ModuleProcessLiveness for FakeProcessLiveness {
7032 fn process_live(&self, _module_id: &str) -> Option<bool> {
7033 self.live
7034 }
7035 }
7036
7037 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7038 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
7039 {
7040 let registry = Arc::new(Registry::default());
7041 let supervisor_handle = SupervisorHandle::new();
7042 let supervisor = Supervisor::new_for_test(
7043 Arc::clone(®istry),
7044 RestartPolicy::new(1, Duration::from_millis(10)),
7045 )
7046 .with_handle(supervisor_handle.clone());
7047 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
7048 let module = supervisor
7049 .spawn(ModuleSpec {
7050 module_id: "stderr-tail-wire".to_string(),
7051 program: fake_aft_stub_path(),
7052 args: Vec::new(),
7053 env: vec![
7054 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
7055 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
7056 ],
7057 reserved: false,
7058 reserved_prefixes: Vec::new(),
7059 protocol: ModuleProtocol::Subc,
7060 overlap: Default::default(),
7061 })
7062 .unwrap();
7063
7064 let deadline = Instant::now() + Duration::from_secs(5);
7065 loop {
7066 let tail = module.stderr_tail(None, None);
7067 if tail
7068 .entries
7069 .iter()
7070 .any(|entry| matches!(entry, TailEntry::ProcessStart))
7071 && tail.entries.iter().any(|entry| {
7072 matches!(
7073 entry,
7074 TailEntry::Line {
7075 truncated: true,
7076 ..
7077 }
7078 )
7079 })
7080 {
7081 break;
7082 }
7083 assert!(
7084 Instant::now() < deadline,
7085 "module did not produce a truncated line and restart boundary: {tail:?}"
7086 );
7087 sleep(Duration::from_millis(10)).await;
7088 }
7089
7090 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
7091 let request = ClientControlRequest::SupervisorStderrTail {
7092 module_id: "stderr-tail-wire".to_string(),
7093 max_lines: None,
7094 max_bytes: None,
7095 };
7096 let frame = Frame::build(
7097 FrameType::Request,
7098 control_flags(),
7099 0,
7100 0,
7101 1,
7102 serde_json::to_vec(&request).unwrap(),
7103 )
7104 .unwrap();
7105 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7106 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
7107 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
7108 serde_json::from_slice(&responses[0].body).unwrap()
7109 else {
7110 panic!("expected supervisor.stderr_tail response");
7111 };
7112
7113 assert!(
7114 tail.entries
7115 .iter()
7116 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
7117 "the control response lost the restart boundary"
7118 );
7119 let Some(StderrTailEntry::Line {
7120 text,
7121 truncated,
7122 at_ms,
7123 }) = tail.entries.iter().find(|entry| {
7124 matches!(
7125 entry,
7126 StderrTailEntry::Line {
7127 truncated: true,
7128 ..
7129 }
7130 )
7131 })
7132 else {
7133 panic!("the control response lost the truncated line");
7134 };
7135 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
7136 assert!(*truncated);
7137 assert!(
7138 at_ms.is_some(),
7139 "the control response lost the line's capture time"
7140 );
7141 }
7142
7143 #[tokio::test(flavor = "current_thread")]
7148 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
7149 let dir = TestTempDir::new("terminals-off-worker");
7150 let journal_path = dir.join("terminals.jsonl");
7151 let registry = Arc::new(Registry::default());
7152 let supervisor_handle = SupervisorHandle::new();
7153 let supervisor =
7154 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
7155 .with_handle(supervisor_handle.clone())
7156 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
7157 let module = supervisor
7158 .spawn(ModuleSpec {
7159 module_id: "terminal-off-worker".to_string(),
7160 program: fake_aft_stub_path(),
7161 args: Vec::new(),
7162 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
7163 reserved: false,
7164 reserved_prefixes: Vec::new(),
7165 protocol: ModuleProtocol::Subc,
7166 overlap: Default::default(),
7167 })
7168 .unwrap();
7169 let deadline = Instant::now() + Duration::from_secs(5);
7170 while module.terminal_history().entries.len() != 2 {
7171 assert!(Instant::now() < deadline, "module did not record two exits");
7172 sleep(Duration::from_millis(10)).await;
7173 }
7174
7175 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
7176 let handler =
7177 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
7178 let frame = Frame::build(
7179 FrameType::Request,
7180 control_flags(),
7181 0,
7182 0,
7183 1,
7184 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
7185 module_id: "terminal-off-worker".to_string(),
7186 })
7187 .unwrap(),
7188 )
7189 .unwrap();
7190 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7191 let spawned_at = std::time::Instant::now();
7192 let read = tokio::spawn({
7193 let handler = Arc::clone(&handler);
7194 async move { handler.handle_control_frame(&ctx, frame).await }
7195 });
7196 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
7199 .await
7200 .unwrap()
7201 .expect("the history read reached its pause");
7202 let elapsed = spawned_at.elapsed();
7203 assert!(
7204 elapsed < Duration::from_secs(2) && !read.is_finished(),
7205 "this task could not run while the history read was paused \
7206 (resumed after {elapsed:?}, read finished: {})",
7207 read.is_finished()
7208 );
7209
7210 drop(release);
7211 let responses = read.await.unwrap().unwrap();
7212 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
7213 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
7214 panic!("expected supervisor.terminals response");
7215 };
7216 assert_eq!(terminals.entries.len(), 2);
7217 assert_eq!(terminals.journal_skipped_lines, 0);
7218 assert_eq!(terminals.journal_read_errors, 0);
7219 }
7220
7221 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7222 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
7223 let registry = Arc::new(Registry::default());
7224 let supervisor_handle = SupervisorHandle::new();
7225 let supervisor =
7226 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
7227 .with_handle(supervisor_handle.clone());
7228 let module = supervisor
7229 .spawn(ModuleSpec {
7230 module_id: "terminal-golden".to_string(),
7231 program: fake_aft_stub_path(),
7232 args: Vec::new(),
7233 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
7234 reserved: false,
7235 reserved_prefixes: Vec::new(),
7236 protocol: ModuleProtocol::Subc,
7237 overlap: Default::default(),
7238 })
7239 .unwrap();
7240
7241 let deadline = Instant::now() + Duration::from_secs(5);
7242 while module.terminal_history().entries.len() != 2 {
7243 assert!(
7244 Instant::now() < deadline,
7245 "module did not retain two terminal exits: {:?}",
7246 module.terminal_history()
7247 );
7248 sleep(Duration::from_millis(10)).await;
7249 }
7250
7251 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
7252 let request = ClientControlRequest::SupervisorTerminals {
7253 module_id: "terminal-golden".to_string(),
7254 };
7255 let frame = Frame::build(
7256 FrameType::Request,
7257 control_flags(),
7258 0,
7259 0,
7260 1,
7261 serde_json::to_vec(&request).unwrap(),
7262 )
7263 .unwrap();
7264 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7265 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
7266 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
7267 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
7268 panic!("expected supervisor.terminals response");
7269 };
7270 assert_eq!(terminals.entries.len(), 2);
7271 assert_eq!(terminals.dropped, 0);
7272
7273 let mut rendered = serde_json::to_value(response).unwrap();
7274 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
7277 for (index, entry) in rendered["entries"]
7278 .as_array_mut()
7279 .expect("terminal response entries array")
7280 .iter_mut()
7281 .enumerate()
7282 {
7283 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
7284 }
7285
7286 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
7287 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
7288 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
7289 if std::env::var_os("UPDATE_GOLDEN").is_some() {
7290 std::fs::write(&golden_path, &serialized).unwrap();
7291 }
7292 let expected: Value =
7293 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
7294 assert_eq!(rendered, expected);
7295 }
7296
7297 #[test]
7298 fn hello_registers_manifest_and_returns_ack() {
7299 let registry = Arc::new(Registry::default());
7300 let handler = ControlHandler::new(Arc::clone(®istry));
7301 let conn = ConnectionId::new(1);
7302
7303 let responses = handler
7304 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
7305 .unwrap();
7306
7307 assert_eq!(responses.len(), 1);
7308 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
7309 assert_eq!(responses[0].header.channel, 0);
7310 assert_eq!(responses[0].header.corr, 7);
7311 let ack = parse_ack(&responses[0]);
7312 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
7313 assert!(ack
7314 .subc_capabilities
7315 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
7316 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
7317 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
7318 assert!(ack
7319 .subc_ops
7320 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
7321 assert!(ack
7322 .subc_ops
7323 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
7324
7325 let registration = registry.get_module("aft").unwrap().unwrap();
7326 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
7327 assert_eq!(registration.state, ChannelState::Active);
7328 assert_eq!(registration.connection_id, conn);
7329 assert_eq!(registration.control_ops, module_baseline_control_ops());
7330 }
7331
7332 #[test]
7333 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
7334 let invalid_identifiers = [
7335 ("case_change", "credentials-Provider/v1"),
7336 ("leading_zero", "credentials-provider/v01"),
7337 ("trailing_hyphen", "credentials-provider-/v1"),
7338 ("consecutive_hyphens", "credentials--provider/v1"),
7339 ("uppercase", "Credentials-provider/v1"),
7340 ("missing_v", "credentials-provider/1"),
7341 ("whitespace", "credentials provider/v1"),
7342 ("zero_version", "credentials-provider/v0"),
7343 ("out_of_range_version", "credentials-provider/v4294967296"),
7344 (
7345 "overlength_name",
7346 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
7347 ),
7348 ];
7349 let mut cases = invalid_identifiers
7350 .into_iter()
7351 .map(|(name, identifier)| {
7352 (
7353 format!("identifier_{name}"),
7354 "capabilities.provides[0]".to_string(),
7355 identifier.to_string(),
7356 json!({ "provides": [identifier] }),
7357 None,
7358 )
7359 })
7360 .collect::<Vec<_>>();
7361 cases.extend([
7362 (
7363 "unknown_need".to_string(),
7364 "capabilities.requires[0].need".to_string(),
7365 "deferred".to_string(),
7366 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
7367 None,
7368 ),
7369 (
7370 "duplicate_provides".to_string(),
7371 "capabilities.provides[1]".to_string(),
7372 "credentials-provider/v1".to_string(),
7373 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
7374 None,
7375 ),
7376 (
7377 "duplicate_must_never_reach".to_string(),
7378 "capabilities.must_never_reach[1]".to_string(),
7379 "credentials-provider/v1".to_string(),
7380 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
7381 None,
7382 ),
7383 (
7384 "duplicate_requires_same_need".to_string(),
7385 "capabilities.requires[1]".to_string(),
7386 "credentials-provider/v1".to_string(),
7387 json!({ "requires": [
7388 { "capability": "credentials-provider/v1", "need": "required" },
7389 { "capability": "credentials-provider/v1", "need": "required" }
7390 ] }),
7391 None,
7392 ),
7393 (
7394 "duplicate_requires_conflicting_need".to_string(),
7395 "capabilities.requires[1]".to_string(),
7396 "credentials-provider/v1".to_string(),
7397 json!({ "requires": [
7398 { "capability": "credentials-provider/v1", "need": "required" },
7399 { "capability": "credentials-provider/v1", "need": "optional" }
7400 ] }),
7401 None,
7402 ),
7403 (
7404 "capabilities_root_pointer".to_string(),
7405 "runtime_computed[0]".to_string(),
7406 "/capabilities".to_string(),
7407 json!({}),
7408 Some(json!(["/capabilities"])),
7409 ),
7410 (
7411 "capabilities_descendant_pointer".to_string(),
7412 "runtime_computed[0]".to_string(),
7413 "/capabilities/provides".to_string(),
7414 json!({}),
7415 Some(json!(["/capabilities/provides"])),
7416 ),
7417 (
7418 "malformed_pointer_without_leading_slash".to_string(),
7419 "runtime_computed[0]".to_string(),
7420 "capabilities".to_string(),
7421 json!({}),
7422 Some(json!(["capabilities"])),
7423 ),
7424 (
7425 "malformed_pointer_escape".to_string(),
7426 "runtime_computed[0]".to_string(),
7427 "/roles/~2/tools".to_string(),
7428 json!({}),
7429 Some(json!(["/roles/~2/tools"])),
7430 ),
7431 (
7432 "unknown_capabilities_field".to_string(),
7433 "capabilities.future".to_string(),
7434 "<array>".to_string(),
7435 json!({ "future": [] }),
7436 None,
7437 ),
7438 ]);
7439
7440 for (index, (name, field, value, capabilities, runtime_computed)) in
7441 cases.into_iter().enumerate()
7442 {
7443 let registry = Arc::new(Registry::default());
7444 let handler = ControlHandler::new(Arc::clone(®istry));
7445 let response = handler
7446 .handle_control(
7447 ConnectionId::new((index + 1) as u64),
7448 capability_grammar_hello_frame(
7449 capabilities,
7450 runtime_computed,
7451 index as u64 + 1,
7452 ),
7453 )
7454 .expect("invalid HELLO returns a refusal");
7455
7456 assert_eq!(response.len(), 1, "{name} must emit one refusal");
7457 let error = parse_error(&response[0]);
7458 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
7459 let message = error["message"]
7460 .as_str()
7461 .expect("error message is a string");
7462 assert!(
7463 message.contains(&field),
7464 "{name}: field missing from {message}"
7465 );
7466 assert!(
7467 message.contains(&value),
7468 "{name}: value missing from {message}"
7469 );
7470 assert_eq!(
7471 registry
7472 .active_registration_count()
7473 .expect("registry reads"),
7474 0,
7475 "{name}: refused HELLO must not create a catalog entry"
7476 );
7477 }
7478 }
7479
7480 #[test]
7481 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
7482 let registry = Arc::new(Registry::default());
7483 let handler = ControlHandler::new(Arc::clone(®istry));
7484 let capabilities = json!({
7485 "provides": ["credentials-provider/v1"],
7486 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
7487 "must_never_reach": ["federation-transport/v1"]
7488 });
7489 let response = handler
7490 .handle_control(
7491 ConnectionId::new(99),
7492 capability_grammar_hello_frame(
7493 capabilities.clone(),
7494 Some(json!(["/roles/0/tools"])),
7495 99,
7496 ),
7497 )
7498 .expect("valid HELLO registers");
7499 assert_eq!(response[0].header.ty, FrameType::HelloAck);
7500
7501 let request = Frame::build(
7502 FrameType::Request,
7503 control_flags(),
7504 0,
7505 0,
7506 100,
7507 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7508 .expect("catalog request serializes"),
7509 )
7510 .expect("catalog request frame builds");
7511 let response = handler
7512 .handle_catalog_list(request, None)
7513 .expect("catalog list succeeds");
7514 let ClientControlResponse::CatalogList { modules, .. } =
7515 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
7516 else {
7517 panic!("catalog request must return catalog.list");
7518 };
7519 assert_eq!(modules.len(), 1);
7520 assert_eq!(
7521 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
7522 capabilities
7523 );
7524 }
7525
7526 #[test]
7527 fn catalog_list_mirrors_management_operation_description() {
7528 let registry = Arc::new(Registry::default());
7529 let handler = ControlHandler::new(Arc::clone(®istry));
7530 let description = "List managed records and return their identifiers and metadata.";
7531 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
7532 manifest.provides = vec![ProviderRole::ManagementSurface {
7533 operations: vec![ManagementOperation {
7534 name: "records.list".to_string(),
7535 kind: ManagementOperationKind::Query,
7536 description: Some(description.to_string()),
7537 }],
7538 config_schema: json!({"type": "object"}),
7539 observability: vec![ObservabilitySurface {
7540 name: "records.stats".to_string(),
7541 kind: ObservabilityKind::Snapshot,
7542 }],
7543 identity_scope: vec![IdentityScope::Project],
7544 concurrency: Concurrency::ModuleManaged,
7545 }];
7546 registry
7547 .register_with_control_ops(
7548 manifest,
7549 PROTOCOL_VERSION,
7550 ConnectionId::new(99),
7551 Vec::new(),
7552 )
7553 .expect("described management manifest registers");
7554
7555 let request = Frame::build(
7556 FrameType::Request,
7557 control_flags(),
7558 0,
7559 0,
7560 100,
7561 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7562 .expect("catalog request serializes"),
7563 )
7564 .expect("catalog request frame builds");
7565 let response = handler
7566 .handle_catalog_list(request, None)
7567 .expect("catalog list succeeds");
7568 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
7569 assert_eq!(
7570 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
7571 "catalog.list must preserve the declared operation description verbatim"
7572 );
7573 }
7574
7575 #[test]
7576 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
7577 let registry = Arc::new(Registry::default());
7578 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
7579 [("vault".to_string(), true), ("squatter".to_string(), true)],
7580 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
7581 );
7582 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
7583 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
7584 provides: vec!["credentials-provider/v1".to_string()],
7585 requires: Vec::new(),
7586 must_never_reach: Vec::new(),
7587 });
7588 let frame = Frame::build(
7589 FrameType::Hello,
7590 control_flags(),
7591 0,
7592 0,
7593 77,
7594 serde_json::to_vec(&ModuleHelloBody {
7595 manifest: squatter,
7596 protocol_ver: PROTOCOL_VERSION,
7597 control_ops: None,
7598 launch_nonce: None,
7599 })
7600 .expect("HELLO serializes"),
7601 )
7602 .expect("HELLO frame builds");
7603 let response = handler
7604 .handle_control(ConnectionId::new(77), frame)
7605 .expect("reserved claim receives a typed refusal");
7606 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
7607 assert_eq!(
7608 registry
7609 .active_registration_count()
7610 .expect("registry reads"),
7611 0,
7612 "a reserved capability refusal must not leave a catalog entry"
7613 );
7614 }
7615
7616 #[test]
7617 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
7618 for settlement in ["timeout", "inconclusive", "drop"] {
7619 let breakers = RouteBindBreakers::default();
7620 let RouteBindAdmission::Admitted {
7621 guard: mut old,
7622 probe: false,
7623 } = breakers.admit("prov")
7624 else {
7625 panic!("ordinary relay admitted")
7626 };
7627 let RouteBindAdmission::Admitted {
7628 guard: mut opener, ..
7629 } = breakers.admit("prov")
7630 else {
7631 panic!("second relay admitted")
7632 };
7633 assert!(
7634 !opener
7635 .record_timeout(1, Duration::ZERO)
7636 .unwrap()
7637 .reopened_after_probe
7638 );
7639 let RouteBindAdmission::Admitted {
7640 guard: mut probe,
7641 probe: true,
7642 } = breakers.admit("prov")
7643 else {
7644 panic!("one cooldown probe admitted")
7645 };
7646 match settlement {
7647 "timeout" => assert!(
7648 !old.record_timeout(1, Duration::ZERO)
7649 .unwrap()
7650 .reopened_after_probe
7651 ),
7652 "inconclusive" => old.record_inconclusive(),
7653 "drop" => drop(old),
7654 _ => unreachable!(),
7655 }
7656 assert!(
7657 matches!(
7658 breakers.admit("prov"),
7659 RouteBindAdmission::Refused {
7660 probe_in_flight: true,
7661 ..
7662 }
7663 ),
7664 "{settlement} of a pre-open relay cannot release the real probe"
7665 );
7666 assert!(
7667 probe
7668 .record_timeout(1, Duration::ZERO)
7669 .unwrap()
7670 .reopened_after_probe
7671 );
7672 assert!(matches!(
7673 breakers.admit("prov"),
7674 RouteBindAdmission::Admitted { probe: true, .. }
7675 ));
7676 }
7677 let breakers = RouteBindBreakers::default();
7678 let admit = || match breakers.admit("prov") {
7679 RouteBindAdmission::Admitted { guard, .. } => guard,
7680 _ => panic!("relay admitted"),
7681 };
7682 admit().record_timeout(1, Duration::ZERO);
7683 let mut old_probe = admit();
7684 breakers.reset_for_new_module_connection("prov");
7685 admit().record_timeout(1, Duration::ZERO);
7686 let _new_probe = admit();
7687 old_probe.record_inconclusive();
7688 assert!(matches!(
7689 breakers.admit("prov"),
7690 RouteBindAdmission::Refused {
7691 probe_in_flight: true,
7692 ..
7693 }
7694 ));
7695 }
7696
7697 #[tokio::test]
7698 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
7699 for candidate in [false, true] {
7700 let registry = Arc::new(Registry::default());
7701 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
7702 [("vault".to_string(), true), ("squatter".to_string(), true)],
7703 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
7704 );
7705 let conn = ConnectionId::new(77);
7706 let (ctx, mut rx) = route_ctx(conn);
7707 let initial = capability_manifest("squatter", &[], &[]);
7708 if candidate {
7709 registry
7710 .register_candidate_with_control_ops(
7711 initial.clone(),
7712 PROTOCOL_VERSION,
7713 conn,
7714 module_baseline_control_ops(),
7715 )
7716 .unwrap();
7717 handler
7718 .forwarding
7719 .register_candidate_module_connection(
7720 conn,
7721 "squatter".to_string(),
7722 PROTOCOL_VERSION,
7723 manifest_concurrency(&initial),
7724 ctx.egress.clone(),
7725 )
7726 .unwrap();
7727 } else {
7728 hello_via_sink(
7729 &handler,
7730 &ctx,
7731 &mut rx,
7732 hello_frame_with_manifest(initial.clone(), 1),
7733 )
7734 .await;
7735 }
7736 let response = handler
7737 .handle_control_frame(
7738 &ctx,
7739 catalog_update_with_capabilities_frame(
7740 2,
7741 capability_manifest("squatter", &["credentials-provider/v1"], &[])
7742 .capabilities
7743 .unwrap(),
7744 ),
7745 )
7746 .await
7747 .unwrap();
7748 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
7749 assert_eq!(
7750 registry
7751 .get_module_by_connection(conn)
7752 .unwrap()
7753 .unwrap()
7754 .manifest,
7755 initial
7756 );
7757 }
7758 }
7759
7760 #[test]
7761 fn server_describe_surfaces_required_capability_verdict_fields() {
7762 let registry = Arc::new(Registry::default());
7763 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
7764 [
7765 ("consumer".to_string(), true),
7766 ("provider".to_string(), false),
7767 ],
7768 BTreeMap::new(),
7769 );
7770 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
7771 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
7772 provides: Vec::new(),
7773 requires: vec![subc_protocol::manifest::CapabilityRequirement {
7774 capability: "credentials-provider/v1".to_string(),
7775 need: subc_protocol::manifest::CapabilityNeed::Required,
7776 }],
7777 must_never_reach: Vec::new(),
7778 });
7779 let hello = Frame::build(
7780 FrameType::Hello,
7781 control_flags(),
7782 0,
7783 0,
7784 78,
7785 serde_json::to_vec(&ModuleHelloBody {
7786 manifest: consumer,
7787 protocol_ver: PROTOCOL_VERSION,
7788 control_ops: None,
7789 launch_nonce: None,
7790 })
7791 .expect("HELLO serializes"),
7792 )
7793 .expect("HELLO frame builds");
7794 handler
7795 .handle_control(ConnectionId::new(78), hello)
7796 .expect("consumer registers");
7797 let describe = Frame::build(
7798 FrameType::Request,
7799 control_flags(),
7800 0,
7801 0,
7802 79,
7803 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
7804 .expect("request serializes"),
7805 )
7806 .expect("describe frame builds");
7807 let response = handler
7808 .handle_server_describe(describe)
7809 .expect("server.describe succeeds");
7810 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
7811 let requirement = &rendered["capability_requirements"][0];
7812 assert_eq!(requirement["consumer"], "consumer");
7813 assert_eq!(requirement["verdict"], "never_provided");
7814 assert_eq!(requirement["episode_seq"], 1);
7815 assert_eq!(requirement["config_satisfiable"], false);
7816 assert_eq!(requirement["runtime_available"], false);
7817 assert!(requirement["detail"]
7818 .as_str()
7819 .expect("detail string")
7820 .contains("credentials-provider/v1"));
7821 }
7822
7823 #[test]
7824 fn catalog_list_omits_capabilities_for_legacy_manifest() {
7825 let registry = Arc::new(Registry::default());
7826 let handler = ControlHandler::new(Arc::clone(®istry));
7827 let hello = handler
7828 .handle_control(
7829 ConnectionId::new(101),
7830 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
7831 )
7832 .expect("legacy HELLO registers");
7833 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
7834
7835 let request = Frame::build(
7836 FrameType::Request,
7837 control_flags(),
7838 0,
7839 0,
7840 102,
7841 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7842 .expect("catalog request serializes"),
7843 )
7844 .expect("catalog request frame builds");
7845 let response = handler
7846 .handle_catalog_list(request, None)
7847 .expect("catalog list succeeds");
7848 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
7849 assert!(
7850 body["modules"][0].get("capabilities").is_none(),
7851 "legacy manifest must retain an absent capabilities field on catalog.list"
7852 );
7853 }
7854
7855 #[test]
7856 fn hello_ack_omits_storage_when_no_storage_config() {
7857 let registry = Arc::new(Registry::default());
7858 let handler = ControlHandler::new(Arc::clone(®istry));
7859 let responses = handler
7860 .handle_control(
7861 ConnectionId::new(1),
7862 hello_frame("aft", PROTOCOL_VERSION, 7),
7863 )
7864 .unwrap();
7865 let ack = parse_ack(&responses[0]);
7866 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
7867 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
7868 }
7869
7870 #[tokio::test]
7871 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
7872 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
7873 let registry = Arc::new(Registry::default());
7874 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
7875 let responses = handler
7876 .handle_control(
7877 ConnectionId::new(1),
7878 hello_frame("aft", PROTOCOL_VERSION, 7),
7879 )
7880 .unwrap();
7881 let ack = parse_ack(&responses[0]);
7882 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
7883
7884 let described = handler
7885 .handle_control_frame(
7886 &route_ctx(ConnectionId::new(2)).0,
7887 Frame::build(
7888 FrameType::Request,
7889 control_flags(),
7890 0,
7891 0,
7892 9,
7893 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
7894 )
7895 .unwrap(),
7896 )
7897 .await
7898 .unwrap();
7899 let ClientControlResponse::ServerDescribe { machine_id, .. } =
7900 serde_json::from_slice(&described[0].body).unwrap()
7901 else {
7902 panic!("server.describe answered with another shape");
7903 };
7904 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
7905 }
7906
7907 #[test]
7908 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
7909 let registry = Arc::new(Registry::default());
7912 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
7913 crate::daemon_config::StorageConfig::Sqlite {
7914 data_home: std::path::PathBuf::from("/data"),
7915 },
7916 ));
7917
7918 let responses = handler
7919 .handle_control(
7920 ConnectionId::new(1),
7921 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
7922 )
7923 .unwrap();
7924 let ack = parse_ack(&responses[0]);
7925 assert_eq!(
7926 ack.storage,
7927 Some(serde_json::json!({
7928 "module_id": "alfonso-routing",
7929 "storage_namespace": "default",
7930 "isolation": { "kind": "module" },
7931 "backend": {
7932 "backend": "sqlite",
7933 "path": "/data/cortexkit/alfonso-routing/store.db"
7934 }
7935 })),
7936 "the delivered descriptor is the module's own sqlite store path"
7937 );
7938 }
7939
7940 #[test]
7941 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
7942 let registry = Arc::new(Registry::default());
7943 let handler = ControlHandler::new(Arc::clone(®istry));
7944 let conn = ConnectionId::new(1);
7945 let responses = handler
7946 .handle_control(
7947 conn,
7948 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
7949 )
7950 .unwrap();
7951 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
7952 let registration = registry.get_module("aft").unwrap().unwrap();
7953 assert_eq!(registration.control_ops, module_baseline_control_ops());
7954
7955 let frame =
7956 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
7957 assert!(handler
7958 .guard_module_control_op(&frame, "aft", "route.bind")
7959 .unwrap()
7960 .is_none());
7961 let error = handler
7962 .guard_module_control_op(&frame, "aft", "test.synthetic")
7963 .unwrap()
7964 .expect("synthetic ungranted op should be rejected");
7965 assert_eq!(error.header.ty, FrameType::Error);
7966 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
7967 }
7968
7969 #[test]
7970 fn hello_control_ops_some_adds_optional_grants() {
7971 let registry = Arc::new(Registry::default());
7972 let handler = ControlHandler::new(Arc::clone(®istry));
7973 handler
7974 .handle_control(
7975 ConnectionId::new(1),
7976 hello_frame_with_control_ops(
7977 "aft",
7978 PROTOCOL_VERSION,
7979 7,
7980 Some(vec![
7981 "future.synthetic".to_string(),
7982 "route.bind".to_string(),
7983 ]),
7984 ),
7985 )
7986 .unwrap();
7987 let registration = registry.get_module("aft").unwrap().unwrap();
7988 assert_eq!(
7989 registration.control_ops,
7990 vec![
7991 "route.bind".to_string(),
7992 "route.status".to_string(),
7993 "future.synthetic".to_string(),
7994 ]
7995 );
7996 let frame =
7997 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
7998 assert!(handler
7999 .guard_module_control_op(&frame, "aft", "future.synthetic")
8000 .unwrap()
8001 .is_none());
8002 }
8003
8004 #[tokio::test]
8005 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
8006 let registry = Arc::new(Registry::default());
8007 let forwarding = Arc::new(ForwardingTable::default());
8008 let handler =
8009 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8010 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
8011 hello_via_sink(
8012 &handler,
8013 &module_ctx,
8014 &mut module_rx,
8015 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
8016 )
8017 .await;
8018
8019 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
8020 let responses = handler
8021 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
8022 .await
8023 .unwrap();
8024 assert_eq!(responses.len(), 1);
8025 assert_eq!(responses[0].header.ty, FrameType::Error);
8026 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
8027 assert!(module_rx.try_recv().is_err());
8028 }
8029
8030 #[tokio::test]
8031 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
8032 let registry = Arc::new(Registry::default());
8033 let forwarding = Arc::new(ForwardingTable::default());
8034 let handler =
8035 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8036 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
8037 hello_via_sink(
8038 &handler,
8039 &module_ctx,
8040 &mut module_rx,
8041 hello_frame_with_control_ops(
8042 "aft",
8043 PROTOCOL_VERSION,
8044 7,
8045 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
8046 ),
8047 )
8048 .await;
8049
8050 let project_root = unique_project_root("demux");
8051 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
8052 let route_handler = handler.clone();
8053 let route_task = tokio::spawn(async move {
8054 route_handler
8055 .handle_control_frame(
8056 &route_client_ctx,
8057 route_open_frame(100, "aft", project_root),
8058 )
8059 .await
8060 .unwrap()
8061 });
8062 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8063 .await
8064 .unwrap()
8065 .unwrap();
8066 assert!(matches!(
8067 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
8068 ModuleControlRequest::RouteBind { .. }
8069 ));
8070
8071 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
8072 let health_handler = handler.clone();
8073 let health_task = tokio::spawn(async move {
8074 health_handler
8075 .handle_control_frame(
8076 &health_client_ctx,
8077 supervisor_health_probe_frame(101, "aft"),
8078 )
8079 .await
8080 .unwrap()
8081 });
8082 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8083 .await
8084 .unwrap()
8085 .unwrap();
8086 assert_eq!(
8087 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
8088 ModuleControlRequest::HealthCheck {}
8089 );
8090
8091 handler
8092 .handle_control_frame(
8093 &module_ctx,
8094 health_response(health_frame.header.corr, HealthStatus::Degraded),
8095 )
8096 .await
8097 .unwrap();
8098 let health_response = health_task.await.unwrap();
8099 assert_eq!(health_response.len(), 1);
8100 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
8101 ClientControlResponse::SupervisorHealthProbe {
8102 module_id,
8103 status,
8104 detail,
8105 metrics,
8106 } => {
8107 assert_eq!(module_id, "aft");
8108 assert_eq!(status, HealthStatus::Degraded);
8109 assert_eq!(detail.as_deref(), Some("warming"));
8110 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
8111 }
8112 other => panic!("unexpected health response: {other:?}"),
8113 }
8114
8115 handler
8116 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8117 .await
8118 .unwrap();
8119 let route_response = route_task.await.unwrap();
8120 assert!(route_response.is_empty());
8121 let published = route_client_rx.recv().await.unwrap();
8122 assert!(matches!(
8123 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8124 ClientControlResponse::RouteOpen { .. }
8125 ));
8126 }
8127
8128 async fn relay_route_open(
8133 handler: &ControlHandler,
8134 client_connection: ConnectionId,
8135 client_egress: &FrameSink,
8136 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
8137 corr: u64,
8138 module_id: &str,
8139 project_root_label: &str,
8140 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
8141 let ctx = RouteCtx {
8142 connection_id: client_connection,
8143 egress: client_egress.clone(),
8144 };
8145 let handler = handler.clone();
8146 let project_root = unique_project_root(project_root_label);
8147 let module_id = module_id.to_string();
8148 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
8149 let task = tokio::spawn(async move {
8150 let _guard = tracing::dispatcher::set_default(&dispatch);
8151 handler
8152 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
8153 .await
8154 .unwrap()
8155 });
8156 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
8157 .await
8158 .expect("module receives the relayed route.bind")
8159 .expect("module egress is open");
8160 (task, bind.frame)
8161 }
8162
8163 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
8164 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
8165 ModuleControlRequest::RouteBind {
8166 route_channel,
8167 epoch,
8168 ..
8169 } => (route_channel, epoch),
8170 other => panic!("expected a route.bind request, got {other:?}"),
8171 }
8172 }
8173
8174 fn published_route(frame: &Frame) -> (u16, u32) {
8175 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
8176 ClientControlResponse::RouteOpen {
8177 route_channel,
8178 route_epoch,
8179 } => (route_channel, route_epoch),
8180 other => panic!("expected a route.open response, got {other:?}"),
8181 }
8182 }
8183
8184 #[tokio::test]
8204 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
8205 let registry = Arc::new(Registry::default());
8206 let forwarding = Arc::new(ForwardingTable::default());
8207 let handler =
8208 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8209
8210 let module_connection = ConnectionId::new(30);
8211 let (module_ctx, mut module_rx) = route_ctx(module_connection);
8212 hello_via_sink(
8213 &handler,
8214 &module_ctx,
8215 &mut module_rx,
8216 hello_frame("aft", PROTOCOL_VERSION, 7),
8217 )
8218 .await;
8219
8220 let dying_client = ConnectionId::new(31);
8221 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
8222
8223 let (first_task, first_bind) = relay_route_open(
8226 &handler,
8227 dying_client,
8228 &dying_ctx.egress,
8229 &mut module_rx,
8230 100,
8231 "aft",
8232 "closing-first",
8233 )
8234 .await;
8235 handler
8236 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
8237 .await
8238 .unwrap();
8239 assert!(first_task.await.unwrap().is_empty());
8240 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
8241
8242 let (second_task, second_bind) = relay_route_open(
8244 &handler,
8245 dying_client,
8246 &dying_ctx.egress,
8247 &mut module_rx,
8248 101,
8249 "aft",
8250 "closing-second",
8251 )
8252 .await;
8253 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
8254
8255 assert!(forwarding
8258 .escalate_client_delivery_failure(
8259 dying_client,
8260 first_channel,
8261 first_epoch,
8262 CloseReason::new(
8263 "module_to_client_delivery_failed",
8264 "client egress refused a module frame",
8265 ),
8266 crate::forwarding::UndeliveredFrame {
8267 module_id: None,
8268 sink: &dying_ctx.egress,
8269 },
8270 )
8271 .unwrap());
8272 assert!(!dying_ctx.egress.is_closed());
8273
8274 let ack = handler
8276 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
8277 .await;
8278 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
8279 if module_loop_error.is_some() {
8280 handler.cleanup_connection(module_connection).unwrap();
8284 }
8285 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8289 .await
8290 .ok()
8291 .flatten();
8292
8293 assert!(
8295 registry
8296 .get_module_by_connection(module_connection)
8297 .unwrap()
8298 .is_some(),
8299 "one client's closing connection ended the shared module connection: \
8300 {module_loop_error:?}"
8301 );
8302 let cotenant = ConnectionId::new(32);
8304 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
8305 let (cotenant_task, cotenant_bind) = relay_route_open(
8306 &handler,
8307 cotenant,
8308 &cotenant_ctx.egress,
8309 &mut module_rx,
8310 102,
8311 "aft",
8312 "closing-cotenant",
8313 )
8314 .await;
8315 handler
8316 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
8317 .await
8318 .unwrap();
8319 assert!(cotenant_task.await.unwrap().is_empty());
8320 let (cotenant_channel, cotenant_epoch) =
8321 published_route(&cotenant_rx.recv().await.unwrap());
8322 assert!(matches!(
8323 forwarding
8324 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
8325 .unwrap(),
8326 DataRoute::Client(DataRouteState::Bound(_))
8327 ));
8328
8329 let goodbye = post_ack_module_frame
8332 .expect("module receives a GOODBYE for the abandoned route channel");
8333 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
8334 assert_eq!(goodbye.header.channel, abandoned_channel);
8335 assert_eq!(goodbye.header.epoch, abandoned_epoch);
8336
8337 assert!(dying_rx.try_recv().is_err());
8341 let second_response = second_task.await.unwrap();
8342 assert_eq!(second_response.len(), 1);
8343 assert_eq!(
8344 parse_error(&second_response[0])["code"],
8345 "target_unavailable"
8346 );
8347 }
8348
8349 #[test]
8356 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
8357 let handler = ControlHandler::default();
8358 let module_connection = ConnectionId::new(30);
8359 let client_connection = ConnectionId::new(31);
8360
8361 handler
8362 .refuse_to_end_module_connection_for_a_client(
8363 module_connection,
8364 77,
8365 ForwardingError::ConnectionClosing {
8366 connection_id: client_connection,
8367 },
8368 )
8369 .expect("a closing client must never end the module connection");
8370
8371 assert!(matches!(
8372 handler.refuse_to_end_module_connection_for_a_client(
8373 module_connection,
8374 78,
8375 ForwardingError::ConnectionClosing {
8376 connection_id: module_connection,
8377 },
8378 ),
8379 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
8380 connection_id
8381 })) if connection_id == module_connection
8382 ));
8383 assert!(matches!(
8384 handler.refuse_to_end_module_connection_for_a_client(
8385 module_connection,
8386 79,
8387 ForwardingError::Poisoned,
8388 ),
8389 Err(RouterError::Forwarding(ForwardingError::Poisoned))
8390 ));
8391 assert!(matches!(
8392 handler.refuse_to_end_module_connection_for_a_client(
8393 module_connection,
8394 80,
8395 ForwardingError::StaleModuleEndpoint,
8396 ),
8397 Err(RouterError::Forwarding(
8398 ForwardingError::StaleModuleEndpoint
8399 ))
8400 ));
8401 }
8402
8403 #[tokio::test]
8416 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
8417 let registry = Arc::new(Registry::default());
8418 let forwarding = Arc::new(ForwardingTable::default());
8419 let supervisor = SupervisorHandle::new();
8420 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8421 let handler =
8422 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8423 .with_supervisor(supervisor);
8424
8425 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
8426 hello_via_sink(
8427 &handler,
8428 &target_ctx,
8429 &mut target_rx,
8430 hello_frame("target", PROTOCOL_VERSION, 1),
8431 )
8432 .await;
8433
8434 let wrong_nonce = handler
8438 .handle_control_frame(
8439 &route_ctx(ConnectionId::new(91)).0,
8440 route_open_frame_with_admission_facts(
8441 20,
8442 "target",
8443 unique_project_root("admission-facts"),
8444 Some(subc_control::ConsumerIdentity {
8445 module_id: "fed".to_string(),
8446 launch_nonce: "not-the-real-nonce".to_string(),
8447 }),
8448 None,
8449 ),
8450 )
8451 .await
8452 .unwrap();
8453 assert_eq!(
8454 parse_error(&wrong_nonce[0])["code"],
8455 "bad_consumer_identity",
8456 "a mismatched launch nonce must be refused, not stamped Reserved"
8457 );
8458
8459 let never_spawned = handler
8463 .handle_control_frame(
8464 &route_ctx(ConnectionId::new(92)).0,
8465 route_open_frame_with_admission_facts(
8466 21,
8467 "target",
8468 unique_project_root("admission-facts"),
8469 Some(subc_control::ConsumerIdentity {
8470 module_id: "never-spawned".to_string(),
8471 launch_nonce: "any-nonce".to_string(),
8472 }),
8473 None,
8474 ),
8475 )
8476 .await
8477 .unwrap();
8478 assert_eq!(
8479 parse_error(&never_spawned[0])["code"],
8480 "bad_consumer_identity",
8481 "an unspawned module_id must be refused rather than accepted for lack of a record"
8482 );
8483 }
8484
8485 #[tokio::test]
8508 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
8509 let registry = Arc::new(Registry::default());
8510 let forwarding = Arc::new(ForwardingTable::default());
8511 let supervisor = SupervisorHandle::new();
8512 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8513 let handler =
8514 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8515 .with_supervisor(supervisor);
8516
8517 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
8518 hello_via_sink(
8519 &handler,
8520 &target_ctx,
8521 &mut target_rx,
8522 hello_frame("target", PROTOCOL_VERSION, 1),
8523 )
8524 .await;
8525
8526 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
8527 let route_handler = handler.clone();
8528 let route_task = tokio::spawn(async move {
8529 route_handler
8530 .handle_control_frame(
8531 &client_ctx,
8532 route_open_frame_with_admission_facts(
8533 30,
8534 "target",
8535 unique_project_root("admission-facts"),
8536 Some(subc_control::ConsumerIdentity {
8537 module_id: "fed".to_string(),
8538 launch_nonce: "fed-nonce".to_string(),
8539 }),
8540 None,
8541 ),
8542 )
8543 .await
8544 .unwrap()
8545 });
8546
8547 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8554 .await
8555 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
8556 .expect("module control channel closed before route.bind");
8557 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8558 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
8559 panic!("expected route.bind")
8560 };
8561 assert_eq!(
8562 principal,
8563 Some(Principal::Reserved {
8564 module_id: "fed".to_string()
8565 }),
8566 "a correctly attested consumer must be stamped Reserved for its own id"
8567 );
8568
8569 handler
8570 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8571 .await
8572 .unwrap();
8573 assert!(route_task.await.unwrap().is_empty());
8574 assert!(
8575 matches!(
8576 serde_json::from_slice::<ClientControlResponse>(
8577 &client_rx.recv().await.unwrap().body
8578 )
8579 .unwrap(),
8580 ClientControlResponse::RouteOpen { .. }
8581 ),
8582 "the route must actually open, not merely avoid an error"
8583 );
8584 }
8585
8586 #[tokio::test(start_paused = true)]
8587 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
8588 let registry = Arc::new(Registry::default());
8589 let forwarding = Arc::new(ForwardingTable::default());
8590 let supervisor = SupervisorHandle::new();
8591 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8592 let handler =
8593 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8594 .with_supervisor(supervisor);
8595
8596 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
8597 hello_via_sink(
8598 &handler,
8599 &target_ctx,
8600 &mut target_rx,
8601 hello_frame("target", PROTOCOL_VERSION, 1),
8602 )
8603 .await;
8604
8605 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
8606 let direct_handler = handler.clone();
8607 let direct_open = tokio::spawn(async move {
8608 direct_handler
8609 .handle_control_frame(
8610 &direct_ctx,
8611 route_open_frame(2, "target", unique_project_root("route-census-direct")),
8612 )
8613 .await
8614 .unwrap()
8615 });
8616 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8617 .await
8618 .expect("no direct route.bind within 5s")
8619 .expect("target control channel closed before direct route.bind");
8620 handler
8621 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
8622 .await
8623 .unwrap();
8624 assert!(direct_open.await.unwrap().is_empty());
8625 let _ = direct_rx.recv().await.unwrap();
8626
8627 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
8628 let reserved_handler = handler.clone();
8629 let reserved_open = tokio::spawn(async move {
8630 reserved_handler
8631 .handle_control_frame(
8632 &reserved_ctx,
8633 route_open_frame_with_admission_facts(
8634 3,
8635 "target",
8636 unique_project_root("admission-facts"),
8637 Some(ConsumerIdentity {
8638 module_id: "fed".to_string(),
8639 launch_nonce: "fed-nonce".to_string(),
8640 }),
8641 None,
8642 ),
8643 )
8644 .await
8645 .unwrap()
8646 });
8647 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8648 .await
8649 .expect("no reserved route.bind within 5s")
8650 .expect("target control channel closed before reserved route.bind");
8651 handler
8652 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
8653 .await
8654 .unwrap();
8655 assert!(reserved_open.await.unwrap().is_empty());
8656 let _ = reserved_rx.recv().await.unwrap();
8657
8658 forwarding
8659 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
8660 .unwrap();
8661 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
8662 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
8663 module_id: Some("target".to_string()),
8664 })
8665 .unwrap();
8666 let census_frame =
8667 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
8668 let response = handler
8669 .handle_control_frame(&census_ctx, census_frame)
8670 .await
8671 .unwrap()
8672 .pop()
8673 .unwrap();
8674 let actual: Value = serde_json::from_slice(&response.body).unwrap();
8675 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
8676 assert!(matches!(
8677 decoded,
8678 ClientControlResponse::SupervisorRoutes { .. }
8679 ));
8680 let routes = actual["modules"][0]["routes"].as_array().unwrap();
8681 assert_eq!(routes.len(), 2);
8682 assert!(routes.iter().all(|route| route["draining"] == true));
8683 assert!(
8686 routes.iter().all(|route| route["drain_reason"] == "reload"),
8687 "draining routes must name the drain's reason: {routes:?}"
8688 );
8689 assert!(routes.iter().any(|route| {
8690 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
8691 }));
8692 assert!(routes.iter().any(|route| {
8693 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
8694 }));
8695
8696 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8697 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
8698 if std::env::var_os("UPDATE_GOLDEN").is_some() {
8699 std::fs::write(
8700 &golden_path,
8701 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
8702 )
8703 .unwrap();
8704 }
8705 let expected: Value =
8706 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
8707 assert_eq!(actual, expected);
8708 }
8709
8710 async fn query_live_roots(
8711 handler: &ControlHandler,
8712 module_ctx: &RouteCtx,
8713 ) -> ModuleControlResponseToModule {
8714 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
8715 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
8716 let response = handler
8717 .handle_control_frame(module_ctx, frame)
8718 .await
8719 .unwrap()
8720 .pop()
8721 .unwrap();
8722 serde_json::from_slice(&response.body).unwrap()
8723 }
8724
8725 #[tokio::test(start_paused = true)]
8726 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
8727 let registry = Arc::new(Registry::default());
8728 let forwarding = Arc::new(ForwardingTable::default());
8729 let handler = ControlHandler::with_forwarding(registry, forwarding);
8730 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
8731 hello_via_sink(
8732 &handler,
8733 &target_ctx,
8734 &mut target_rx,
8735 hello_frame("target", PROTOCOL_VERSION, 1),
8736 )
8737 .await;
8738 let root = unique_project_root("live-roots-known");
8739 let path = ProjectRootId::from_path_allowing_missing(root.path())
8740 .unwrap()
8741 .as_path()
8742 .to_path_buf();
8743 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
8744 let open_handler = handler.clone();
8745 let opened = tokio::spawn(async move {
8746 open_handler
8747 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
8748 .await
8749 .unwrap()
8750 });
8751 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8752 .await
8753 .unwrap()
8754 .unwrap();
8755 handler
8756 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
8757 .await
8758 .unwrap();
8759 assert!(opened.await.unwrap().is_empty());
8760 let _ = client_rx.recv().await.unwrap();
8761
8762 let root = unique_project_root("live-roots-pending");
8763 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
8764 .unwrap()
8765 .as_path()
8766 .to_path_buf();
8767 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
8768 let open_handler = handler.clone();
8769 let pending = tokio::spawn(async move {
8770 open_handler
8771 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
8772 .await
8773 .unwrap()
8774 });
8775 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8776 .await
8777 .unwrap()
8778 .unwrap();
8779 let actual = query_live_roots(&handler, &target_ctx).await;
8780 let ModuleControlResponseToModule::LiveRoots {
8781 roots,
8782 unknown_root_bindings,
8783 total_bindings,
8784 } = actual
8785 else {
8786 panic!("expected live roots")
8787 };
8788 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
8789 assert_eq!(unknown_root_bindings, 0);
8790 assert_eq!(
8791 roots.len(),
8792 2,
8793 "root-known arm must retain each canonical root"
8794 );
8795 assert_eq!(
8796 total_bindings,
8797 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
8798 );
8799 let counts = roots
8800 .iter()
8801 .map(|root| (root.project_root.clone(), root.bound, root.pending))
8802 .collect::<Vec<_>>();
8803 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
8804 expected.sort_by(|a, b| a.0.cmp(&b.0));
8805 assert_eq!(
8806 counts, expected,
8807 "roots must sort by path and count pending separately"
8808 );
8809 handler
8810 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
8811 .await
8812 .unwrap();
8813 assert!(pending.await.unwrap().is_empty());
8814 }
8815
8816 #[tokio::test(start_paused = true)]
8817 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
8818 let forwarding = Arc::new(ForwardingTable::default());
8819 let handler =
8820 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
8821 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
8822 hello_via_sink(
8823 &handler,
8824 &target_ctx,
8825 &mut target_rx,
8826 hello_frame("target", PROTOCOL_VERSION, 1),
8827 )
8828 .await;
8829 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
8830 let pending = forwarding
8831 .begin_route_bind_relay_for_test(
8832 client_ctx.connection_id,
8833 client_ctx.egress.clone(),
8834 2,
8835 "target",
8836 )
8837 .unwrap();
8838 forwarding
8839 .complete_pending_relay(
8840 target_ctx.connection_id,
8841 pending.corr,
8842 RouteBindRelayOutcome::Accepted,
8843 )
8844 .unwrap();
8845 let actual = query_live_roots(&handler, &target_ctx).await;
8846 let ModuleControlResponseToModule::LiveRoots {
8847 roots,
8848 unknown_root_bindings,
8849 total_bindings,
8850 } = actual
8851 else {
8852 panic!("expected live roots")
8853 };
8854 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
8855 assert_eq!(
8856 unknown_root_bindings, 1,
8857 "unknown-root arm must not read as no bindings"
8858 );
8859 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
8860 assert_eq!(
8861 total_bindings,
8862 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
8863 );
8864 }
8865
8866 #[tokio::test(start_paused = true)]
8874 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
8875 let forwarding = Arc::new(ForwardingTable::default());
8876 let handler =
8877 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
8878 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
8879 let replies = handler
8880 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
8881 .await
8882 .unwrap();
8883 let queued_by_hello = module_rx.len();
8884
8885 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
8886 let open_handler = handler.clone();
8887 let open = tokio::spawn(async move {
8888 open_handler
8889 .handle_control_frame(
8890 &client_ctx,
8891 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
8892 )
8893 .await
8894 .unwrap()
8895 });
8896 let mut spins = 0;
8898 while module_rx.len() == queued_by_hello {
8899 spins += 1;
8900 assert!(spins < 10_000, "route.open never queued a route.bind");
8901 tokio::task::yield_now().await;
8902 }
8903
8904 for reply in replies {
8906 module_ctx.egress.send(reply).await.unwrap();
8907 }
8908
8909 let first = module_rx.recv().await.unwrap().frame;
8910 assert_eq!(
8911 first.header.ty,
8912 FrameType::HelloAck,
8913 "the first frame a registering module reads must be its HELLO_ACK"
8914 );
8915 assert_eq!(first.header.corr, 7);
8916 let second = module_rx.recv().await.unwrap().frame;
8917 assert_eq!(second.header.ty, FrameType::Request);
8918 assert!(
8919 matches!(
8920 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
8921 ModuleControlRequest::RouteBind { .. }
8922 ),
8923 "the route.bind follows the ack"
8924 );
8925 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
8926
8927 handler
8928 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
8929 .await
8930 .unwrap();
8931 assert!(open.await.unwrap().is_empty());
8932 let _ = client_rx.recv().await.unwrap();
8933 }
8934
8935 #[tokio::test(start_paused = true)]
8936 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
8937 let handler = ControlHandler::with_forwarding(
8938 Arc::new(Registry::default()),
8939 Arc::new(ForwardingTable::default()),
8940 );
8941 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
8942 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
8943 hello_via_sink(
8944 &handler,
8945 &first_ctx,
8946 &mut first_rx,
8947 hello_frame("first", PROTOCOL_VERSION, 1),
8948 )
8949 .await;
8950 hello_via_sink(
8951 &handler,
8952 &second_ctx,
8953 &mut second_rx,
8954 hello_frame("second", PROTOCOL_VERSION, 2),
8955 )
8956 .await;
8957 let root = unique_project_root("second-only");
8958 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
8959 let cloned = handler.clone();
8960 let open = tokio::spawn(async move {
8961 cloned
8962 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
8963 .await
8964 .unwrap()
8965 });
8966 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
8967 .await
8968 .unwrap()
8969 .unwrap();
8970 let first = query_live_roots(&handler, &first_ctx).await;
8971 let second = query_live_roots(&handler, &second_ctx).await;
8972 assert!(
8973 matches!(
8974 first,
8975 ModuleControlResponseToModule::LiveRoots {
8976 total_bindings: 0,
8977 ..
8978 }
8979 ),
8980 "cross-module scope must not expose another module's roots"
8981 );
8982 assert!(
8983 matches!(
8984 second,
8985 ModuleControlResponseToModule::LiveRoots {
8986 total_bindings: 1,
8987 ..
8988 }
8989 ),
8990 "second module must see its pending route"
8991 );
8992 handler
8993 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
8994 .await
8995 .unwrap();
8996 assert!(open.await.unwrap().is_empty());
8997 }
8998
8999 #[tokio::test(start_paused = true)]
9000 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
9001 let handler = ControlHandler::with_forwarding(
9002 Arc::new(Registry::default()),
9003 Arc::new(ForwardingTable::default()),
9004 );
9005 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
9006 hello_via_sink(
9007 &handler,
9008 &target_ctx,
9009 &mut target_rx,
9010 hello_frame("target", PROTOCOL_VERSION, 1),
9011 )
9012 .await;
9013 let actual = query_live_roots(&handler, &target_ctx).await;
9014 let ModuleControlResponseToModule::LiveRoots {
9015 roots,
9016 unknown_root_bindings,
9017 total_bindings,
9018 } = actual
9019 else {
9020 panic!("expected live roots")
9021 };
9022 assert!(roots.is_empty());
9023 assert_eq!(unknown_root_bindings, 0);
9024 assert_eq!(total_bindings, 0);
9025 assert_eq!(
9026 total_bindings,
9027 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
9028 );
9029 }
9030
9031 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
9038 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9039 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
9040 let text = std::fs::read_to_string(&path)
9041 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
9042 let vectors: Vec<(String, Value)> = text
9043 .lines()
9044 .filter(|line| !line.trim().is_empty())
9045 .map(|line| {
9046 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
9047 let id = entry["corpus_id"]
9048 .as_str()
9049 .expect("every vector carries a corpus_id")
9050 .to_string();
9051 (id, entry["package"].clone())
9052 })
9053 .collect();
9054 assert_eq!(
9058 vectors.len(),
9059 3,
9060 "vendored fed corpus changed size; re-sync from subc-federation"
9061 );
9062
9063 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
9078 let richest = vectors
9079 .iter()
9080 .filter_map(|(_, package)| package.as_object())
9081 .map(|object| {
9082 object
9083 .keys()
9084 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
9085 .count()
9086 })
9087 .max()
9088 .unwrap_or(0);
9089 assert!(
9090 richest >= 2,
9091 "vendored corpus no longer carries a package with unmodelled fields, \
9092 so the relay test can no longer distinguish a verbatim relay from a lossy one"
9093 );
9094
9095 vectors
9096 }
9097
9098 #[tokio::test]
9106 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
9107 for (corpus_id, package) in fed_admission_facts_vectors() {
9108 let registry = Arc::new(Registry::default());
9109 let forwarding = Arc::new(ForwardingTable::default());
9110 let supervisor = SupervisorHandle::new();
9111 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9112 let handler =
9113 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9114 .with_supervisor(supervisor)
9115 .with_admission_facts_config(
9116 Some("fed".to_string()),
9117 Some(vec!["target".to_string()]),
9118 );
9119
9120 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
9121 hello_via_sink(
9122 &handler,
9123 &target_ctx,
9124 &mut target_rx,
9125 hello_frame("target", PROTOCOL_VERSION, 1),
9126 )
9127 .await;
9128
9129 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
9130 let route_handler = handler.clone();
9131 let expected = package.clone();
9132 let route_task = tokio::spawn(async move {
9133 route_handler
9134 .handle_control_frame(
9135 &client_ctx,
9136 route_open_frame_with_admission_facts(
9137 20,
9138 "target",
9139 unique_project_root("admission-facts"),
9140 Some(subc_control::ConsumerIdentity {
9141 module_id: "fed".to_string(),
9142 launch_nonce: "fed-nonce".to_string(),
9143 }),
9144 Some(package),
9145 ),
9146 )
9147 .await
9148 .unwrap()
9149 });
9150
9151 let bind_frame = target_rx.recv().await.unwrap();
9152 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9153 let ModuleControlRequest::RouteBind {
9154 admission_facts, ..
9155 } = bind
9156 else {
9157 panic!("{corpus_id}: expected route.bind")
9158 };
9159 assert_eq!(
9160 admission_facts,
9161 Some(expected),
9162 "{corpus_id}: relay must not add, drop or reshape any field"
9163 );
9164
9165 handler
9166 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
9167 .await
9168 .unwrap();
9169 route_task.await.unwrap();
9170 }
9171 }
9172
9173 #[tokio::test]
9174 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
9175 let registry = Arc::new(Registry::default());
9176 let forwarding = Arc::new(ForwardingTable::default());
9177 let supervisor = SupervisorHandle::new();
9178 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9179 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
9180 let handler =
9181 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9182 .with_supervisor(supervisor)
9183 .with_admission_facts_config(
9184 Some("fed".to_string()),
9185 Some(vec!["target".to_string()]),
9186 );
9187
9188 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
9189 hello_via_sink(
9190 &handler,
9191 &target_ctx,
9192 &mut target_rx,
9193 hello_frame("target", PROTOCOL_VERSION, 1),
9194 )
9195 .await;
9196 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
9197 hello_via_sink(
9198 &handler,
9199 &other_ctx,
9200 &mut other_rx,
9201 hello_frame("other", PROTOCOL_VERSION, 2),
9202 )
9203 .await;
9204
9205 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
9206 let expected_facts = facts.clone();
9207 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
9208 let route_handler = handler.clone();
9209 let route_task = tokio::spawn(async move {
9210 route_handler
9211 .handle_control_frame(
9212 &client_ctx,
9213 route_open_frame_with_admission_facts(
9214 10,
9215 "target",
9216 unique_project_root("admission-facts"),
9217 Some(subc_control::ConsumerIdentity {
9218 module_id: "fed".to_string(),
9219 launch_nonce: "fed-nonce".to_string(),
9220 }),
9221 Some(facts.clone()),
9222 ),
9223 )
9224 .await
9225 .unwrap()
9226 });
9227 let bind_frame = target_rx.recv().await.unwrap();
9228 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9229 let ModuleControlRequest::RouteBind {
9230 admission_facts, ..
9231 } = bind
9232 else {
9233 panic!("expected route.bind")
9234 };
9235 assert_eq!(admission_facts, Some(expected_facts));
9236 handler
9237 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
9238 .await
9239 .unwrap();
9240 assert!(route_task.await.unwrap().is_empty());
9241 assert!(matches!(
9242 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
9243 .unwrap(),
9244 ClientControlResponse::RouteOpen { .. }
9245 ));
9246
9247 let direct = handler
9248 .handle_control_frame(
9249 &route_ctx(ConnectionId::new(73)).0,
9250 route_open_frame_with_admission_facts(
9251 11,
9252 "target",
9253 unique_project_root("admission-facts"),
9254 None,
9255 Some(json!({"x": 1})),
9256 ),
9257 )
9258 .await
9259 .unwrap();
9260 assert_eq!(
9261 parse_error(&direct[0])["code"],
9262 "admission_facts_not_permitted"
9263 );
9264
9265 let different_reserved = handler
9266 .handle_control_frame(
9267 &route_ctx(ConnectionId::new(77)).0,
9268 route_open_frame_with_admission_facts(
9269 15,
9270 "target",
9271 unique_project_root("admission-facts"),
9272 Some(subc_control::ConsumerIdentity {
9273 module_id: "other".to_string(),
9274 launch_nonce: "other-nonce".to_string(),
9275 }),
9276 Some(json!({"x": 1})),
9277 ),
9278 )
9279 .await
9280 .unwrap();
9281 assert_eq!(
9282 parse_error(&different_reserved[0])["code"],
9283 "admission_facts_not_permitted"
9284 );
9285
9286 let other_target = handler
9287 .handle_control_frame(
9288 &route_ctx(ConnectionId::new(74)).0,
9289 route_open_frame_with_admission_facts(
9290 12,
9291 "other",
9292 unique_project_root("admission-facts"),
9293 Some(subc_control::ConsumerIdentity {
9294 module_id: "fed".to_string(),
9295 launch_nonce: "fed-nonce".to_string(),
9296 }),
9297 Some(json!({"x": 1})),
9298 ),
9299 )
9300 .await
9301 .unwrap();
9302 assert_eq!(
9303 parse_error(&other_target[0])["code"],
9304 "admission_facts_target_not_allowed"
9305 );
9306
9307 let nonexistent = handler
9308 .handle_control_frame(
9309 &route_ctx(ConnectionId::new(75)).0,
9310 route_open_frame_with_admission_facts(
9311 13,
9312 "missing",
9313 unique_project_root("admission-facts"),
9314 None,
9315 Some(json!({"x": 1})),
9316 ),
9317 )
9318 .await
9319 .unwrap();
9320 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
9321
9322 let described = handler
9323 .handle_control_frame(
9324 &route_ctx(ConnectionId::new(76)).0,
9325 Frame::build(
9326 FrameType::Request,
9327 control_flags(),
9328 0,
9329 0,
9330 14,
9331 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
9332 )
9333 .unwrap(),
9334 )
9335 .await
9336 .unwrap();
9337 let ClientControlResponse::ServerDescribe { capabilities, .. } =
9338 serde_json::from_slice(&described[0].body).unwrap()
9339 else {
9340 panic!("expected server.describe response")
9341 };
9342 assert!(capabilities
9343 .iter()
9344 .any(|cap| cap == "admission_facts_relay_v1"));
9345 }
9346
9347 #[tokio::test]
9348 async fn admission_facts_without_configured_carrier_are_rejected() {
9349 let registry = Arc::new(Registry::default());
9350 let forwarding = Arc::new(ForwardingTable::default());
9351 let handler = ControlHandler::with_forwarding(registry, forwarding);
9352 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
9353 hello_via_sink(
9354 &handler,
9355 &target_ctx,
9356 &mut target_rx,
9357 hello_frame("target", PROTOCOL_VERSION, 1),
9358 )
9359 .await;
9360
9361 let responses = handler
9362 .handle_control_frame(
9363 &route_ctx(ConnectionId::new(79)).0,
9364 route_open_frame_with_admission_facts(
9365 16,
9366 "target",
9367 unique_project_root("admission-facts"),
9368 None,
9369 Some(json!({"x": 1})),
9370 ),
9371 )
9372 .await
9373 .unwrap();
9374 assert_eq!(
9375 parse_error(&responses[0])["code"],
9376 "admission_facts_not_permitted"
9377 );
9378 }
9379
9380 #[tokio::test]
9381 async fn route_open_relays_consumer_capabilities_verbatim() {
9382 let registry = Arc::new(Registry::default());
9383 let forwarding = Arc::new(ForwardingTable::default());
9384 let handler =
9385 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9386 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
9387 hello_via_sink(
9388 &handler,
9389 &module_ctx,
9390 &mut module_rx,
9391 hello_frame("aft", PROTOCOL_VERSION, 7),
9392 )
9393 .await;
9394
9395 let expected = vec!["elicitation".to_string(), "roots".to_string()];
9396 let expected_for_request = expected.clone();
9397 let project_root = unique_project_root("consumer-capabilities-present");
9398 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
9399 let route_handler = handler.clone();
9400 let route_task = tokio::spawn(async move {
9401 route_handler
9402 .handle_control_frame(
9403 &client_ctx,
9404 route_open_frame_with_consumer_capabilities(
9405 401,
9406 "aft",
9407 project_root,
9408 Some(expected_for_request),
9409 ),
9410 )
9411 .await
9412 .unwrap()
9413 });
9414 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9415 .await
9416 .unwrap()
9417 .unwrap();
9418 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9419 let ModuleControlRequest::RouteBind {
9420 consumer_capabilities,
9421 ..
9422 } = bind
9423 else {
9424 panic!("expected route.bind request, got {bind:?}");
9425 };
9426 assert_eq!(consumer_capabilities, Some(expected.clone()));
9427
9428 handler
9429 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9430 .await
9431 .unwrap();
9432 let route_response = route_task.await.unwrap();
9433 assert!(route_response.is_empty());
9434 let published = client_rx.recv().await.unwrap();
9435 assert!(matches!(
9436 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9437 ClientControlResponse::RouteOpen { .. }
9438 ));
9439 }
9440
9441 #[tokio::test]
9442 async fn route_open_without_consumer_capabilities_relays_none() {
9443 let registry = Arc::new(Registry::default());
9444 let forwarding = Arc::new(ForwardingTable::default());
9445 let handler =
9446 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9447 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
9448 hello_via_sink(
9449 &handler,
9450 &module_ctx,
9451 &mut module_rx,
9452 hello_frame("aft", PROTOCOL_VERSION, 7),
9453 )
9454 .await;
9455
9456 let project_root = unique_project_root("consumer-capabilities-absent");
9457 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
9458 let route_handler = handler.clone();
9459 let route_task = tokio::spawn(async move {
9460 route_handler
9461 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
9462 .await
9463 .unwrap()
9464 });
9465 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9466 .await
9467 .unwrap()
9468 .unwrap();
9469 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9470 let ModuleControlRequest::RouteBind {
9471 consumer_capabilities,
9472 ..
9473 } = bind
9474 else {
9475 panic!("expected route.bind request, got {bind:?}");
9476 };
9477 assert_eq!(consumer_capabilities, None);
9478
9479 handler
9480 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9481 .await
9482 .unwrap();
9483 let route_response = route_task.await.unwrap();
9484 assert!(route_response.is_empty());
9485 let published = client_rx.recv().await.unwrap();
9486 assert!(matches!(
9487 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9488 ClientControlResponse::RouteOpen { .. }
9489 ));
9490 }
9491
9492 async fn bind_role_versions_for(
9496 sent: Option<BTreeMap<String, String>>,
9497 connection: u64,
9498 ) -> Option<BTreeMap<String, String>> {
9499 let registry = Arc::new(Registry::default());
9500 let forwarding = Arc::new(ForwardingTable::default());
9501 let handler =
9502 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9503 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
9504 hello_via_sink(
9505 &handler,
9506 &module_ctx,
9507 &mut module_rx,
9508 hello_frame("aft", PROTOCOL_VERSION, 7),
9509 )
9510 .await;
9511 let project_root = unique_project_root("role-versions");
9512 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
9513 let route_handler = handler.clone();
9514 let route_task = tokio::spawn(async move {
9515 route_handler
9516 .handle_control_frame(
9517 &client_ctx,
9518 route_open_frame_with_role_versions(403, "aft", project_root, sent),
9519 )
9520 .await
9521 .unwrap()
9522 });
9523 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9524 .await
9525 .expect("a well-formed route.open reaches the module as a bind")
9526 .unwrap();
9527 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9528 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
9529 panic!("expected route.bind request, got {bind:?}");
9530 };
9531 handler
9532 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9533 .await
9534 .unwrap();
9535 assert!(route_task.await.unwrap().is_empty());
9536 let published = client_rx.recv().await.unwrap();
9537 assert!(matches!(
9538 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9539 ClientControlResponse::RouteOpen { .. }
9540 ));
9541 role_versions
9542 }
9543
9544 #[tokio::test]
9545 async fn route_open_relays_role_versions_verbatim() {
9546 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
9547 assert_eq!(
9548 bind_role_versions_for(Some(sent.clone()), 141).await,
9549 Some(sent)
9550 );
9551 }
9552
9553 #[tokio::test]
9556 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
9557 assert_eq!(bind_role_versions_for(None, 143).await, None);
9558 assert_eq!(
9559 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
9560 None
9561 );
9562 }
9563
9564 #[tokio::test]
9565 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
9566 let registry = Arc::new(Registry::default());
9567 let forwarding = Arc::new(ForwardingTable::default());
9568 let handler =
9569 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9570 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
9571 hello_via_sink(
9572 &handler,
9573 &module_ctx,
9574 &mut module_rx,
9575 hello_frame("aft", PROTOCOL_VERSION, 7),
9576 )
9577 .await;
9578
9579 let nine: BTreeMap<String, String> = (0..9)
9580 .map(|index| (format!("role-{index}"), "v1".to_string()))
9581 .collect();
9582 for (label, malformed) in [
9583 (
9584 "invalid role name",
9585 role_versions(&[("Tool_Provider", "v1")]),
9586 ),
9587 ("invalid version", role_versions(&[("tool-provider", "v0")])),
9588 ("nine entries", nine),
9589 ] {
9590 let responses = tokio::time::timeout(
9593 Duration::from_secs(1),
9594 handler.handle_control_frame(
9595 &route_ctx(ConnectionId::new(148)).0,
9596 route_open_frame_with_role_versions(
9597 404,
9598 "aft",
9599 unique_project_root("role-versions-malformed"),
9600 Some(malformed),
9601 ),
9602 ),
9603 )
9604 .await
9605 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
9606 .unwrap();
9607 assert_eq!(responses.len(), 1, "{label}");
9608 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
9609 let error = parse_error(&responses[0]);
9610 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
9611 assert_eq!(
9612 error["detail"]["field"], "role_versions",
9613 "{label}: {error}"
9614 );
9615 assert!(
9616 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
9617 "{label}: a malformed declaration is terminal"
9618 );
9619 assert!(
9620 module_rx.try_recv().is_err(),
9621 "{label}: the module must never see a bind"
9622 );
9623 }
9624 }
9625
9626 #[tokio::test]
9630 async fn route_role_versions_capability_is_advertised() {
9631 let handler = ControlHandler::new(Arc::new(Registry::default()));
9632 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
9633 let ack = hello_via_sink(
9634 &handler,
9635 &ctx,
9636 &mut rx,
9637 hello_frame("m", PROTOCOL_VERSION, 1),
9638 )
9639 .await;
9640 let ack = parse_ack(&ack);
9641 assert!(
9642 ack.subc_capabilities
9643 .iter()
9644 .any(|c| c == "route-role-versions/v1"),
9645 "{:?}",
9646 ack.subc_capabilities
9647 );
9648
9649 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
9650 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
9651 let reply = handler
9652 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
9653 .await
9654 .unwrap()
9655 .pop()
9656 .unwrap();
9657 let ClientControlResponse::ServerDescribe { capabilities, .. } =
9658 serde_json::from_slice(&reply.body).unwrap()
9659 else {
9660 panic!("not a server.describe reply");
9661 };
9662 assert!(
9663 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
9664 "{capabilities:?}"
9665 );
9666 }
9667
9668 #[tokio::test]
9669 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
9670 let registry = Arc::new(Registry::default());
9671 let forwarding = Arc::new(ForwardingTable::default());
9672 let handler =
9673 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9674 .with_health_probe_timeout(Duration::from_secs(5));
9675 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
9676 hello_via_sink(
9677 &handler,
9678 &module_ctx,
9679 &mut module_rx,
9680 non_routable_hello_frame_with_control_ops(
9681 "mcp",
9682 300,
9683 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9684 ),
9685 )
9686 .await;
9687 assert!(registry
9688 .get_module("mcp")
9689 .unwrap()
9690 .unwrap()
9691 .manifest
9692 .provides
9693 .is_empty());
9694
9695 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
9696 let route_response = handler
9697 .handle_control_frame(
9698 &route_client_ctx,
9699 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
9700 )
9701 .await
9702 .unwrap();
9703 assert_eq!(route_response[0].header.ty, FrameType::Error);
9704 assert_eq!(
9705 parse_error(&route_response[0])["code"],
9706 "target_unavailable"
9707 );
9708 assert!(parse_error(&route_response[0])["message"]
9709 .as_str()
9710 .unwrap()
9711 .contains("does not provide the requested target"));
9712 assert!(module_rx.try_recv().is_err());
9713
9714 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
9715 let health_handler = handler.clone();
9716 let health_task = tokio::spawn(async move {
9717 health_handler
9718 .handle_control_frame(
9719 &health_client_ctx,
9720 supervisor_health_probe_frame(302, "mcp"),
9721 )
9722 .await
9723 .unwrap()
9724 });
9725 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9726 .await
9727 .unwrap()
9728 .unwrap();
9729 assert_eq!(
9730 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
9731 ModuleControlRequest::HealthCheck {}
9732 );
9733 handler
9734 .handle_control_frame(
9735 &module_ctx,
9736 health_response(health_frame.header.corr, HealthStatus::Ok),
9737 )
9738 .await
9739 .unwrap();
9740 let health_response = health_task.await.unwrap();
9741 assert_eq!(health_response[0].header.ty, FrameType::Response);
9742 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
9743 ClientControlResponse::SupervisorHealthProbe {
9744 module_id, status, ..
9745 } => {
9746 assert_eq!(module_id, "mcp");
9747 assert_eq!(status, HealthStatus::Ok);
9748 }
9749 other => panic!("unexpected health response: {other:?}"),
9750 }
9751
9752 forwarding
9757 .cleanup_connection(module_ctx.connection_id)
9758 .unwrap();
9759 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
9760 let cleanup_response = tokio::time::timeout(
9761 Duration::from_millis(200),
9762 handler.handle_control_frame(
9763 &cleanup_probe_ctx,
9764 supervisor_health_probe_frame(303, "mcp"),
9765 ),
9766 )
9767 .await
9768 .expect("probe should fail immediately when the control lane is gone")
9769 .unwrap();
9770 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
9771 assert_eq!(
9772 parse_error(&cleanup_response[0])["code"],
9773 "target_unavailable"
9774 );
9775 assert!(parse_error(&cleanup_response[0])["message"]
9776 .as_str()
9777 .unwrap()
9778 .contains("no module connection"));
9779
9780 handler
9781 .cleanup_connection(module_ctx.connection_id)
9782 .unwrap();
9783 }
9784
9785 #[tokio::test]
9786 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
9787 let registry = Arc::new(Registry::default());
9788 let supervisor_handle = SupervisorHandle::new();
9789 let supervisor =
9790 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
9791 .with_handle(supervisor_handle.clone())
9792 .with_connection_file_path(
9793 std::env::temp_dir()
9794 .join(format!("subc-route-open-warming-{}", std::process::id())),
9795 );
9796 let module = supervisor
9797 .supervise_configured(
9798 ModuleSpec {
9799 module_id: "warming".to_string(),
9800 program: fake_aft_stub_path(),
9801 args: Vec::new(),
9802 env: Vec::new(),
9803 reserved: false,
9804 reserved_prefixes: Vec::new(),
9805 protocol: ModuleProtocol::Subc,
9806 overlap: Default::default(),
9807 },
9808 true,
9809 )
9810 .unwrap();
9811 assert_eq!(module.state().unwrap(), ModuleState::Running);
9812
9813 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9814 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
9815 let response = handler
9816 .handle_control_frame(
9817 &ctx,
9818 route_open_frame(304, "warming", unique_project_root("warming")),
9819 )
9820 .await
9821 .unwrap();
9822 module.stop().await.unwrap();
9823
9824 assert_eq!(response[0].header.ty, FrameType::Error);
9825 let error = parse_error(&response[0]);
9826 assert_eq!(error["code"], "module_warming");
9827 assert!(error["message"]
9828 .as_str()
9829 .unwrap()
9830 .contains("state=running, enabled=true, live=false"));
9831 }
9832
9833 #[test]
9834 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
9835 let handler = ControlHandler::new(Arc::new(Registry::default()));
9836 let capture = EventCapture::default();
9837 let _subscriber =
9838 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9839 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
9840 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
9841 let pending = (0..limit).collect::<Vec<_>>();
9842 let response = handler
9843 .route_open_capacity_refusal(
9844 &ctx,
9845 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
9846 "busy",
9847 pending.len(),
9848 limit,
9849 )
9850 .unwrap();
9851 assert_eq!(parse_error(&response)["code"], "target_unavailable");
9852 let event = capture
9853 .events()
9854 .into_iter()
9855 .find(|event| {
9856 event.target == "control"
9857 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
9858 })
9859 .expect("connection admission refusal event");
9860 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
9861 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
9862 }
9863
9864 #[test]
9865 fn route_open_target_cap_logs_admission_reason_and_capacity() {
9866 let handler = ControlHandler::new(Arc::new(Registry::default()));
9867 let capture = EventCapture::default();
9868 let _subscriber =
9869 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9870 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
9871 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
9872 let guards = (0..limit)
9873 .map(|_| {
9874 handler
9875 .route_bind_concurrency
9876 .try_admit("busy", limit)
9877 .unwrap()
9878 })
9879 .collect::<Vec<_>>();
9880 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
9881 Err(in_flight) => in_flight,
9882 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
9883 };
9884 let response = handler
9885 .route_open_target_capacity_refusal(
9886 &ctx,
9887 &route_open_frame(397, "busy", unique_project_root("target-cap")),
9888 "busy",
9889 in_flight,
9890 )
9891 .unwrap();
9892 assert_eq!(parse_error(&response)["code"], "target_unavailable");
9893 let event = capture
9894 .events()
9895 .into_iter()
9896 .find(|event| {
9897 event.target == "control"
9898 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
9899 })
9900 .expect("target admission refusal event");
9901 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
9902 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
9903 drop(guards);
9904 }
9905
9906 #[tokio::test]
9911 async fn route_open_refusal_names_the_check_that_refused() {
9912 let handler = ControlHandler::new(Arc::new(Registry::default()));
9913 let capture = EventCapture::default();
9914 let _subscriber =
9915 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9916 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
9917 let response = handler
9918 .handle_control_frame(
9919 &ctx,
9920 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
9921 )
9922 .await
9923 .unwrap();
9924
9925 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
9926 let event = capture
9927 .events()
9928 .into_iter()
9929 .find(|event| {
9930 event.target == "control"
9931 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
9932 })
9933 .expect("route.open refusal event");
9934 assert_eq!(
9935 event.fields.get("reason"),
9936 Some(&"\"not_registered\"".to_string())
9937 );
9938 }
9939
9940 #[tokio::test]
9941 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
9942 let registry = Arc::new(Registry::default());
9943 let supervisor_handle = SupervisorHandle::new();
9944 let supervisor =
9945 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
9946 .with_handle(supervisor_handle.clone())
9947 .with_connection_file_path(std::env::temp_dir().join(format!(
9948 "subc-route-open-refusal-info-{}",
9949 std::process::id()
9950 )));
9951 let module = supervisor
9952 .supervise_configured(
9953 ModuleSpec {
9954 module_id: "warming".to_string(),
9955 program: fake_aft_stub_path(),
9956 args: Vec::new(),
9957 env: Vec::new(),
9958 reserved: false,
9959 reserved_prefixes: Vec::new(),
9960 protocol: ModuleProtocol::Subc,
9961 overlap: Default::default(),
9962 },
9963 true,
9964 )
9965 .unwrap();
9966 assert_eq!(module.state().unwrap(), ModuleState::Running);
9967
9968 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9969 assert!(handler
9970 .counters()
9971 .snapshot()
9972 .get("route_open_refused_by_code")
9973 .is_none());
9974 let capture = EventCapture::default();
9975 let _subscriber =
9976 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9977 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
9978 let response = handler
9979 .handle_control_frame(
9980 &ctx,
9981 route_open_frame(394, "warming", unique_project_root("refusal-info")),
9982 )
9983 .await
9984 .unwrap();
9985 module.stop().await.unwrap();
9986
9987 assert_eq!(parse_error(&response[0])["code"], "module_warming");
9988 let event = capture
9989 .events()
9990 .into_iter()
9991 .find(|event| {
9992 event.target == "control"
9993 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
9994 })
9995 .expect("route.open refusal event");
9996 assert_eq!(
9997 event.fields.get("module_id"),
9998 Some(&"\"warming\"".to_string())
9999 );
10000 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
10001 assert_eq!(
10002 event.fields.get("reason"),
10003 Some(&"\"supervised_not_registered\"".to_string())
10004 );
10005 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
10006 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
10007 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
10008 assert_eq!(
10009 handler.counters().snapshot()["route_open_refused_by_code"],
10010 json!({ "module_warming": 1 })
10011 );
10012 }
10013
10014 const OUTAGE_START: &str = "route.open refusing module: not serving";
10015 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
10016
10017 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
10018 capture
10019 .events()
10020 .into_iter()
10021 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
10022 .collect()
10023 }
10024
10025 fn supervise_stub(
10026 registry: &Arc<Registry>,
10027 module_id: &str,
10028 enabled: bool,
10029 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
10030 let supervisor_handle = SupervisorHandle::new();
10031 let supervisor =
10032 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
10033 .with_handle(supervisor_handle.clone())
10034 .with_connection_file_path(std::env::temp_dir().join(format!(
10035 "subc-route-outage-{module_id}-{}",
10036 std::process::id()
10037 )));
10038 let module = supervisor
10039 .supervise_configured(
10040 ModuleSpec {
10041 module_id: module_id.to_string(),
10042 program: fake_aft_stub_path(),
10043 args: Vec::new(),
10044 env: Vec::new(),
10045 reserved: false,
10046 reserved_prefixes: Vec::new(),
10047 protocol: ModuleProtocol::Subc,
10048 overlap: Default::default(),
10049 },
10050 enabled,
10051 )
10052 .unwrap();
10053 (supervisor_handle, module)
10054 }
10055
10056 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
10057 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
10058 module_id: module_id.to_string(),
10059 drain_timeout_ms: Some(50),
10060 })
10061 .unwrap();
10062 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
10063 }
10064
10065 #[test]
10069 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
10070 let registry = Arc::new(Registry::default());
10071 let forwarding = Arc::new(ForwardingTable::default());
10072 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10073 let second = ControlHandler::with_forwarding(registry, forwarding);
10074 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
10075 }
10076
10077 #[tokio::test(flavor = "current_thread")]
10081 async fn route_open_unknown_module_refusals_add_no_outage_state() {
10082 let handler = ControlHandler::new(Arc::new(Registry::default()));
10083 let capture = EventCapture::default();
10084 let _subscriber =
10085 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10086 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
10087 for corr in 0..8 {
10088 let response = handler
10089 .handle_control_frame(
10090 &ctx,
10091 route_open_frame(
10092 380 + corr,
10093 &format!("nobody-{corr}"),
10094 unique_project_root("outage-unknown"),
10095 ),
10096 )
10097 .await
10098 .unwrap();
10099 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
10100 }
10101
10102 assert_eq!(handler.route_outages.tracked_module_count(), 0);
10103 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
10104 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
10105 }
10106
10107 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
10112 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
10113 let registry = Arc::new(Registry::default());
10114 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
10115 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10116 let capture = EventCapture::default();
10117 let _subscriber =
10118 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10119 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
10120 handler.route_outages.record_accepted("outage-restart");
10123
10124 let response = handler
10125 .handle_control_frame(
10126 &ctx,
10127 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
10128 )
10129 .await
10130 .unwrap();
10131 assert_eq!(response[0].header.ty, FrameType::Error);
10132 let starts = outage_lines(&capture, OUTAGE_START);
10133 assert_eq!(starts.len(), 1, "{starts:?}");
10134 assert_eq!(starts[0].level, tracing::Level::WARN);
10135 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
10136 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
10137 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
10138 handler.route_outages.record_accepted("outage-restart");
10139 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
10140
10141 let restart = handler
10142 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
10143 .await
10144 .unwrap();
10145 assert_eq!(
10146 restart[0].header.ty,
10147 FrameType::Response,
10148 "{:?}",
10149 parse_error(&restart[0])
10150 );
10151 handler
10152 .handle_control_frame(
10153 &ctx,
10154 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
10155 )
10156 .await
10157 .unwrap();
10158 module.stop().await.unwrap();
10159
10160 let starts = outage_lines(&capture, OUTAGE_START);
10161 assert_eq!(starts.len(), 2, "{starts:?}");
10162 assert_eq!(starts[1].level, tracing::Level::INFO);
10163 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
10164 }
10165
10166 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
10170 async fn failed_operator_restart_leaves_no_operator_mark() {
10171 let registry = Arc::new(Registry::default());
10172 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
10173 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10174 let capture = EventCapture::default();
10175 let _subscriber =
10176 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10177 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
10178 handler.route_outages.record_accepted("outage-disabled");
10179
10180 let restart = handler
10181 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
10182 .await
10183 .unwrap();
10184 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
10185 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
10186
10187 handler
10188 .handle_control_frame(
10189 &ctx,
10190 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
10191 )
10192 .await
10193 .unwrap();
10194 let starts = outage_lines(&capture, OUTAGE_START);
10195 assert_eq!(starts.len(), 1, "{starts:?}");
10196 assert_eq!(starts[0].level, tracing::Level::WARN);
10197 }
10198
10199 #[tokio::test(flavor = "current_thread")]
10200 async fn route_open_unknown_module_escapes_target_module_id() {
10201 let handler = ControlHandler::new(Arc::new(Registry::default()));
10202 let capture = EventCapture::default();
10203 let _subscriber =
10204 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10205 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
10206 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
10207 let response = handler
10208 .handle_control_frame(
10209 &ctx,
10210 route_open_frame(
10211 395,
10212 hostile_module_id,
10213 unique_project_root("hostile-target-module-id"),
10214 ),
10215 )
10216 .await
10217 .unwrap();
10218
10219 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
10220 let event = capture
10221 .events()
10222 .into_iter()
10223 .find(|event| {
10224 event.target == "control"
10225 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
10226 })
10227 .expect("route.open unknown-module refusal event");
10228 let logged = event.fields.get("module_id").expect("module_id field");
10229 assert!(!logged.bytes().any(|byte| byte < 0x20));
10230 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
10231 }
10232
10233 #[tokio::test(flavor = "current_thread")]
10234 async fn route_open_module_rejection_uses_daemon_counter_key() {
10235 let registry = Arc::new(Registry::default());
10236 let forwarding = Arc::new(ForwardingTable::default());
10237 let handler =
10238 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10239 let module_connection = ConnectionId::new(95);
10240 let (module_ctx, mut module_rx) = route_ctx(module_connection);
10241 hello_via_sink(
10242 &handler,
10243 &module_ctx,
10244 &mut module_rx,
10245 hello_frame("aft", PROTOCOL_VERSION, 395),
10246 )
10247 .await;
10248
10249 let client_connection = ConnectionId::new(96);
10250 let (client_ctx, _client_rx) = route_ctx(client_connection);
10251 let capture = EventCapture::default();
10252 let _subscriber =
10253 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10254 let (route_task, bind) = relay_route_open(
10255 &handler,
10256 client_connection,
10257 &client_ctx.egress,
10258 &mut module_rx,
10259 396,
10260 "aft",
10261 "hostile-module-code",
10262 )
10263 .await;
10264 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
10265 let rejection = Frame::build(
10266 FrameType::Error,
10267 control_flags(),
10268 0,
10269 0,
10270 bind.header.corr,
10271 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
10272 )
10273 .unwrap();
10274 handler
10275 .handle_control_frame(&module_ctx, rejection)
10276 .await
10277 .unwrap();
10278
10279 let response = route_task.await.unwrap();
10280 assert_eq!(parse_error(&response[0])["code"], hostile_code);
10281 let counters = handler.counters().snapshot();
10282 assert_eq!(
10283 counters["route_open_refused_by_code"],
10284 json!({ "module_rejected": 1 })
10285 );
10286 assert!(counters["route_open_refused_by_code"]
10287 .get(hostile_code)
10288 .is_none());
10289
10290 let event = capture
10291 .events()
10292 .into_iter()
10293 .find(|event| {
10294 event.target == "control"
10295 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
10296 })
10297 .expect("route.open module-rejection refusal event");
10298 let logged = event.fields.get("module_code").expect("module_code field");
10299 assert!(!logged.bytes().any(|byte| byte < 0x20));
10300 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
10301 }
10302
10303 #[tokio::test]
10304 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
10305 let registry = Arc::new(Registry::default());
10306 let supervisor_handle = SupervisorHandle::new();
10307 let missing_program = std::env::temp_dir().join(format!(
10308 "subc-route-open-missing-program-{}",
10309 std::process::id()
10310 ));
10311 let supervisor =
10312 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
10313 .with_handle(supervisor_handle.clone());
10314 let module = supervisor
10315 .supervise_configured(
10316 ModuleSpec {
10317 module_id: "failed".to_string(),
10318 program: missing_program,
10319 args: Vec::new(),
10320 env: Vec::new(),
10321 reserved: false,
10322 reserved_prefixes: Vec::new(),
10323 protocol: ModuleProtocol::Subc,
10324 overlap: Default::default(),
10325 },
10326 true,
10327 )
10328 .unwrap();
10329 assert_eq!(module.state().unwrap(), ModuleState::Failed);
10330
10331 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10332 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
10333 let response = handler
10334 .handle_control_frame(
10335 &ctx,
10336 route_open_frame(305, "failed", unique_project_root("failed")),
10337 )
10338 .await
10339 .unwrap();
10340
10341 assert_eq!(response[0].header.ty, FrameType::Error);
10342 let error = parse_error(&response[0]);
10343 assert_eq!(error["code"], "target_unavailable");
10344 assert!(error["message"]
10345 .as_str()
10346 .unwrap()
10347 .contains("state=failed, enabled=true, live=false"));
10348 }
10349
10350 #[tokio::test]
10351 async fn route_open_role_mismatch_remains_target_unavailable() {
10352 let registry = Arc::new(Registry::default());
10353 let handler = ControlHandler::new(Arc::clone(®istry));
10354 handler
10355 .handle_control(
10356 ConnectionId::new(41),
10357 non_routable_hello_frame_with_control_ops("health-only", 306, None),
10358 )
10359 .unwrap();
10360
10361 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
10362 let response = handler
10363 .handle_control_frame(
10364 &ctx,
10365 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
10366 )
10367 .await
10368 .unwrap();
10369
10370 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10371 assert!(parse_error(&response[0])["message"]
10372 .as_str()
10373 .unwrap()
10374 .contains("does not provide the requested target"));
10375 }
10376
10377 #[tokio::test]
10378 async fn route_open_inactive_registration_remains_target_unavailable() {
10379 let registry = Arc::new(Registry::default());
10380 let handler = ControlHandler::new(Arc::clone(®istry));
10381 handler
10382 .handle_control(
10383 ConnectionId::new(43),
10384 hello_frame("inactive", PROTOCOL_VERSION, 308),
10385 )
10386 .unwrap();
10387 assert!(registry
10388 .set_module_state_for_test("inactive", ChannelState::Closed)
10389 .unwrap());
10390
10391 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
10392 let response = handler
10393 .handle_control_frame(
10394 &ctx,
10395 route_open_frame(309, "inactive", unique_project_root("inactive")),
10396 )
10397 .await
10398 .unwrap();
10399
10400 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10401 assert!(parse_error(&response[0])["message"]
10402 .as_str()
10403 .unwrap()
10404 .contains("is not active"));
10405 }
10406
10407 #[tokio::test]
10408 async fn late_health_reply_is_recorded_through_the_module_response_path() {
10409 let registry = Arc::new(Registry::default());
10410 let forwarding = Arc::new(ForwardingTable::default());
10411 let supervisor_handle = SupervisorHandle::new();
10412 let supervisor =
10413 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
10414 .with_forwarding(Arc::clone(&forwarding))
10415 .with_handle(supervisor_handle.clone());
10416 let module = supervisor
10417 .supervise_configured(
10418 crate::ModuleSpec {
10419 module_id: "late-health-response".to_string(),
10420 program: PathBuf::from("disabled-module"),
10421 args: Vec::new(),
10422 env: Vec::new(),
10423 reserved: false,
10424 reserved_prefixes: Vec::new(),
10425 protocol: ModuleProtocol::Subc,
10426 overlap: Default::default(),
10427 },
10428 false,
10429 )
10430 .unwrap();
10431 let handler =
10432 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10433 .with_supervisor(supervisor_handle);
10434 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
10435 handler
10436 .handle_control_frame(
10437 &module_ctx,
10438 hello_frame_with_control_ops(
10439 "late-health-response",
10440 PROTOCOL_VERSION,
10441 7,
10442 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10443 ),
10444 )
10445 .await
10446 .unwrap();
10447 let probe_started_at = Instant::now() - Duration::from_millis(80);
10448 let pending = forwarding
10449 .begin_health_probe_rpc_for(
10450 "late-health-response",
10451 MODULE_CONTROL_OP_HEALTH_CHECK,
10452 probe_started_at,
10453 Instant::now() - Duration::from_millis(1),
10454 )
10455 .unwrap();
10456 assert!(forwarding
10457 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
10458 .unwrap());
10459
10460 let responses = handler
10461 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
10462 .await
10463 .unwrap();
10464
10465 assert!(responses.is_empty());
10466 let health = module.status().unwrap().health;
10467 assert_eq!(health.late_answer_count, 1);
10468 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
10469 }
10470
10471 #[tokio::test]
10472 async fn health_probe_timeout_and_module_death_are_typed() {
10473 let registry = Arc::new(Registry::default());
10474 let forwarding = Arc::new(ForwardingTable::default());
10475 let handler =
10476 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10477 .with_health_probe_timeout(Duration::from_millis(50));
10478 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
10479 hello_via_sink(
10480 &handler,
10481 &module_ctx,
10482 &mut module_rx,
10483 hello_frame_with_control_ops(
10484 "aft",
10485 PROTOCOL_VERSION,
10486 7,
10487 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10488 ),
10489 )
10490 .await;
10491
10492 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10493 let responses = handler
10494 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
10495 .await
10496 .unwrap();
10497 assert_eq!(responses[0].header.ty, FrameType::Error);
10498 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
10499 let _ = module_rx.try_recv();
10500
10501 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
10502 let health_handler = handler.clone();
10503 let death_task = tokio::spawn(async move {
10504 health_handler
10505 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
10506 .await
10507 .unwrap()
10508 });
10509 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10510 .await
10511 .unwrap()
10512 .unwrap();
10513 handler
10514 .cleanup_connection(module_ctx.connection_id)
10515 .unwrap();
10516 let responses = death_task.await.unwrap();
10517 assert_eq!(responses[0].header.ty, FrameType::Error);
10518 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
10519 }
10520
10521 #[test]
10522 fn hello_requires_exact_protocol_version() {
10523 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
10524 let registry = Arc::new(Registry::default());
10525 let handler = ControlHandler::new(Arc::clone(®istry));
10526 let responses = handler
10527 .handle_control(
10528 ConnectionId::new(connection),
10529 hello_frame("aft", offered, 9),
10530 )
10531 .unwrap();
10532
10533 assert_eq!(responses.len(), 1);
10534 assert_eq!(responses[0].header.ty, FrameType::Error);
10535 let error = parse_error(&responses[0]);
10536 assert_eq!(error["code"], "version_unsupported");
10537 assert!(registry.get_module("aft").unwrap().is_none());
10538 assert_eq!(registry.active_registration_count().unwrap(), 0);
10539 }
10540 }
10541
10542 #[test]
10543 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
10544 let registry = Arc::new(Registry::default());
10545 let forwarding = Arc::new(ForwardingTable::default());
10546 let handler =
10547 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10548 let module_connection = ConnectionId::new(301);
10549 let registration = registry
10550 .register_with_control_ops(
10551 manifest("aft-push", PROTOCOL_VERSION),
10552 PROTOCOL_VERSION,
10553 module_connection,
10554 module_baseline_control_ops(),
10555 )
10556 .unwrap();
10557 let (module_tx, _module_rx) = mpsc::channel(8);
10558 let endpoint = forwarding
10559 .register_module_connection(
10560 module_connection,
10561 "aft-push".to_string(),
10562 PROTOCOL_VERSION,
10563 manifest_concurrency(®istration.manifest),
10564 FrameSink::new(module_tx),
10565 )
10566 .unwrap();
10567
10568 let unknown = Frame::build(
10570 FrameType::Push,
10571 control_flags(),
10572 0,
10573 0,
10574 5,
10575 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
10576 )
10577 .unwrap();
10578 let out = handler.handle_status_update(endpoint, unknown).unwrap();
10579 assert!(
10580 out.is_empty(),
10581 "unknown push op must be ignored, got {out:?}"
10582 );
10583
10584 let malformed = Frame::build(
10586 FrameType::Push,
10587 control_flags(),
10588 0,
10589 0,
10590 6,
10591 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
10592 )
10593 .unwrap();
10594 let out = handler.handle_status_update(endpoint, malformed).unwrap();
10595 assert_eq!(out.len(), 1);
10596 assert_eq!(out[0].header.ty, FrameType::Error);
10597 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
10598 }
10599
10600 #[test]
10601 fn hello_rejected_when_connection_already_owns_client_routes() {
10602 let registry = Arc::new(Registry::default());
10603 let forwarding = Arc::new(ForwardingTable::default());
10604 let handler =
10605 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10606 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
10608 let client_connection = ConnectionId::new(202);
10609
10610 let responses = handler
10613 .handle_control(
10614 client_connection,
10615 hello_frame("aft-second", PROTOCOL_VERSION, 9),
10616 )
10617 .unwrap();
10618 assert_eq!(responses[0].header.ty, FrameType::Error);
10619 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
10620 assert!(registry.get_module("aft-second").unwrap().is_none());
10621 }
10622
10623 #[tokio::test]
10624 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
10625 let registry = Arc::new(Registry::default());
10626 let forwarding = Arc::new(ForwardingTable::default());
10627 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
10628 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
10629 hello_via_sink(
10630 &handler,
10631 &module_ctx,
10632 &mut module_rx,
10633 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
10634 )
10635 .await;
10636 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
10637 let pending = forwarding
10638 .begin_route_bind_relay_for_test(
10639 client_ctx.connection_id,
10640 client_ctx.egress.clone(),
10641 2,
10642 "alpha",
10643 )
10644 .unwrap();
10645 forwarding
10646 .complete_pending_relay(
10647 module_ctx.connection_id,
10648 pending.corr,
10649 RouteBindRelayOutcome::Accepted,
10650 )
10651 .unwrap();
10652 client_rx.try_recv().unwrap();
10653 for module_id in ["beta", "alpha"] {
10654 let replies = handler
10655 .handle_control_frame(
10656 &module_ctx,
10657 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
10658 )
10659 .await
10660 .unwrap();
10661 assert_eq!(replies.len(), 1, "second HELLO must be refused");
10662 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
10663 }
10664 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
10665 assert!(registry.get_module("beta").unwrap().is_none());
10666 assert!(matches!(
10667 forwarding
10668 .lookup_data_route(
10669 client_ctx.connection_id,
10670 pending.client_channel,
10671 pending.client_epoch,
10672 )
10673 .unwrap(),
10674 DataRoute::Client(DataRouteState::Bound(_))
10675 ));
10676 assert!(handler
10677 .hello_launch_nonces
10678 .lock()
10679 .unwrap()
10680 .presented(module_ctx.connection_id, Some("alpha-nonce")));
10681 assert!(module_rx.try_recv().is_err());
10682 }
10683
10684 #[test]
10685 fn reserved_module_hello_requires_matching_launch_nonce() {
10686 let registry = Arc::new(Registry::default());
10687 let supervisor = SupervisorHandle::new();
10688 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
10691 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
10692
10693 let no_nonce = handler
10695 .handle_control(
10696 ConnectionId::new(1),
10697 hello_frame("vault", PROTOCOL_VERSION, 1),
10698 )
10699 .unwrap();
10700 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
10701 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
10702 assert!(registry.get_module("vault").unwrap().is_none());
10703
10704 let wrong = handler
10706 .handle_control(
10707 ConnectionId::new(2),
10708 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
10709 )
10710 .unwrap();
10711 assert_eq!(wrong[0].header.ty, FrameType::Error);
10712 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
10713 assert!(registry.get_module("vault").unwrap().is_none());
10714
10715 let ok = handler
10717 .handle_control(
10718 ConnectionId::new(3),
10719 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
10720 )
10721 .unwrap();
10722 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
10723 assert!(registry.get_module("vault").unwrap().is_some());
10724 }
10725
10726 #[test]
10727 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
10728 let registry = Arc::new(Registry::default());
10729 let supervisor = SupervisorHandle::new();
10730 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
10731 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
10732 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
10733
10734 let squat = handler
10735 .handle_control(
10736 ConnectionId::new(1),
10737 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
10738 )
10739 .unwrap();
10740 assert_eq!(squat[0].header.ty, FrameType::Error);
10741 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
10742 assert!(parse_error(&squat[0])["message"]
10743 .as_str()
10744 .unwrap()
10745 .contains("fed:"));
10746
10747 let accepted_peer = handler
10748 .handle_control(
10749 ConnectionId::new(2),
10750 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
10751 )
10752 .unwrap();
10753 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
10754
10755 let accepted_short = handler
10756 .handle_control(
10757 ConnectionId::new(3),
10758 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
10759 )
10760 .unwrap();
10761 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
10762
10763 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
10764 let response = handler
10765 .handle_control(
10766 ConnectionId::new(conn),
10767 hello_frame(module_id, PROTOCOL_VERSION, conn),
10768 )
10769 .unwrap();
10770 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
10771 }
10772 }
10773
10774 #[test]
10775 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
10776 let registry = Arc::new(Registry::default());
10777 let supervisor = SupervisorHandle::new();
10778 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
10779 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
10780 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
10781 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
10782
10783 let owner_nonce = handler
10784 .handle_control(
10785 ConnectionId::new(1),
10786 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
10787 )
10788 .unwrap();
10789 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
10790 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
10791 assert!(registry.get_module("fed:special").unwrap().is_none());
10792
10793 let exact_nonce = handler
10794 .handle_control(
10795 ConnectionId::new(2),
10796 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
10797 )
10798 .unwrap();
10799 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
10800 assert!(registry.get_module("fed:special").unwrap().is_some());
10801 }
10802
10803 #[test]
10804 fn non_reserved_module_ignores_launch_nonce() {
10805 let registry = Arc::new(Registry::default());
10806 let handler = ControlHandler::new(Arc::clone(®istry));
10809 let no_nonce = handler
10810 .handle_control(
10811 ConnectionId::new(1),
10812 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
10813 )
10814 .unwrap();
10815 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
10816 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
10817
10818 let echoed_nonce = handler
10819 .handle_control(
10820 ConnectionId::new(2),
10821 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
10822 )
10823 .unwrap();
10824 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
10825 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
10826 }
10827
10828 #[test]
10829 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
10830 let handler = ControlHandler::default();
10831 let conn = ConnectionId::new(1);
10832 let malformed = Frame::build(
10833 FrameType::Hello,
10834 control_flags(),
10835 0,
10836 0,
10837 3,
10838 b"{not json".to_vec(),
10839 )
10840 .unwrap();
10841
10842 let error = handler.handle_control(conn, malformed).unwrap();
10843 assert_eq!(error[0].header.ty, FrameType::Error);
10844 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
10845
10846 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
10847 let pong = handler.handle_control(conn, ping).unwrap();
10848 assert_eq!(pong[0].header.ty, FrameType::Pong);
10849 assert_eq!(pong[0].header.corr, 4);
10850 }
10851
10852 #[test]
10853 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
10854 let registry = Arc::new(Registry::default());
10855 let handler = ControlHandler::new(Arc::clone(®istry));
10856
10857 handler
10858 .handle_control(
10859 ConnectionId::new(1),
10860 hello_frame("aft", PROTOCOL_VERSION, 1),
10861 )
10862 .unwrap();
10863 let duplicate = handler
10864 .handle_control(
10865 ConnectionId::new(2),
10866 hello_frame("aft", PROTOCOL_VERSION, 2),
10867 )
10868 .unwrap();
10869
10870 assert_eq!(duplicate[0].header.ty, FrameType::Error);
10871 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
10872 let registration = registry.get_module("aft").unwrap().unwrap();
10873 assert_eq!(registration.connection_id, ConnectionId::new(1));
10874 }
10875
10876 #[test]
10877 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
10878 let registry = Arc::new(Registry::default());
10879 let forwarding = Arc::new(ForwardingTable::default());
10880 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
10881 let handler =
10882 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10883 .with_process_liveness(process_liveness);
10884 let (ctx, route_channel, route_epoch) =
10885 bind_liveness_route(®istry, &forwarding, "aft-dead");
10886 let responses = handler
10887 .handle_route_poll(
10888 &ctx,
10889 route_poll_frame(41, PollKind::Liveness, route_channel),
10890 route_channel,
10891 route_epoch,
10892 PollKind::Liveness,
10893 )
10894 .unwrap();
10895
10896 assert_eq!(responses.len(), 1);
10897 assert_eq!(responses[0].header.ty, FrameType::Response);
10898 assert_route_poll_liveness(&responses[0], false);
10899 }
10900
10901 #[test]
10902 fn liveness_poll_without_process_source_uses_bound_route() {
10903 let registry = Arc::new(Registry::default());
10904 let forwarding = Arc::new(ForwardingTable::default());
10905 let handler =
10906 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10907 let (ctx, route_channel, route_epoch) =
10908 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
10909 let responses = handler
10910 .handle_route_poll(
10911 &ctx,
10912 route_poll_frame(42, PollKind::Liveness, route_channel),
10913 route_channel,
10914 route_epoch,
10915 PollKind::Liveness,
10916 )
10917 .unwrap();
10918
10919 assert_route_poll_liveness(&responses[0], true);
10920 }
10921
10922 #[test]
10923 fn liveness_poll_untracked_process_source_uses_bound_route() {
10924 let registry = Arc::new(Registry::default());
10925 let forwarding = Arc::new(ForwardingTable::default());
10926 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
10927 let handler =
10928 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10929 .with_process_liveness(process_liveness);
10930 let (ctx, route_channel, route_epoch) =
10931 bind_liveness_route(®istry, &forwarding, "aft-untracked");
10932 let responses = handler
10933 .handle_route_poll(
10934 &ctx,
10935 route_poll_frame(43, PollKind::Liveness, route_channel),
10936 route_channel,
10937 route_epoch,
10938 PollKind::Liveness,
10939 )
10940 .unwrap();
10941
10942 assert_route_poll_liveness(&responses[0], true);
10943 }
10944
10945 #[tokio::test]
10946 async fn unknown_op_returns_unknown_control_op() {
10947 let handler = ControlHandler::default();
10948 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
10949 let request = Frame::build(
10950 FrameType::Request,
10951 control_flags(),
10952 0,
10953 0,
10954 55,
10955 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
10956 )
10957 .unwrap();
10958
10959 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
10960
10961 assert_eq!(response.len(), 1);
10962 assert_eq!(response[0].header.ty, FrameType::Error);
10963 assert_eq!(response[0].header.corr, 55);
10964 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
10965 }
10966
10967 #[tokio::test]
10968 async fn supervisor_provenance_rejects_unknown_exact_module() {
10969 let handler = ControlHandler::default();
10970 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
10971 let request = Frame::build(
10972 FrameType::Request,
10973 control_flags(),
10974 0,
10975 0,
10976 57,
10977 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
10978 )
10979 .unwrap();
10980
10981 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
10982
10983 assert_eq!(response.len(), 1);
10984 assert_eq!(response[0].header.ty, FrameType::Error);
10985 assert_eq!(response[0].header.corr, 57);
10986 let error = parse_error(&response[0]);
10987 assert_eq!(error["code"], "unknown_module");
10988 assert_eq!(error["message"], "module_id 'missing' is not supervised");
10989 }
10990
10991 #[test]
10992 fn provenance_probe_override_keeps_handler_tests_deterministic() {
10993 let expected = subc_control::RunningImageAgreement::Unavailable {
10994 reason: subc_control::RunningImageUnavailableReason::HashFailed,
10995 };
10996 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
10997 assert_eq!(handler.provenance_probe_override, Some(expected));
10998 }
10999
11000 #[test]
11001 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
11002 let verdict = reload_verdict(
11003 std::path::Path::new("/bin/new"),
11004 Some(std::path::Path::new("/bin/old")),
11005 subc_control::RunningImageAgreement::Unavailable {
11006 reason: subc_control::RunningImageUnavailableReason::HashFailed,
11007 },
11008 );
11009 assert!(matches!(
11010 verdict.path,
11011 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
11012 if configured == std::path::Path::new("/bin/new")
11013 && spawned_from == std::path::Path::new("/bin/old")
11014 ));
11015 }
11016
11017 #[test]
11018 fn reload_verdict_detects_replaced_image_at_same_path() {
11019 let image = subc_control::RunningImageAgreement::Mismatch {
11020 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
11021 digest: "old".into(),
11022 },
11023 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
11024 digest: "new".into(),
11025 },
11026 };
11027 let verdict = reload_verdict(
11028 std::path::Path::new("/bin/same"),
11029 Some(std::path::Path::new("/bin/same")),
11030 image.clone(),
11031 );
11032 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
11033 assert_eq!(verdict.image, image);
11034 }
11035
11036 #[test]
11037 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
11038 let image = subc_control::RunningImageAgreement::Unavailable {
11039 reason: subc_control::RunningImageUnavailableReason::NotRunning,
11040 };
11041 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
11042 assert_eq!(
11043 verdict.path,
11044 subc_control::ReloadPathAgreement::Unavailable {
11045 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
11046 }
11047 );
11048 assert_eq!(verdict.image, image);
11049
11050 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
11051 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
11052 };
11053 let verdict = reload_verdict(
11054 std::path::Path::new("/bin/same"),
11055 Some(std::path::Path::new("/bin/same")),
11056 unconfirmed.clone(),
11057 );
11058 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
11059 assert_eq!(verdict.image, unconfirmed);
11060 }
11061
11062 #[test]
11063 fn reload_verdict_preserves_each_image_unavailability_reason() {
11064 use subc_control::RunningImageUnavailableReason as Reason;
11065
11066 for reason in [
11067 Reason::NotRunning,
11068 Reason::UnsupportedPlatform,
11069 Reason::RunningExecutableUnreadable,
11070 Reason::SpawnedPathUnreadable,
11071 Reason::HashFailed,
11072 Reason::ProcessIdentityUnconfirmed,
11073 Reason::Unknown("future_probe_reason".to_string()),
11074 ] {
11075 let image = subc_control::RunningImageAgreement::Unavailable {
11076 reason: reason.clone(),
11077 };
11078 let verdict = reload_verdict(
11079 std::path::Path::new("/bin/same"),
11080 Some(std::path::Path::new("/bin/same")),
11081 image.clone(),
11082 );
11083 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
11084 assert_eq!(verdict.image, image, "{reason:?}");
11085 }
11086 }
11087
11088 #[tokio::test]
11089 async fn malformed_control_bodies_return_invalid_control_body() {
11090 let handler = ControlHandler::default();
11091 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
11092
11093 for (corr, body) in [
11094 (56, br#"{"route_channel":1}"#.as_slice()),
11095 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
11096 (
11097 58,
11098 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
11099 ),
11100 ] {
11101 let request = Frame::build(
11102 FrameType::Request,
11103 control_flags(),
11104 0,
11105 0,
11106 corr,
11107 body.to_vec(),
11108 )
11109 .unwrap();
11110 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
11111
11112 assert_eq!(response.len(), 1);
11113 assert_eq!(response[0].header.ty, FrameType::Error);
11114 assert_eq!(response[0].header.corr, corr);
11115 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
11116 }
11117 }
11118
11119 #[tokio::test]
11120 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
11121 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
11122 let registry = Arc::new(Registry::default());
11123 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
11124 let router = Router::with_control_handler(Arc::clone(&control));
11125 let connection = router.begin_connection();
11126 let (ctx, mut rx) = route_ctx(connection.id());
11127
11128 router
11129 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
11130 .await
11131 .unwrap();
11132 let response = rx.recv().await.unwrap();
11133 let ack = parse_ack(&response);
11134 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
11135 let channel = 1;
11136
11137 let goodbye =
11138 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
11139 router.route_for_connection(&ctx, goodbye).await.unwrap();
11140 assert!(rx.try_recv().is_err());
11141 assert!(registry.get_module("aft").unwrap().is_none());
11142
11143 router
11144 .route_for_connection(&ctx, channel_request(channel, 13))
11145 .await
11146 .unwrap();
11147 let error_frame = rx.recv().await.unwrap();
11148 assert_eq!(error_frame.header.ty, FrameType::Error);
11149 assert_eq!(error_frame.header.channel, channel);
11150 let captured = crate::router::test_log::captured_logs(&logs);
11151 assert_eq!(
11152 captured
11153 .lines()
11154 .filter(|line| {
11155 line.contains("module_id=aft")
11156 && line.contains("reason=explicit_goodbye")
11157 && line.contains("module registration ended")
11158 })
11159 .count(),
11160 1,
11161 "unexpected GOODBYE registry log: {captured}"
11162 );
11163 }
11164
11165 #[tokio::test]
11166 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
11167 let registry = Arc::new(Registry::default());
11168 let handler = ControlHandler::new(registry).with_capability_config(
11169 [("prov".to_string(), true), ("cons".to_string(), true)],
11170 BTreeMap::new(),
11171 );
11172 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
11173 register_capability_manifest(
11174 &handler,
11175 &provider_ctx,
11176 &mut provider_rx,
11177 capability_manifest("prov", &["thing/v1"], &[]),
11178 1,
11179 )
11180 .await;
11181 let mut consumer = capability_manifest("cons", &[], &[]);
11182 consumer.capabilities.as_mut().unwrap().requires.push(
11183 subc_protocol::manifest::CapabilityRequirement {
11184 capability: "thing/v1".to_string(),
11185 need: subc_protocol::manifest::CapabilityNeed::Required,
11186 },
11187 );
11188 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
11189 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
11190 assert_eq!(
11191 handler.capability_evaluator.verdict("cons", "thing/v1"),
11192 Some(CapabilityVerdict::Provided)
11193 );
11194 let (mut client_rx, _) = open_route_for_capability_test(
11195 &handler,
11196 &provider_ctx,
11197 &mut provider_rx,
11198 703,
11199 3,
11200 "prov",
11201 None,
11202 )
11203 .await;
11204 let goodbye =
11205 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
11206 handler
11207 .handle_control_frame(&provider_ctx, goodbye)
11208 .await
11209 .unwrap();
11210 assert_eq!(
11211 handler.capability_evaluator.verdict("cons", "thing/v1"),
11212 Some(CapabilityVerdict::NeverProvided)
11213 );
11214 let closed = client_rx
11215 .try_recv()
11216 .expect("GOODBYE pushes route.closed before route GOODBYE");
11217 assert!(
11218 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
11219 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
11220 );
11221 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
11222 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
11223 }
11224
11225 #[tokio::test]
11226 async fn dropping_router_connection_releases_registration() {
11227 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
11228 let registry = Arc::new(Registry::default());
11229 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
11230 let router = Router::with_control_handler(Arc::clone(&control));
11231 let connection = router.begin_connection();
11232 let connection_id = connection.id();
11233 let (ctx, mut rx) = route_ctx(connection_id);
11234
11235 router
11236 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
11237 .await
11238 .unwrap();
11239 let response = rx.recv().await.unwrap();
11240 let ack = parse_ack(&response);
11241 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
11242 assert!(registry.get_module("aft").unwrap().is_some());
11243
11244 drop(connection);
11245
11246 assert!(registry.get_module("aft").unwrap().is_none());
11247 assert_eq!(registry.active_registration_count().unwrap(), 0);
11248
11249 control
11250 .cleanup_connection(ConnectionId::new(u64::MAX))
11251 .unwrap();
11252 let captured = crate::router::test_log::captured_logs(&logs);
11253 println!("captured registry lifecycle logs:\n{captured}");
11254 let events: Vec<_> = captured
11255 .lines()
11256 .filter(|line| {
11257 line.contains("module registered module_id=aft ")
11258 || line.contains("module registration ended")
11259 })
11260 .collect();
11261 assert_eq!(
11262 events.len(),
11263 2,
11264 "unexpected registry lifecycle logs: {captured}"
11265 );
11266 assert!(events[0].contains("module registered module_id=aft "));
11267 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
11268 assert!(events[1].contains(&format!(
11269 "module_id=aft connection_id={}",
11270 connection_id.get()
11271 )));
11272 assert!(events[1].contains("reason=connection_closed"));
11273 assert!(events[1].contains("module registration ended"));
11274 assert_eq!(
11275 captured
11276 .lines()
11277 .filter(|line| line.contains("module registered module_id=aft "))
11278 .count(),
11279 1,
11280 "legacy registration admission line must appear once: {captured}"
11281 );
11282 }
11283
11284 #[tokio::test]
11285 async fn hello_registration_keeps_legacy_module_registered_line_once() {
11286 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
11287 let registry = Arc::new(Registry::default());
11288 let control = ControlHandler::new(Arc::clone(®istry));
11289 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
11290 hello_via_sink(
11291 &control,
11292 &ctx,
11293 &mut rx,
11294 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
11295 )
11296 .await;
11297
11298 let captured = crate::router::test_log::captured_logs(&logs);
11299 let admissions: Vec<_> = captured
11300 .lines()
11301 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
11302 .collect();
11303 assert_eq!(
11304 admissions.len(),
11305 1,
11306 "legacy admission line must remain exactly once: {captured}"
11307 );
11308 assert!(admissions[0].contains("routable_provider=true"));
11309 assert!(admissions[0].contains("connection_id=777"));
11310 }
11311
11312 fn capability_manifest(
11313 module_id: &str,
11314 provides: &[&str],
11315 must_never_reach: &[&str],
11316 ) -> ModuleManifest {
11317 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
11318 manifest.capabilities = Some(CapabilityDeclarations {
11319 provides: provides
11320 .iter()
11321 .map(|capability| (*capability).to_string())
11322 .collect(),
11323 requires: Vec::new(),
11324 must_never_reach: must_never_reach
11325 .iter()
11326 .map(|capability| (*capability).to_string())
11327 .collect(),
11328 });
11329 manifest
11330 }
11331
11332 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
11333 Frame::build(
11334 FrameType::Hello,
11335 control_flags(),
11336 0,
11337 0,
11338 corr,
11339 serde_json::to_vec(&ModuleHelloBody {
11340 protocol_ver: manifest.protocol_ver,
11341 manifest,
11342 control_ops: None,
11343 launch_nonce: None,
11344 })
11345 .expect("capability test HELLO serializes"),
11346 )
11347 .expect("capability test HELLO frame builds")
11348 }
11349
11350 fn catalog_update_with_capabilities_frame(
11351 corr: u64,
11352 capabilities: CapabilityDeclarations,
11353 ) -> Frame {
11354 Frame::build(
11355 FrameType::Request,
11356 control_flags(),
11357 0,
11358 0,
11359 corr,
11360 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
11361 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
11362 capabilities: Some(capabilities),
11363 ready: None,
11364 })
11365 .expect("capability catalog.update serializes"),
11366 )
11367 .expect("capability catalog.update frame builds")
11368 }
11369
11370 async fn register_capability_manifest(
11371 handler: &ControlHandler,
11372 ctx: &RouteCtx,
11373 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
11374 manifest: ModuleManifest,
11375 corr: u64,
11376 ) {
11377 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
11378 }
11379
11380 async fn open_route_for_capability_test(
11381 handler: &ControlHandler,
11382 target_ctx: &RouteCtx,
11383 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
11384 client_connection_id: u64,
11385 corr: u64,
11386 target_module_id: &str,
11387 consumer_identity: Option<ConsumerIdentity>,
11388 ) -> (
11389 mpsc::Receiver<crate::router::OutboundFrame>,
11390 ModuleControlRequest,
11391 ) {
11392 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
11393 let route_handler = handler.clone();
11394 let target_module_id = target_module_id.to_string();
11395 let route_task = tokio::spawn(async move {
11396 route_handler
11397 .handle_control_frame(
11398 &client_ctx,
11399 route_open_frame_with_admission_facts(
11400 corr,
11401 &target_module_id,
11402 unique_project_root("admission-facts"),
11403 consumer_identity,
11404 None,
11405 ),
11406 )
11407 .await
11408 .expect("capability test route.open succeeds")
11409 });
11410 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
11411 .await
11412 .expect("capability test route.open must reach route.bind")
11413 .expect("target control receiver stays open");
11414 let bind_request: ModuleControlRequest =
11415 serde_json::from_slice(&bind.body).expect("route.bind decodes");
11416 handler
11417 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
11418 .await
11419 .expect("capability test route.bind ACK succeeds");
11420 assert!(route_task.await.expect("route.open task joins").is_empty());
11421 let opened = client_rx
11422 .recv()
11423 .await
11424 .expect("successful route.open publishes a response");
11425 assert!(matches!(
11426 serde_json::from_slice::<ClientControlResponse>(&opened.body),
11427 Ok(ClientControlResponse::RouteOpen { .. })
11428 ));
11429 (client_rx, bind_request)
11430 }
11431
11432 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
11433 assert_eq!(frame.header.ty, FrameType::Push);
11434 assert_eq!(frame.header.channel, 0);
11435 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
11436 .expect("route.closed control push decodes");
11437 let ClientControlPush::RouteClosed { channels, .. } = &push else {
11438 panic!("expected route.closed");
11439 };
11440 assert_eq!(channels.len(), 1, "exactly one violating route closed");
11441 let channels = channels.clone();
11442 assert_eq!(
11443 push,
11444 ClientControlPush::RouteClosed {
11445 module_id: target_module_id.to_string(),
11446 channels,
11447 reason: RouteCloseReason::CapabilityDenied,
11448 drained: false,
11449 abandoned: 0,
11450 excluded_subscriptions: 0,
11451 terminal: Some(false),
11452 }
11453 );
11454 }
11455
11456 #[tokio::test]
11457 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
11458 let registry = Arc::new(Registry::default());
11459 let forwarding = Arc::new(ForwardingTable::default());
11460 let supervisor = SupervisorHandle::new();
11461 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11462 let handler =
11463 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11464 .with_supervisor(supervisor);
11465 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
11466 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
11467 register_capability_manifest(
11468 &handler,
11469 &target_ctx,
11470 &mut target_rx,
11471 capability_manifest("target", &["credentials-provider/v1"], &[]),
11472 1,
11473 )
11474 .await;
11475 register_capability_manifest(
11476 &handler,
11477 &opener_ctx,
11478 &mut opener_rx,
11479 capability_manifest("opener", &[], &["credentials-provider/v1"]),
11480 2,
11481 )
11482 .await;
11483
11484 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
11485 let replies = handler
11486 .handle_control_frame(
11487 &client_ctx,
11488 route_open_frame_with_admission_facts(
11489 3,
11490 "target",
11491 unique_project_root("admission-facts"),
11492 Some(ConsumerIdentity {
11493 module_id: "opener".to_string(),
11494 launch_nonce: "opener-nonce".to_string(),
11495 }),
11496 None,
11497 ),
11498 )
11499 .await
11500 .expect("denied route.open returns a typed frame");
11501 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
11502 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
11503 assert!(
11504 target_rx.try_recv().is_err(),
11505 "forbidden route.open must not relay route.bind"
11506 );
11507 }
11508
11509 #[tokio::test]
11510 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
11511 let registry = Arc::new(Registry::default());
11512 let forwarding = Arc::new(ForwardingTable::default());
11513 let supervisor = SupervisorHandle::new();
11514 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11515 let handler =
11516 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11517 .with_supervisor(supervisor);
11518 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
11519 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
11520 register_capability_manifest(
11521 &handler,
11522 &target_ctx,
11523 &mut target_rx,
11524 capability_manifest("target", &["credentials-provider/v1"], &[]),
11525 1,
11526 )
11527 .await;
11528 register_capability_manifest(
11529 &handler,
11530 &old_opener_ctx,
11531 &mut old_opener_rx,
11532 capability_manifest("opener", &[], &[]),
11533 2,
11534 )
11535 .await;
11536 let (mut client_rx, _) = open_route_for_capability_test(
11537 &handler,
11538 &target_ctx,
11539 &mut target_rx,
11540 712,
11541 3,
11542 "target",
11543 Some(ConsumerIdentity {
11544 module_id: "opener".to_string(),
11545 launch_nonce: "opener-nonce".to_string(),
11546 }),
11547 )
11548 .await;
11549 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
11550
11551 handler
11552 .cleanup_connection(old_opener_ctx.connection_id)
11553 .expect("old opener registration cleans up");
11554 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
11555 register_capability_manifest(
11556 &handler,
11557 &new_opener_ctx,
11558 &mut new_opener_rx,
11559 capability_manifest("opener", &[], &["credentials-provider/v1"]),
11560 4,
11561 )
11562 .await;
11563
11564 assert_capability_denied_push(
11565 client_rx
11566 .try_recv()
11567 .expect("HELLO deny addition must emit route.closed")
11568 .frame,
11569 "target",
11570 );
11571 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
11572 assert!(matches!(
11573 target_rx.try_recv(),
11574 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
11575 ));
11576 }
11577
11578 #[tokio::test]
11579 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
11580 let registry = Arc::new(Registry::default());
11581 let forwarding = Arc::new(ForwardingTable::default());
11582 let supervisor = SupervisorHandle::new();
11583 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11584 let handler =
11585 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11586 .with_supervisor(supervisor);
11587 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
11588 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
11589 register_capability_manifest(
11590 &handler,
11591 &target_ctx,
11592 &mut target_rx,
11593 capability_manifest("target", &[], &[]),
11594 1,
11595 )
11596 .await;
11597 register_capability_manifest(
11598 &handler,
11599 &opener_ctx,
11600 &mut opener_rx,
11601 capability_manifest("opener", &[], &["credentials-provider/v1"]),
11602 2,
11603 )
11604 .await;
11605 let (mut client_rx, _) = open_route_for_capability_test(
11606 &handler,
11607 &target_ctx,
11608 &mut target_rx,
11609 722,
11610 3,
11611 "target",
11612 Some(ConsumerIdentity {
11613 module_id: "opener".to_string(),
11614 launch_nonce: "opener-nonce".to_string(),
11615 }),
11616 )
11617 .await;
11618 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
11619
11620 let replies = handler
11621 .handle_control_frame(
11622 &target_ctx,
11623 catalog_update_with_capabilities_frame(
11624 4,
11625 CapabilityDeclarations {
11626 provides: vec!["credentials-provider/v1".to_string()],
11627 requires: Vec::new(),
11628 must_never_reach: Vec::new(),
11629 },
11630 ),
11631 )
11632 .await
11633 .expect("claim catalog.update succeeds");
11634 assert!(matches!(
11635 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
11636 Ok(ModuleControlResponseToModule::CatalogUpdate {})
11637 ));
11638 assert_capability_denied_push(
11639 client_rx
11640 .try_recv()
11641 .expect("claim addition must emit route.closed")
11642 .frame,
11643 "target",
11644 );
11645 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
11646 assert!(matches!(
11647 target_rx.try_recv(),
11648 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
11649 ));
11650 }
11651
11652 #[tokio::test]
11653 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
11654 let registry = Arc::new(Registry::default());
11655 let forwarding = Arc::new(ForwardingTable::default());
11656 let supervisor = SupervisorHandle::new();
11657 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11658 let handler =
11659 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11660 .with_supervisor(supervisor);
11661 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
11662 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
11663 register_capability_manifest(
11664 &handler,
11665 &target_ctx,
11666 &mut target_rx,
11667 capability_manifest("target", &["credentials-provider/v1"], &[]),
11668 1,
11669 )
11670 .await;
11671 register_capability_manifest(
11672 &handler,
11673 &opener_ctx,
11674 &mut opener_rx,
11675 capability_manifest("opener", &[], &[]),
11676 2,
11677 )
11678 .await;
11679 let (mut client_rx, _) = open_route_for_capability_test(
11680 &handler,
11681 &target_ctx,
11682 &mut target_rx,
11683 732,
11684 3,
11685 "target",
11686 Some(ConsumerIdentity {
11687 module_id: "opener".to_string(),
11688 launch_nonce: "opener-nonce".to_string(),
11689 }),
11690 )
11691 .await;
11692 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
11693
11694 handler
11695 .handle_control_frame(
11696 &target_ctx,
11697 catalog_update_with_capabilities_frame(
11698 4,
11699 CapabilityDeclarations {
11700 provides: Vec::new(),
11701 requires: Vec::new(),
11702 must_never_reach: Vec::new(),
11703 },
11704 ),
11705 )
11706 .await
11707 .expect("claim removal catalog.update succeeds");
11708 assert_eq!(
11709 forwarding.active_binding_count().unwrap(),
11710 1,
11711 "removing an attested target claim must leave the route census unchanged"
11712 );
11713 assert!(
11714 client_rx.try_recv().is_err(),
11715 "claim removal must not emit route.closed capability_denied"
11716 );
11717 assert!(
11718 target_rx.try_recv().is_err(),
11719 "claim removal must not send the target a route GOODBYE"
11720 );
11721 }
11722
11723 #[tokio::test]
11726 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
11727 let registry = Arc::new(Registry::default());
11728 let forwarding = Arc::new(ForwardingTable::default());
11729 let supervisor = SupervisorHandle::new();
11730 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11731 let handler =
11732 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11733 .with_supervisor(supervisor);
11734 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
11735 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
11736 register_capability_manifest(
11737 &handler,
11738 &target_ctx,
11739 &mut target_rx,
11740 capability_manifest("target", &["credentials-provider/v1"], &[]),
11741 1,
11742 )
11743 .await;
11744 register_capability_manifest(
11745 &handler,
11746 &opener_ctx,
11747 &mut opener_rx,
11748 capability_manifest("opener", &[], &["credentials-provider/v1"]),
11749 2,
11750 )
11751 .await;
11752
11753 let (_client_rx, bind) = open_route_for_capability_test(
11754 &handler,
11755 &target_ctx,
11756 &mut target_rx,
11757 742,
11758 3,
11759 "target",
11760 None,
11761 )
11762 .await;
11763 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
11764 panic!("direct scope-honesty route must bind");
11765 };
11766 assert_eq!(principal, Some(Principal::Direct));
11767 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
11768 }
11769
11770 #[tokio::test]
11773 async fn must_never_reach_self_route_is_capability_forbidden() {
11774 let registry = Arc::new(Registry::default());
11775 let forwarding = Arc::new(ForwardingTable::default());
11776 let supervisor = SupervisorHandle::new();
11777 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
11778 let handler =
11779 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11780 .with_supervisor(supervisor);
11781 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
11782 register_capability_manifest(
11783 &handler,
11784 &self_ctx,
11785 &mut self_rx,
11786 capability_manifest(
11787 "self-provider",
11788 &["credentials-provider/v1"],
11789 &["credentials-provider/v1"],
11790 ),
11791 1,
11792 )
11793 .await;
11794
11795 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
11796 let replies = handler
11797 .handle_control_frame(
11798 &client_ctx,
11799 route_open_frame_with_admission_facts(
11800 2,
11801 "self-provider",
11802 unique_project_root("admission-facts"),
11803 Some(ConsumerIdentity {
11804 module_id: "self-provider".to_string(),
11805 launch_nonce: "self-nonce".to_string(),
11806 }),
11807 None,
11808 ),
11809 )
11810 .await
11811 .expect("self-route refusal returns a typed frame");
11812 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
11813 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
11814 assert!(
11815 self_rx.try_recv().is_err(),
11816 "self denial must not relay route.bind"
11817 );
11818 }
11819
11820 #[test]
11821 fn unsupported_channel_zero_frame_returns_error() {
11822 let handler = ControlHandler::default();
11823 let request = Frame::build(
11824 FrameType::Request,
11825 control_flags(),
11826 0,
11827 0,
11828 21,
11829 b"opaque".to_vec(),
11830 )
11831 .unwrap();
11832
11833 let response = handler
11834 .handle_control(ConnectionId::new(1), request)
11835 .unwrap();
11836
11837 assert_eq!(response[0].header.ty, FrameType::Error);
11838 assert_eq!(
11839 parse_error(&response[0])["code"],
11840 "unsupported_control_frame"
11841 );
11842 }
11843
11844 mod swap {
11849 use super::*;
11850
11851 const INCUMBENT: ConnectionId = ConnectionId::new(30);
11852 const CANDIDATE: ConnectionId = ConnectionId::new(40);
11853
11854 struct Swap {
11855 registry: Arc<Registry>,
11856 forwarding: Arc<ForwardingTable>,
11857 handler: ControlHandler,
11858 incumbent_ctx: RouteCtx,
11859 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
11860 candidate_ctx: RouteCtx,
11861 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
11862 }
11863
11864 async fn swap_with_incumbent() -> Swap {
11865 let registry = Arc::new(Registry::default());
11866 let forwarding = Arc::new(ForwardingTable::default());
11867 let handler =
11868 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11869 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
11870 hello_via_sink(
11871 &handler,
11872 &incumbent_ctx,
11873 &mut incumbent_rx,
11874 hello_frame("aft", PROTOCOL_VERSION, 7),
11875 )
11876 .await;
11877 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
11878 Swap {
11879 registry,
11880 forwarding,
11881 handler,
11882 incumbent_ctx,
11883 incumbent_rx,
11884 candidate_ctx,
11885 candidate_rx,
11886 }
11887 }
11888
11889 fn register_candidate(swap: &Swap, ready: Option<bool>) {
11890 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
11891 candidate_manifest.ready = ready;
11892 let registration = swap
11893 .registry
11894 .register_candidate_with_control_ops(
11895 candidate_manifest,
11896 PROTOCOL_VERSION,
11897 CANDIDATE,
11898 module_baseline_control_ops(),
11899 )
11900 .unwrap();
11901 swap.forwarding
11902 .register_candidate_module_connection(
11903 CANDIDATE,
11904 "aft".to_string(),
11905 PROTOCOL_VERSION,
11906 manifest_concurrency(®istration.manifest),
11907 swap.candidate_ctx.egress.clone(),
11908 )
11909 .unwrap();
11910 }
11911
11912 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
11913 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
11914 swap.registry.promote_candidate("aft").unwrap().unwrap();
11915 cutover.incumbent.unwrap()
11916 }
11917
11918 fn keyed_total(counters: &Value, key: &str) -> u64 {
11919 counters[key]
11920 .as_object()
11921 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
11922 .unwrap_or(0)
11923 }
11924
11925 #[tokio::test]
11926 async fn replacement_logs_old_end_and_new_admission_once() {
11927 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
11928 let mut swap = swap_with_incumbent().await;
11929 swap.handler
11930 .supervisor
11931 .open_swap("aft", "candidate-nonce".to_string());
11932 hello_via_sink(
11933 &swap.handler,
11934 &swap.candidate_ctx,
11935 &mut swap.candidate_rx,
11936 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
11937 )
11938 .await;
11939 cutover(&swap);
11940 swap.handler.cleanup_connection(INCUMBENT).unwrap();
11941
11942 let captured = crate::router::test_log::captured_logs(&logs);
11943 println!("captured replacement registry logs:\n{captured}");
11944 let new_admissions: Vec<_> = captured
11945 .lines()
11946 .filter(|line| {
11947 line.contains("connection_id=40")
11948 && line.contains("swap candidate registered; not routable until cutover")
11949 })
11950 .collect();
11951 assert_eq!(
11952 new_admissions.len(),
11953 1,
11954 "unexpected admission logs: {captured}"
11955 );
11956 assert!(new_admissions[0].contains("module_id=aft"));
11957 assert!(new_admissions[0].contains("connection_id=40"));
11958 assert!(
11959 new_admissions[0].contains("swap candidate registered; not routable until cutover")
11960 );
11961 assert!(new_admissions[0].contains("ready=true"));
11962
11963 let old_admissions: Vec<_> = captured
11964 .lines()
11965 .filter(|line| {
11966 line.contains("module registered module_id=aft ")
11967 && line.contains("connection_id=30")
11968 })
11969 .collect();
11970 assert_eq!(
11971 old_admissions.len(),
11972 1,
11973 "unexpected incumbent admission: {captured}"
11974 );
11975
11976 let old_ends: Vec<_> = captured
11977 .lines()
11978 .filter(|line| {
11979 line.contains("connection_id=30") && line.contains("module registration ended")
11980 })
11981 .collect();
11982 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
11983 assert!(old_ends[0].contains("module_id=aft"));
11984 assert!(old_ends[0].contains("connection_id=30"));
11985 assert!(old_ends[0].contains("reason=replaced"));
11986 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
11987 assert_eq!(
11988 captured
11989 .lines()
11990 .filter(|line| line.contains("module registration promoted"))
11991 .count(),
11992 1,
11993 "unexpected promotion logs: {captured}"
11994 );
11995 }
11996
11997 #[tokio::test]
12003 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
12004 let mut swap = swap_with_incumbent().await;
12005 let handler = swap.handler.clone();
12006
12007 let cotenant = ConnectionId::new(31);
12009 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
12010 let (cotenant_task, cotenant_bind) = relay_route_open(
12011 &handler,
12012 cotenant,
12013 &cotenant_ctx.egress,
12014 &mut swap.incumbent_rx,
12015 100,
12016 "aft",
12017 "swap-cotenant",
12018 )
12019 .await;
12020 handler
12021 .handle_control_frame(
12022 &swap.incumbent_ctx,
12023 route_bind_ack(cotenant_bind.header.corr),
12024 )
12025 .await
12026 .unwrap();
12027 assert!(cotenant_task.await.unwrap().is_empty());
12028 let (cotenant_channel, cotenant_epoch) =
12029 published_route(&cotenant_rx.recv().await.unwrap());
12030
12031 let caller = ConnectionId::new(32);
12033 let (caller_ctx, mut caller_rx) = route_ctx(caller);
12034 let (caller_task, caller_bind) = relay_route_open(
12035 &handler,
12036 caller,
12037 &caller_ctx.egress,
12038 &mut swap.incumbent_rx,
12039 101,
12040 "aft",
12041 "swap-caller",
12042 )
12043 .await;
12044 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
12045
12046 register_candidate(&swap, None);
12047 cutover(&swap);
12048
12049 let ack = handler
12051 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
12052 .await;
12053 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
12054 if module_loop_error.is_some() {
12055 handler.cleanup_connection(INCUMBENT).unwrap();
12058 }
12059
12060 assert!(
12062 cotenant_rx.try_recv().is_err(),
12063 "the co-tenant route on the incumbent was torn down by one late ack: \
12064 {module_loop_error:?}"
12065 );
12066 assert!(matches!(
12067 swap.forwarding
12068 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
12069 .unwrap(),
12070 DataRoute::Client(DataRouteState::Bound(_))
12071 ));
12072 assert_eq!(module_loop_error, None);
12073 assert!(swap
12074 .registry
12075 .get_module_by_connection(INCUMBENT)
12076 .unwrap()
12077 .is_some());
12078
12079 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
12081 .await
12082 .expect("the incumbent is told to drop the abandoned binding")
12083 .unwrap()
12084 .frame;
12085 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
12086 assert_eq!(goodbye.header.channel, abandoned_channel);
12087 assert_eq!(goodbye.header.epoch, abandoned_epoch);
12088 assert!(swap.incumbent_rx.try_recv().is_err());
12089
12090 let response = caller_task.await.unwrap();
12092 assert_eq!(response.len(), 1);
12093 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
12094 assert!(caller_rx.try_recv().is_err());
12095
12096 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
12100 let counters = handler.counters().snapshot();
12101 assert_eq!(
12102 keyed_total(&counters, "route_open_accepted_by_principal"),
12103 1
12104 );
12105 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
12106 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
12107 }
12108
12109 #[tokio::test]
12113 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
12114 let mut swap = swap_with_incumbent().await;
12115 register_candidate(&swap, None);
12116 let incumbent = cutover(&swap);
12117 swap.forwarding
12118 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
12119 .unwrap()
12120 .expect("the incumbent is still registered");
12121
12122 let client = ConnectionId::new(33);
12123 let (client_ctx, mut client_rx) = route_ctx(client);
12124 let route_handler = swap.handler.clone();
12125 let open_ctx = RouteCtx {
12126 connection_id: client,
12127 egress: client_ctx.egress.clone(),
12128 };
12129 let mut route_task = tokio::spawn(async move {
12130 route_handler
12131 .handle_control_frame(
12132 &open_ctx,
12133 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
12134 )
12135 .await
12136 .unwrap()
12137 });
12138 let bind = tokio::select! {
12139 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
12140 response = &mut route_task => {
12141 let response = response.unwrap();
12142 panic!(
12143 "post-cutover route.open was refused instead of relayed to the candidate: {}",
12144 parse_error(&response[0])["code"]
12145 );
12146 }
12147 };
12148 swap.handler
12149 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
12150 .await
12151 .unwrap();
12152 assert!(route_task.await.unwrap().is_empty());
12153 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
12154 match swap
12155 .forwarding
12156 .lookup_data_route(client, channel, epoch)
12157 .unwrap()
12158 {
12159 DataRoute::Client(DataRouteState::Bound(route)) => {
12160 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
12161 }
12162 other => panic!("expected a bound route on the candidate, got {other:?}"),
12163 }
12164 assert!(swap.incumbent_rx.try_recv().is_err());
12165 }
12166
12167 #[tokio::test]
12172 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
12173 let swap = swap_with_incumbent().await;
12174 register_candidate(&swap, Some(false));
12175 let update = Frame::build(
12176 FrameType::Request,
12177 control_flags(),
12178 0,
12179 0,
12180 55,
12181 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
12182 provides: manifest("aft", PROTOCOL_VERSION).provides,
12183 capabilities: None,
12184 ready: Some(true),
12185 })
12186 .unwrap(),
12187 )
12188 .unwrap();
12189
12190 let replies = swap
12191 .handler
12192 .handle_control_frame(&swap.candidate_ctx, update)
12193 .await
12194 .unwrap();
12195
12196 assert_eq!(replies.len(), 1);
12197 assert_eq!(
12198 replies[0].header.ty,
12199 FrameType::Response,
12200 "candidate catalog.update was refused: {:?}",
12201 serde_json::from_slice::<Value>(&replies[0].body).ok()
12202 );
12203 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
12204 assert_eq!(
12205 swap.registry
12206 .get_module("aft")
12207 .unwrap()
12208 .unwrap()
12209 .connection_id,
12210 INCUMBENT
12211 );
12212 }
12213 }
12214
12215 mod swap_admission {
12219 use super::*;
12220
12221 const INCUMBENT_NONCE: &str = "incumbent-nonce";
12222 const CANDIDATE_NONCE: &str = "candidate-nonce";
12223
12224 fn handler_with_incumbent(
12225 module_id: &str,
12226 reserved: bool,
12227 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
12228 let registry = Arc::new(Registry::default());
12229 let supervisor = SupervisorHandle::new();
12230 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
12231 if reserved {
12232 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
12233 }
12234 let handler =
12235 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
12236 let incumbent = handler
12237 .handle_control(
12238 ConnectionId::new(1),
12239 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
12240 )
12241 .unwrap();
12242 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
12243 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
12244 (registry, supervisor, handler)
12245 }
12246
12247 #[test]
12253 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
12254 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
12255
12256 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
12257 let replies = handler
12258 .handle_control(
12259 ConnectionId::new(connection),
12260 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
12261 )
12262 .unwrap();
12263 assert_eq!(replies[0].header.ty, FrameType::Error);
12264 assert_eq!(
12265 parse_error(&replies[0])["code"],
12266 "swap_token_invalid",
12267 "nonce {nonce:?}"
12268 );
12269 }
12270 assert!(registry.get_candidate("aft").unwrap().is_none());
12271 assert_eq!(
12272 registry.get_module("aft").unwrap().unwrap().connection_id,
12273 ConnectionId::new(1)
12274 );
12275
12276 let admitted = handler
12279 .handle_control(
12280 ConnectionId::new(4),
12281 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
12282 )
12283 .unwrap();
12284 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
12285 assert_eq!(
12286 registry
12287 .get_candidate("aft")
12288 .unwrap()
12289 .unwrap()
12290 .connection_id,
12291 ConnectionId::new(4)
12292 );
12293 assert_eq!(
12294 registry.get_module("aft").unwrap().unwrap().connection_id,
12295 ConnectionId::new(1),
12296 "the candidate must not take the active slot"
12297 );
12298 let replayed = handler
12299 .handle_control(
12300 ConnectionId::new(5),
12301 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
12302 )
12303 .unwrap();
12304 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
12305
12306 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
12310 let squatter = handler
12311 .handle_control(
12312 ConnectionId::new(6),
12313 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
12314 )
12315 .unwrap();
12316 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
12317 assert!(
12318 registry.get_module("aft").unwrap().is_none(),
12319 "a squatter took the active slot of an id being swapped"
12320 );
12321 }
12322
12323 #[test]
12328 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
12329 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
12330
12331 let replies = handler
12332 .handle_control(
12333 ConnectionId::new(2),
12334 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
12335 )
12336 .unwrap();
12337
12338 assert_eq!(
12339 replies[0].header.ty,
12340 FrameType::HelloAck,
12341 "reserved candidate refused: {:?}",
12342 serde_json::from_slice::<Value>(&replies[0].body).ok()
12343 );
12344 assert_eq!(
12345 registry
12346 .get_candidate("vault")
12347 .unwrap()
12348 .unwrap()
12349 .connection_id,
12350 ConnectionId::new(2)
12351 );
12352 }
12353
12354 #[test]
12357 fn without_an_open_swap_the_ordinary_gates_decide() {
12358 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
12359 supervisor.close_swap("vault");
12360
12361 let candidate = handler
12362 .handle_control(
12363 ConnectionId::new(2),
12364 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
12365 )
12366 .unwrap();
12367 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
12368 let duplicate = handler
12369 .handle_control(
12370 ConnectionId::new(3),
12371 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
12372 )
12373 .unwrap();
12374 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
12375 assert!(registry.get_candidate("vault").unwrap().is_none());
12376 }
12377 }
12378
12379 mod scopes {
12383 use subc_protocol::scope::{
12384 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
12385 ScopeStatus,
12386 };
12387
12388 use super::*;
12389
12390 const OWNER: &str = "prefrontal-core";
12391
12392 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
12393 ScopeRecord {
12394 scope_ref: scope_ref.to_string(),
12395 scope_epoch,
12396 kind: ScopeKind::Head,
12397 parent: None,
12398 child_owners: Vec::new(),
12399 carriers: Vec::new(),
12400 attributes: Default::default(),
12401 }
12402 }
12403
12404 async fn call(
12405 handler: &ControlHandler,
12406 ctx: &RouteCtx,
12407 request: &ModuleControlRequestFromModule,
12408 ) -> Frame {
12409 let body = serde_json::to_vec(request).unwrap();
12410 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
12411 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
12412 assert_eq!(replies.len(), 1, "{replies:?}");
12413 replies.pop().unwrap()
12414 }
12415
12416 async fn sync(
12417 handler: &ControlHandler,
12418 ctx: &RouteCtx,
12419 generation: u64,
12420 scopes: Vec<ScopeRecord>,
12421 ) -> Result<ModuleControlResponseToModule, String> {
12422 let reply = call(
12423 handler,
12424 ctx,
12425 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
12426 )
12427 .await;
12428 match reply.header.ty {
12429 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
12430 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
12431 }
12432 }
12433
12434 async fn describe(
12435 handler: &ControlHandler,
12436 ctx: &RouteCtx,
12437 owner: &str,
12438 scope_ref: &str,
12439 ) -> ModuleControlResponseToModule {
12440 let reply = call(
12441 handler,
12442 ctx,
12443 &ModuleControlRequestFromModule::ScopeDescribe {
12444 owner: Principal::Reserved {
12445 module_id: owner.to_string(),
12446 },
12447 scope_ref: scope_ref.to_string(),
12448 },
12449 )
12450 .await;
12451 assert_eq!(
12452 reply.header.ty,
12453 FrameType::Response,
12454 "{:?}",
12455 parse_error(&reply)
12456 );
12457 serde_json::from_slice(&reply.body).unwrap()
12458 }
12459
12460 async fn module(
12462 handler: &ControlHandler,
12463 connection: u64,
12464 module_id: &str,
12465 nonce: Option<&str>,
12466 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
12467 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
12468 hello_via_sink(
12469 handler,
12470 &ctx,
12471 &mut rx,
12472 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
12473 )
12474 .await;
12475 (ctx, rx)
12476 }
12477
12478 #[tokio::test]
12481 async fn a_client_connection_cannot_sync_or_describe() {
12482 let handler = ControlHandler::new(Arc::new(Registry::default()));
12483 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
12484 for request in [
12485 ModuleControlRequestFromModule::ScopeSync {
12486 generation: 1,
12487 scopes: vec![head("s", 1)],
12488 },
12489 ModuleControlRequestFromModule::ScopeDescribe {
12490 owner: Principal::Direct,
12491 scope_ref: "s".to_string(),
12492 },
12493 ] {
12494 let reply = call(&handler, &ctx, &request).await;
12495 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
12496 }
12497 assert!(
12498 !handler
12499 .scopes
12500 .read()
12501 .unwrap()
12502 .describe(
12503 &Principal::Reserved {
12504 module_id: OWNER.to_string()
12505 },
12506 "s"
12507 )
12508 .owner_synced
12509 );
12510 }
12511
12512 #[tokio::test]
12515 async fn a_module_without_a_supervised_launch_cannot_sync() {
12516 let handler = ControlHandler::new(Arc::new(Registry::default()));
12517 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
12518 assert_eq!(
12519 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
12520 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
12521 );
12522 }
12523
12524 #[tokio::test]
12525 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
12526 let supervisor = SupervisorHandle::new();
12527 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
12528 let handler = ControlHandler::new(Arc::new(Registry::default()))
12529 .with_supervisor(supervisor.clone());
12530 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
12531 sync(&handler, &incumbent, 1, vec![head("s", 1)])
12532 .await
12533 .expect("the current launch syncs");
12534
12535 supervisor.open_swap(OWNER, "n2".to_string());
12538 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
12539 assert_eq!(
12540 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
12541 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
12542 );
12543 sync(&handler, &incumbent, 2, vec![head("s", 1)])
12544 .await
12545 .expect("the serving owner syncs during the swap");
12546
12547 supervisor.close_swap(OWNER);
12550 assert_eq!(
12551 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
12552 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
12553 );
12554 sync(&handler, &incumbent, 3, vec![head("s", 1)])
12555 .await
12556 .expect("the serving owner syncs after the rollback");
12557 handler.cleanup_connection(candidate.connection_id).unwrap();
12558
12559 supervisor.open_swap(OWNER, "n3".to_string());
12564 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
12565 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
12566 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
12567 .await
12568 .expect("the promoted launch takes authority");
12569 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
12570 panic!("unexpected reply {reply:?}");
12571 };
12572 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
12573 assert_eq!(
12574 sync(&handler, &incumbent, 4, Vec::new()).await,
12575 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
12576 );
12577 }
12578
12579 #[tokio::test]
12582 async fn closing_the_authority_connection_frees_sync_authority() {
12583 let supervisor = SupervisorHandle::new();
12584 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
12585 let handler = ControlHandler::new(Arc::new(Registry::default()))
12586 .with_supervisor(supervisor.clone());
12587 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
12588 sync(&handler, &first, 10, vec![head("s", 1)])
12589 .await
12590 .unwrap();
12591 handler.cleanup_connection(first.connection_id).unwrap();
12592
12593 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
12594 sync(&handler, &second, 1, vec![head("s", 1)])
12595 .await
12596 .expect("the next connection takes the released authority");
12597 }
12598
12599 #[tokio::test]
12600 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
12601 let supervisor = SupervisorHandle::new();
12602 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
12603 let handler =
12604 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
12605 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
12606 sync(&handler, &first, 10, vec![head("s", 1)])
12607 .await
12608 .unwrap();
12609 handler
12610 .handle_control_frame(
12611 &first,
12612 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
12613 )
12614 .await
12615 .unwrap();
12616 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
12617 sync(&handler, &second, 1, vec![head("s", 1)])
12618 .await
12619 .expect("GOODBYE releases authority even if the old socket remains open");
12620 }
12621
12622 #[tokio::test]
12623 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
12624 let registry = Arc::new(Registry::default());
12625 let supervisor_handle = SupervisorHandle::new();
12626 let supervisor =
12627 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
12628 .with_handle(supervisor_handle.clone())
12629 .with_daemon_incarnation("incarnation-7".to_string());
12630 supervisor
12633 .supervise_configured(
12634 ModuleSpec {
12635 module_id: OWNER.to_string(),
12636 program: PathBuf::from("/nonexistent/prefrontal-core"),
12637 args: Vec::new(),
12638 env: Vec::new(),
12639 reserved: false,
12640 reserved_prefixes: Vec::new(),
12641 protocol: ModuleProtocol::Subc,
12642 overlap: Default::default(),
12643 },
12644 false,
12645 )
12646 .unwrap();
12647 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
12648 let handler =
12649 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12650 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
12651
12652 let ModuleControlResponseToModule::ScopeDescribe {
12654 status,
12655 daemon_incarnation,
12656 owner_synced,
12657 owner_configured,
12658 scope,
12659 ..
12660 } = describe(&handler, &reader, OWNER, "s").await
12661 else {
12662 panic!("not a describe reply");
12663 };
12664 assert_eq!(status, ScopeStatus::NotLive);
12665 assert_eq!(daemon_incarnation, "incarnation-7");
12666 assert!(!owner_synced);
12667 assert!(owner_configured);
12668 assert!(scope.is_none());
12669
12670 let ModuleControlResponseToModule::ScopeDescribe {
12673 status,
12674 owner_configured,
12675 ..
12676 } = describe(&handler, &reader, "ghost", "s").await
12677 else {
12678 panic!("not a describe reply");
12679 };
12680 assert_eq!(status, ScopeStatus::NotLive);
12681 assert!(!owner_configured);
12682
12683 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
12685 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
12686 let ModuleControlResponseToModule::ScopeDescribe {
12687 status,
12688 scope_epoch,
12689 owner_synced,
12690 scope,
12691 ..
12692 } = describe(&handler, &reader, OWNER, "s").await
12693 else {
12694 panic!("not a describe reply");
12695 };
12696 assert_eq!(status, ScopeStatus::Live);
12697 assert_eq!(scope_epoch, Some(4));
12698 assert!(owner_synced);
12699 let stamp = scope.expect("a live scope carries its stamp");
12700 assert!(
12701 stamp.owner_authorized,
12702 "prefrontal-core is the default authority"
12703 );
12704 assert_eq!(stamp.kind, ScopeKind::Head);
12705 }
12706
12707 #[tokio::test]
12708 async fn scope_authority_owners_decides_owner_authorized() {
12709 let supervisor = SupervisorHandle::new();
12710 supervisor.set_spawn_nonce("broca", "b1".to_string());
12711 let handler = ControlHandler::new(Arc::new(Registry::default()))
12712 .with_supervisor(supervisor)
12713 .with_scope_authority_owners(vec!["broca".to_string()]);
12714 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
12715 let mut gated = head("s", 1);
12716 gated.attributes.agent_id = Some("agent".to_string());
12717 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
12718 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
12719 describe(&handler, &broca, "broca", "s").await
12720 else {
12721 panic!("not a describe reply");
12722 };
12723 assert!(scope.unwrap().owner_authorized);
12724 }
12725
12726 #[tokio::test]
12730 async fn scope_ops_and_the_scopes_capability_are_advertised() {
12731 let handler = ControlHandler::new(Arc::new(Registry::default()));
12732 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
12733 let ack = hello_via_sink(
12734 &handler,
12735 &ctx,
12736 &mut rx,
12737 hello_frame("m", PROTOCOL_VERSION, 1),
12738 )
12739 .await;
12740 let ack = parse_ack(&ack);
12741 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
12742 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
12743 }
12744 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
12745
12746 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
12747 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12748 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12749 let reply = handler
12750 .handle_control_frame(&client, frame)
12751 .await
12752 .unwrap()
12753 .pop()
12754 .unwrap();
12755 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12756 serde_json::from_slice(&reply.body).unwrap()
12757 else {
12758 panic!("not a server.describe reply");
12759 };
12760 assert!(
12761 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
12762 "{capabilities:?}"
12763 );
12764 }
12765
12766 const PLEXUS: &str = "plexus";
12769 const OTHER: &str = "other";
12770 const AFT: &str = "aft";
12771 const BROCA: &str = "broca";
12772 const MAGIC: &str = "magic-context";
12773
12774 fn nonce(module_id: &str) -> String {
12775 format!("nonce-{module_id}")
12776 }
12777
12778 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
12779 let (tx, rx) = mpsc::channel(64);
12780 (
12781 RouteCtx {
12782 connection_id: ConnectionId::new(connection),
12783 egress: FrameSink::new(tx),
12784 },
12785 rx,
12786 )
12787 }
12788
12789 struct Rig {
12793 handler: ControlHandler,
12794 forwarding: Arc<ForwardingTable>,
12795 owner: RouteCtx,
12796 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12797 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
12798 generation: u64,
12799 next_connection: u64,
12800 _supervisor: Supervisor,
12801 }
12802
12803 async fn rig() -> Rig {
12804 rig_with_flow_support(true).await
12805 }
12806
12807 async fn rig_with_flow_support(flow_support: bool) -> Rig {
12808 let registry = Arc::new(Registry::default());
12809 let forwarding = Arc::new(ForwardingTable::default());
12810 let supervisor_handle = SupervisorHandle::new();
12811 let supervisor =
12812 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
12813 .with_handle(supervisor_handle.clone());
12814 supervisor
12815 .supervise_configured(
12816 ModuleSpec {
12817 module_id: OWNER.to_string(),
12818 program: PathBuf::from("/nonexistent/prefrontal-core"),
12819 args: Vec::new(),
12820 env: Vec::new(),
12821 reserved: false,
12822 reserved_prefixes: Vec::new(),
12823 protocol: ModuleProtocol::Subc,
12824 overlap: Default::default(),
12825 },
12826 false,
12827 )
12828 .unwrap();
12829 for module_id in [OWNER, AFT, BROCA, MAGIC] {
12830 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
12831 }
12832 let handler =
12833 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12834 .with_supervisor(supervisor_handle);
12835 let (owner, mut owner_rx) = wide_ctx(1);
12836 hello_via_sink(
12837 &handler,
12838 &owner,
12839 &mut owner_rx,
12840 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
12841 )
12842 .await;
12843 let mut modules = BTreeMap::new();
12844 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
12845 let (ctx, mut rx) = wide_ctx(connection);
12846 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
12847 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
12848 body["manifest"]["provenance"] =
12852 serde_json::json!({"wire_crate_version": "0.29.0"});
12853 if flow_support {
12854 body["manifest"]["capabilities"] =
12855 serde_json::json!({"provides": ["flow-scopes/v1"]});
12856 }
12857 let hello = Frame::build(
12858 FrameType::Hello,
12859 control_flags(),
12860 0,
12861 0,
12862 connection,
12863 serde_json::to_vec(&body).unwrap(),
12864 )
12865 .unwrap();
12866 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
12867 modules.insert(module_id.to_string(), (ctx, rx));
12868 }
12869 Rig {
12870 handler,
12871 forwarding,
12872 owner,
12873 _owner_rx: owner_rx,
12874 modules,
12875 generation: 0,
12876 next_connection: 100,
12877 _supervisor: supervisor,
12878 }
12879 }
12880
12881 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
12882 ScopeCarrier {
12883 principal: Principal::Reserved {
12884 module_id: module_id.to_string(),
12885 },
12886 targets: targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()),
12887 }
12888 }
12889
12890 fn session(scope_epoch: u64) -> ScopeRecord {
12893 let mut record = head("s", scope_epoch);
12894 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
12895 record.attributes.agent_id = Some("agent-1".to_string());
12896 record.attributes.delegates = true;
12897 record
12898 }
12899
12900 impl Rig {
12901 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
12902 self.generation += 1;
12903 sync(&self.handler, &self.owner, self.generation, scopes)
12904 .await
12905 .expect("the owner's sync is accepted");
12906 }
12907
12908 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
12909 ScopeSelector {
12910 owner: Principal::Reserved {
12911 module_id: OWNER.to_string(),
12912 },
12913 scope_ref: scope_ref.to_string(),
12914 scope_epoch,
12915 }
12916 }
12917
12918 fn open_frame(
12919 &mut self,
12920 opener: Option<&str>,
12921 target: &str,
12922 scope: Option<ScopeSelector>,
12923 ) -> (
12924 RouteCtx,
12925 mpsc::Receiver<crate::router::OutboundFrame>,
12926 Frame,
12927 ) {
12928 self.next_connection += 1;
12929 let (ctx, rx) = wide_ctx(self.next_connection);
12930 let root = unique_project_root("scoped-open");
12931 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
12932 target: RouteTarget::ToolProvider {
12933 module_id: target.to_string(),
12934 },
12935 identity: BindIdentity::new(
12936 root.path().to_path_buf(),
12937 "unit".to_string(),
12938 "session".to_string(),
12939 ),
12940 consumer_identity: opener.map(|module_id| ConsumerIdentity {
12941 module_id: module_id.to_string(),
12942 launch_nonce: nonce(module_id),
12943 }),
12944 consumer_capabilities: None,
12945 role_versions: None,
12946 admission_facts: None,
12947 scope,
12948 })
12949 .unwrap();
12950 let frame = Frame::build(
12951 FrameType::Request,
12952 control_flags(),
12953 0,
12954 0,
12955 self.next_connection,
12956 body,
12957 )
12958 .unwrap();
12959 (ctx, rx, frame)
12960 }
12961
12962 async fn refused(
12964 &mut self,
12965 opener: Option<&str>,
12966 target: &str,
12967 scope: Option<ScopeSelector>,
12968 ) -> String {
12969 self.refusal_body(opener, target, scope).await["code"]
12970 .as_str()
12971 .unwrap()
12972 .to_string()
12973 }
12974
12975 async fn refusal_body(
12976 &mut self,
12977 opener: Option<&str>,
12978 target: &str,
12979 scope: Option<ScopeSelector>,
12980 ) -> Value {
12981 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
12982 let replies = tokio::time::timeout(
12983 Duration::from_secs(2),
12984 self.handler.handle_control_frame(&ctx, frame),
12985 )
12986 .await
12987 .expect("the open must be refused before waiting for a bind ack")
12988 .unwrap();
12989 assert_eq!(replies.len(), 1, "{replies:?}");
12990 assert_eq!(replies[0].header.ty, FrameType::Error);
12991 let (_, module_rx) = self.modules.get_mut(target).unwrap();
12992 assert!(
12993 module_rx.try_recv().is_err(),
12994 "a refused open relays nothing"
12995 );
12996 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
12997 parse_error(&replies[0])
12998 }
12999
13000 async fn relayed(
13002 &mut self,
13003 opener: Option<&str>,
13004 target: &str,
13005 scope: Option<ScopeSelector>,
13006 ) -> Relayed {
13007 let (ctx, rx, frame) = self.open_frame(opener, target, scope);
13008 let handler = self.handler.clone();
13009 let task_ctx = ctx.clone();
13010 let task = tokio::spawn(async move {
13011 handler
13012 .handle_control_frame(&task_ctx, frame)
13013 .await
13014 .unwrap()
13015 });
13016 let (_, module_rx) = self.modules.get_mut(target).unwrap();
13017 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
13018 .await
13019 .expect("the target receives the relayed route.bind")
13020 .unwrap()
13021 .frame;
13022 Relayed {
13023 target: target.to_string(),
13024 client: ctx,
13025 client_rx: rx,
13026 task,
13027 bind,
13028 }
13029 }
13030
13031 async fn ack(&self, relayed: &Relayed) {
13032 let (module, _) = &self.modules[&relayed.target];
13033 self.handler
13034 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
13035 .await
13036 .unwrap();
13037 }
13038
13039 async fn bound(
13041 &mut self,
13042 opener: Option<&str>,
13043 target: &str,
13044 scope: Option<ScopeSelector>,
13045 ) -> Bound {
13046 let relayed = self.relayed(opener, target, scope).await;
13047 self.ack(&relayed).await;
13048 let Relayed {
13049 target,
13050 client,
13051 mut client_rx,
13052 task,
13053 bind,
13054 } = relayed;
13055 assert!(
13056 task.await.unwrap().is_empty(),
13057 "the open is answered by commit"
13058 );
13059 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
13060 Bound {
13061 target,
13062 client,
13063 client_rx,
13064 channel,
13065 epoch,
13066 bind,
13067 }
13068 }
13069
13070 fn live(&self, route: &Bound) -> bool {
13071 matches!(
13072 self.forwarding
13073 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
13074 .unwrap(),
13075 DataRoute::Client(DataRouteState::Bound(_))
13076 )
13077 }
13078 }
13079
13080 struct Relayed {
13081 target: String,
13082 client: RouteCtx,
13083 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13084 task: tokio::task::JoinHandle<Vec<Frame>>,
13085 bind: Frame,
13086 }
13087
13088 struct Bound {
13089 target: String,
13090 client: RouteCtx,
13091 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13092 channel: u16,
13093 epoch: u32,
13094 bind: Frame,
13095 }
13096
13097 impl Bound {
13098 fn closed_reason(&mut self) -> RouteCloseReason {
13101 let mut reason = None;
13102 let mut goodbye = false;
13103 while let Ok(outbound) = self.client_rx.try_recv() {
13104 let frame = outbound.frame;
13105 match frame.header.ty {
13106 FrameType::Goodbye => {
13107 assert_eq!(
13108 (frame.header.channel, frame.header.epoch),
13109 (self.channel, self.epoch)
13110 );
13111 goodbye = true;
13112 }
13113 FrameType::Push => {
13114 let ClientControlPush::RouteClosed {
13115 reason: r,
13116 module_id,
13117 ..
13118 } = serde_json::from_slice(&frame.body).unwrap()
13119 else {
13120 panic!("unexpected push");
13121 };
13122 assert_eq!(module_id, self.target);
13123 reason = Some(r);
13124 }
13125 other => panic!("unexpected frame {other:?}"),
13126 }
13127 }
13128 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
13129 reason.expect("the client is told why the route closed")
13130 }
13131
13132 fn untouched(&mut self) -> bool {
13133 self.client_rx.try_recv().is_err()
13134 }
13135
13136 fn stamp(&self) -> Option<ScopeStamp> {
13137 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
13138 ModuleControlRequest::RouteBind { scope, .. } => scope,
13139 other => panic!("expected a route.bind, got {other:?}"),
13140 }
13141 }
13142 }
13143
13144 #[tokio::test]
13145 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
13146 ) {
13147 let mut rig = rig().await;
13148 let mut record = session(1);
13149 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
13150 record.child_owners = vec![Principal::Reserved {
13151 module_id: MAGIC.to_string(),
13152 }];
13153 rig.sync(vec![record]).await;
13154 let scope = || Some(rig_selector("s", Some(1)));
13155
13156 rig.bound(Some(OWNER), PLEXUS, scope()).await;
13159 rig.bound(Some(AFT), OTHER, scope()).await;
13160 rig.bound(Some(BROCA), PLEXUS, scope()).await;
13161
13162 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
13166 assert_eq!(
13167 rig.refused(opener, target, scope()).await,
13168 error_codes::SCOPE_NOT_CARRIER,
13169 "{opener:?} -> {target}"
13170 );
13171 }
13172 }
13173
13174 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
13175 ScopeSelector {
13176 owner: Principal::Reserved {
13177 module_id: OWNER.to_string(),
13178 },
13179 scope_ref: scope_ref.to_string(),
13180 scope_epoch,
13181 }
13182 }
13183
13184 #[tokio::test]
13185 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
13186 let mut rig = rig().await;
13187 rig.sync(vec![session(1)]).await;
13188 for opener in [OWNER, AFT] {
13189 assert_eq!(
13190 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
13191 .await,
13192 error_codes::SCOPE_EPOCH_REQUIRED,
13193 "{opener}"
13194 );
13195 }
13196 }
13197
13198 #[tokio::test]
13199 async fn admission_separates_not_synced_not_live_and_ended() {
13200 let mut rig = rig().await;
13201 let code = rig
13203 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13204 .await;
13205 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
13206 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
13207
13208 let ghost = ScopeSelector {
13210 owner: Principal::Reserved {
13211 module_id: "ghost".to_string(),
13212 },
13213 scope_ref: "s".to_string(),
13214 scope_epoch: Some(1),
13215 };
13216 assert_eq!(
13217 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
13218 error_codes::SCOPE_NOT_LIVE
13219 );
13220
13221 rig.sync(vec![session(2)]).await;
13222 assert_eq!(
13223 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
13224 .await,
13225 error_codes::SCOPE_NOT_LIVE
13226 );
13227 for epoch in [1, 3] {
13228 assert_eq!(
13229 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
13230 .await,
13231 error_codes::SCOPE_ENDED,
13232 "epoch {epoch}"
13233 );
13234 }
13235 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
13237 .await;
13238 }
13239
13240 #[tokio::test]
13241 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
13242 let mut rig = rig().await;
13243 rig.sync(vec![session(1)]).await;
13244 let route = rig
13245 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13246 .await;
13247 let stamp = route.stamp().expect("a scoped bind carries the stamp");
13248 assert_eq!(stamp.scope_ref, "s");
13249 assert_eq!(stamp.scope_epoch, 1);
13250 assert_eq!(stamp.kind, ScopeKind::Head);
13251 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
13252 assert!(stamp.attributes.delegates);
13253 assert!(stamp.owner_authorized);
13254 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
13255 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
13256
13257 let (broca, mut broca_rx) = wide_ctx(50);
13260 hello_via_sink(
13261 &rig.handler,
13262 &broca,
13263 &mut broca_rx,
13264 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
13265 )
13266 .await;
13267 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
13268 .await
13269 .unwrap();
13270 let own = ScopeSelector {
13271 owner: Principal::Reserved {
13272 module_id: BROCA.to_string(),
13273 },
13274 scope_ref: "b".to_string(),
13275 scope_epoch: Some(1),
13276 };
13277 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
13278 assert!(!route.stamp().unwrap().owner_authorized);
13279 }
13280
13281 #[tokio::test]
13282 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
13283 let mut rig = rig().await;
13284 let mut record = head("s", 1);
13285 record.carriers = vec![carrier(AFT, None)];
13286 let flow_id = "Flow:run-7/step_2!~";
13287 record.attributes.flow_id = Some(flow_id.to_string());
13288 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
13289 .await
13290 .unwrap();
13291 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
13292 panic!("not a sync reply");
13293 };
13294 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
13295 let route = rig
13296 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13297 .await;
13298 assert!(rig.live(&route), "the stamped bind committed");
13299 let stamp = route.stamp().expect("a flow scope carries a stamp");
13300 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
13301 assert_eq!(stamp.attributes.agent_id, None);
13302 assert!(!stamp.attributes.delegates);
13303 assert!(stamp.owner_authorized);
13304 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
13305 assert_eq!(unscoped.stamp(), None);
13306 }
13307
13308 #[tokio::test]
13309 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
13310 let mut rig = rig_with_flow_support(false).await;
13311 let mut record = session(1);
13312 record.attributes.flow_id = Some("flow:7".to_string());
13313 rig.sync(vec![record]).await;
13314 for opener in [OWNER, AFT] {
13315 let body = rig
13316 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
13317 .await;
13318 assert_eq!(body["code"], "target_flow_unsupported");
13319 let message = body["message"].as_str().unwrap();
13320 for required in [PLEXUS, "flow-scopes/v1"] {
13321 assert!(message.contains(required), "{message}");
13322 }
13323 }
13324 }
13325
13326 #[tokio::test]
13327 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
13328 let mut rig = rig_with_flow_support(true).await;
13329 let mut record = session(1);
13330 record.attributes.flow_id = Some("flow:7".to_string());
13331 rig.sync(vec![record]).await;
13332 for opener in [OWNER, AFT] {
13333 let route = rig
13334 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
13335 .await;
13336 assert!(rig.live(&route));
13337 assert_eq!(
13338 route.stamp().unwrap().attributes.flow_id.as_deref(),
13339 Some("flow:7")
13340 );
13341 }
13342 }
13343
13344 #[tokio::test]
13345 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
13346 use std::future::Future;
13347
13348 let mut rig = rig_with_flow_support(true).await;
13349 let mut record = session(1);
13350 record.attributes.flow_id = Some("flow:7".to_string());
13351 rig.sync(vec![record]).await;
13352 let (client, mut client_rx, frame) =
13353 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
13354 for _ in 0..64 {
13357 client.egress.try_send(route_bind_ack(1)).unwrap();
13358 }
13359 let handler = rig.handler.clone();
13360 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
13361 std::future::poll_fn(|cx| {
13362 assert!(open.as_mut().poll(cx).is_pending());
13363 std::task::Poll::Ready(())
13364 })
13365 .await;
13366 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
13367
13368 let old_connection = rig.modules[PLEXUS].0.connection_id;
13369 rig.handler.cleanup_connection(old_connection).unwrap();
13370 let (replacement, mut replacement_rx) = wide_ctx(200);
13371 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
13372 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
13373 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
13374 let hello = Frame::build(
13375 FrameType::Hello,
13376 control_flags(),
13377 0,
13378 0,
13379 200,
13380 serde_json::to_vec(&body).unwrap(),
13381 )
13382 .unwrap();
13383 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
13384
13385 client_rx.try_recv().unwrap();
13386 let replies = tokio::time::timeout(Duration::from_secs(2), open)
13387 .await
13388 .expect("the replacement is refused without waiting for a bind ack")
13389 .unwrap();
13390 assert_eq!(replies.len(), 1);
13391 let body = parse_error(&replies[0]);
13392 assert_eq!(body["code"], "target_flow_unsupported");
13393 for required in [PLEXUS, "flow-scopes/v1"] {
13394 assert!(body["message"].as_str().unwrap().contains(required));
13395 }
13396 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
13397 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
13398 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
13399 assert!(rig
13400 .handler
13401 .registry
13402 .get_module_by_connection(replacement.connection_id)
13403 .unwrap()
13404 .is_some());
13405 }
13406
13407 #[tokio::test]
13408 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
13409 {
13410 let mut rig = rig_with_flow_support(false).await;
13411 rig.sync(vec![session(1)]).await;
13412 let route = rig
13413 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13414 .await;
13415 assert!(rig.live(&route));
13416 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
13417 let mut record = session(1);
13418 record.attributes.flow_id = Some("flow:7".to_string());
13419 rig.sync(vec![record]).await;
13420 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
13421 assert!(rig.live(&unscoped));
13422 assert_eq!(unscoped.stamp(), None);
13423 }
13424
13425 #[tokio::test]
13426 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
13427 let mut rig = rig().await;
13428 let mut record = session(1);
13429 record.attributes.flow_id = Some("flow:7".to_string());
13430 rig.sync(vec![record.clone()]).await;
13431 let before = rig.forwarding.published_scope_tag(OWNER, "s");
13432 let mut owner_route = rig
13433 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
13434 .await;
13435 let mut carrier_route = rig
13436 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
13437 .await;
13438 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
13439 rig.sync(vec![record.clone()]).await;
13440 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
13441 assert!(rig.live(&owner_route) && owner_route.untouched());
13442 assert!(rig.live(&carrier_route) && carrier_route.untouched());
13443
13444 record.attributes.flow_id = Some("flow:8".to_string());
13445 rig.sync(vec![record]).await;
13446 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
13447 let before = before.unwrap();
13448 assert_eq!(after.scope_epoch, before.scope_epoch);
13449 assert!(after.version > before.version);
13450 for route in [&mut owner_route, &mut carrier_route] {
13451 assert!(!rig.live(route));
13452 assert_eq!(
13453 route.closed_reason(),
13454 RouteCloseReason::ScopeDelegationChanged
13455 );
13456 }
13457 assert!(rig.live(&unscoped) && unscoped.untouched());
13458 for target in [PLEXUS, OTHER] {
13461 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
13462 let goodbye = module_rx
13463 .try_recv()
13464 .expect("the provider sees the drain")
13465 .frame;
13466 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
13467 assert!(module_rx.try_recv().is_err());
13468 }
13469 let rebound = rig
13470 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13471 .await;
13472 assert_eq!(
13473 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
13474 Some("flow:8")
13475 );
13476 }
13477
13478 #[tokio::test]
13483 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
13484 let mut rig = rig().await;
13485 rig.sync(vec![session(1)]).await;
13486 let mut cotenant = rig
13487 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
13488 .await;
13489
13490 let mut changed = session(1);
13491 changed.child_owners.push(Principal::Reserved {
13492 module_id: MAGIC.to_string(),
13493 });
13494 let mut ended = None;
13495 for (code, next) in [
13496 (error_codes::SCOPE_CHANGED, vec![changed]),
13497 (error_codes::SCOPE_ENDED, Vec::new()),
13498 ] {
13499 let relayed = rig
13500 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13501 .await;
13502 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
13503 ended = Some(next.is_empty());
13504 rig.sync(next).await;
13505 rig.ack(&relayed).await;
13506 let replies = relayed.task.await.unwrap();
13507 assert_eq!(replies.len(), 1, "{replies:?}");
13508 assert_eq!(parse_error(&replies[0])["code"], code);
13509 assert_eq!(
13510 subc_protocol::error_codes::is_retryable_route_open(code),
13511 code == error_codes::SCOPE_CHANGED
13512 );
13513 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
13514 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
13516 let mut goodbyes = Vec::new();
13519 while let Ok(outbound) = plexus_rx.try_recv() {
13520 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
13521 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
13522 }
13523 assert!(
13524 goodbyes.contains(&(bind_channel, bind_epoch)),
13525 "{goodbyes:?}"
13526 );
13527 assert!(rig
13528 .handler
13529 .registry
13530 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
13531 .unwrap()
13532 .is_some());
13533 }
13534 assert_eq!(ended, Some(true));
13535 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
13538 }
13539
13540 #[tokio::test]
13543 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
13544 struct Case {
13545 name: &'static str,
13546 change: fn(&mut ScopeRecord),
13547 closed: [Option<RouteCloseReason>; 4],
13550 }
13551 use RouteCloseReason::*;
13552 let cases = [
13553 Case {
13554 name: "a carrier entry removed",
13555 change: |r| {
13556 r.carriers.retain(|c| {
13557 c.principal
13558 != Principal::Reserved {
13559 module_id: AFT.to_string(),
13560 }
13561 })
13562 },
13563 closed: [None, Some(ScopeCarrierRemoved), None, None],
13564 },
13565 Case {
13566 name: "a target removed from a carrier",
13567 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
13568 closed: [None, None, None, Some(ScopeCarrierRemoved)],
13569 },
13570 Case {
13571 name: "a bare carrier narrowed to targets",
13572 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
13573 closed: [None, Some(ScopeCarrierRemoved), None, None],
13574 },
13575 Case {
13576 name: "delegates turned off",
13577 change: |r| r.attributes.delegates = false,
13578 closed: [Some(ScopeDelegationChanged); 4],
13579 },
13580 Case {
13581 name: "agent_id changed",
13582 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
13583 closed: [Some(ScopeDelegationChanged); 4],
13584 },
13585 Case {
13586 name: "a carrier added, child owners changed, the record re-sent",
13587 change: |r| {
13588 r.carriers.push(carrier(MAGIC, None));
13589 r.child_owners.push(Principal::Reserved {
13590 module_id: MAGIC.to_string(),
13591 });
13592 },
13593 closed: [None; 4],
13594 },
13595 Case {
13596 name: "a target added",
13597 change: |r| {
13598 r.carriers[1]
13599 .targets
13600 .as_mut()
13601 .unwrap()
13602 .push("third".to_string())
13603 },
13604 closed: [None; 4],
13605 },
13606 Case {
13607 name: "delegates turned on",
13608 change: |r| r.attributes.delegates = true,
13609 closed: [None; 4],
13610 },
13611 ];
13612 for case in cases {
13613 let mut rig = rig().await;
13614 rig.sync(vec![session(1)]).await;
13615 let scope = || Some(rig_selector("s", Some(1)));
13616 let mut routes = [
13617 rig.bound(Some(OWNER), PLEXUS, scope()).await,
13618 rig.bound(Some(AFT), PLEXUS, scope()).await,
13619 rig.bound(Some(BROCA), PLEXUS, scope()).await,
13620 rig.bound(Some(BROCA), OTHER, scope()).await,
13621 ];
13622 let mut record = session(1);
13623 (case.change)(&mut record);
13624 rig.sync(vec![record]).await;
13625 for (index, expected) in case.closed.iter().enumerate() {
13626 let route = &mut routes[index];
13627 match expected {
13628 Some(reason) => {
13629 assert!(!rig.live(route), "{}: route {index} still live", case.name);
13630 assert_eq!(
13631 route.closed_reason(),
13632 *reason,
13633 "{}: route {index}",
13634 case.name
13635 );
13636 }
13637 None => {
13638 assert!(rig.live(route), "{}: route {index} closed", case.name);
13639 assert!(
13640 route.untouched(),
13641 "{}: route {index} was told something",
13642 case.name
13643 );
13644 }
13645 }
13646 }
13647 }
13648 }
13649
13650 #[tokio::test]
13651 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
13652 for next in [Vec::new(), vec![session(2)]] {
13654 let mut rig = rig().await;
13655 rig.sync(vec![session(1)]).await;
13656 let mut route = rig
13657 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13658 .await;
13659 rig.sync(next).await;
13660 assert!(!rig.live(&route));
13661 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
13662 }
13663
13664 let mut rig = rig().await;
13667 let mut child = session(1);
13668 child.scope_ref = "child".to_string();
13669 child.kind = ScopeKind::Worker;
13670 child.parent = Some(ScopeParent {
13671 owner: Principal::Reserved {
13672 module_id: OWNER.to_string(),
13673 },
13674 scope_ref: "s".to_string(),
13675 scope_epoch: 1,
13676 });
13677 rig.sync(vec![session(1), child.clone()]).await;
13678 let mut child_route = rig
13679 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
13680 .await;
13681 assert_eq!(
13682 child_route.stamp().unwrap().parent_state,
13683 Some(ParentState::Linked)
13684 );
13685 rig.sync(vec![child]).await;
13686 assert!(!rig.live(&child_route));
13687 assert_eq!(
13688 child_route.closed_reason(),
13689 RouteCloseReason::ScopeParentEnded
13690 );
13691 }
13692
13693 #[tokio::test]
13694 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
13695 ) {
13696 let mut rig = rig().await;
13697 rig.sync(vec![session(1)]).await;
13698 let mut route = rig
13699 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13700 .await;
13701 let before = rig.forwarding.published_scope_tag(OWNER, "s");
13702 rig.sync(vec![session(1)]).await;
13703 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
13704 assert!(rig.live(&route) && route.untouched());
13705
13706 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
13710 .forwarding
13711 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
13712 .unwrap()
13713 else {
13714 panic!("the route is bound");
13715 };
13716 binding.flow.acquire_tagged(9, false).await.unwrap();
13717 let mut widened = session(1);
13718 widened.carriers.push(carrier(MAGIC, None));
13719 rig.sync(vec![widened]).await;
13720 assert!(rig.live(&route) && route.untouched());
13721 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
13722 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
13723 assert_eq!(binding.flow.in_flight(), 1);
13726 binding
13727 .flow
13728 .acquire_tagged(10, false)
13729 .await
13730 .expect("the flow is still open");
13731 }
13732
13733 #[tokio::test]
13736 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
13737 let mut rig = rig().await;
13738 rig.sync(vec![session(1)]).await;
13739 let mut on_incumbent = rig
13740 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13741 .await;
13742
13743 let (candidate, _candidate_rx) = wide_ctx(9);
13746 let registration = rig
13747 .handler
13748 .registry
13749 .register_candidate_with_control_ops(
13750 manifest(PLEXUS, PROTOCOL_VERSION),
13751 PROTOCOL_VERSION,
13752 candidate.connection_id,
13753 module_baseline_control_ops(),
13754 )
13755 .unwrap();
13756 rig.forwarding
13757 .register_candidate_module_connection(
13758 candidate.connection_id,
13759 PLEXUS.to_string(),
13760 PROTOCOL_VERSION,
13761 manifest_concurrency(®istration.manifest),
13762 candidate.egress.clone(),
13763 )
13764 .unwrap();
13765 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
13766 rig.handler
13767 .registry
13768 .promote_candidate(PLEXUS)
13769 .unwrap()
13770 .unwrap();
13771 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
13772
13773 rig.sync(Vec::new()).await;
13774 assert!(!rig.live(&on_incumbent));
13775 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
13776 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
13777 let goodbye = incumbent_rx
13778 .try_recv()
13779 .expect("the superseded endpoint is told")
13780 .frame;
13781 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
13782 }
13783 }
13784}
13785
13786#[cfg(test)]
13787mod concurrency_default_exposure_tests {
13788 use super::*;
13789
13790 fn hello_body(role_json: &str) -> Vec<u8> {
13791 format!(
13792 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
13793 )
13794 .into_bytes()
13795 }
13796
13797 fn manifest_from(body: &[u8]) -> ModuleManifest {
13798 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
13799 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
13800 .expect("manifest parses")
13801 }
13802
13803 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
13804
13805 #[test]
13806 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
13807 let body = hello_body(&format!(
13808 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
13809 ));
13810 let manifest = manifest_from(&body);
13811 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
13814 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
13815 }
13816
13817 #[test]
13818 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
13819 let body = hello_body(&format!(
13820 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
13821 ));
13822 let manifest = manifest_from(&body);
13823 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
13824 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
13825 }
13826
13827 #[test]
13828 fn non_management_roles_are_never_reported() {
13829 let body = hello_body(
13830 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
13831 );
13832 let manifest = manifest_from(&body);
13833 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
13834 }
13835}