1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 path::{Path, PathBuf},
5 sync::{Arc, Mutex, RwLock},
6 time::{Duration, Instant as StdInstant},
7};
8
9use serde::{Deserialize, Serialize};
10use subc_control::{
11 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
12 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
13 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
14 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
15 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
16 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
17 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
18};
19use subc_protocol::{
20 error_codes,
21 manifest::{
22 validate_hello_capability_grammar, validate_hello_self_signal_declarations,
23 CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
24 ProviderRole,
25 },
26 scope::{
27 ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
28 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_DESCRIBE_OP, SCOPE_SYNC_OP,
29 },
30 session::{
31 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
32 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
33 MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE, ROLE_VERSIONS_FIELD,
34 },
35 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
36 Priority, RouteTarget, PROTOCOL_VERSION,
37};
38use tokio::time::{timeout_at, Instant};
39use tracing::{debug, info, warn};
40
41use crate::{
42 capability_requirements::{
43 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
44 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
45 RuntimeModule,
46 },
47 daemon_config::RestartRequiredSection,
48 forwarding::{
49 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
50 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
51 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
52 },
53 observability::{
54 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
55 },
56 provenance::{
57 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
58 },
59 registry::{ChannelState, ConnectionId, Registry, RegistryError},
60 router::{RouteCtx, RouterError},
61 scopes::{BoundScope, HelloLaunchNonces, ScopeTable},
62 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
63 stderr_tail::{CaptureState, TailEntry},
64 supervise::{
65 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
66 SupervisorHandle, SwapHelloAdmission,
67 },
68 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
69};
70
71pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
77
78const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
79const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
80const CAP_PING_PONG: &str = "ping_pong_v1";
81const CAP_SESSION_ATTACH: &str = "session_attach_v1";
82const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
83
84const SUBC_CONTROL_OPS: &[&str] = &[
85 ops::SERVER_DESCRIBE,
86 ops::CATALOG_LIST,
87 ops::ROUTE_OPEN,
88 ops::ROUTE_POLL,
89 ops::ROUTE_CLOSING,
90 ops::ROUTE_CLOSED,
91 ops::SUPERVISOR_LIST,
92 ops::SUPERVISOR_RESTART,
93 ops::SUPERVISOR_SWAP,
94 ops::SUPERVISOR_RELOAD,
95 ops::SUPERVISOR_RESCAN,
96 ops::SUPERVISOR_RELEASE_RESERVED,
97 ops::SUPERVISOR_SET_ENABLED,
98 ops::SUPERVISOR_HEALTH_PROBE,
99 ops::SUPERVISOR_HEALTH,
100 ops::SUPERVISOR_STDERR_TAIL,
101 ops::SUPERVISOR_TERMINALS,
102 ops::SUPERVISOR_ROUTES,
103 ops::SUPERVISOR_PROVENANCE,
104 ops::SUPERVISOR_SPAWN_SNAPSHOT,
105 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
106];
107
108const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
109 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
110 "supervisor.live_roots",
111 SCOPE_SYNC_OP,
112 SCOPE_DESCRIBE_OP,
113];
114
115const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
119
120const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
121
122pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
130
131pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
146
147pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
165
166const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
167const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
168
169fn reload_verdict(
170 configured: &Path,
171 spawned_from: Option<&Path>,
172 image: subc_control::RunningImageAgreement,
173) -> PendingReloadVerdict {
174 let path = match spawned_from {
175 Some(spawned_from) if configured == spawned_from => ReloadPathAgreement::Match,
176 Some(spawned_from) => ReloadPathAgreement::Mismatch {
177 configured: configured.to_path_buf(),
178 spawned_from: spawned_from.to_path_buf(),
179 },
180 None => ReloadPathAgreement::Unavailable {
181 reason: if matches!(
182 image,
183 subc_control::RunningImageAgreement::Unavailable {
184 reason: subc_control::RunningImageUnavailableReason::NotRunning
185 }
186 ) {
187 ReloadPathUnavailableReason::NotRunning
188 } else {
189 ReloadPathUnavailableReason::SpawnedPathUnavailable
190 },
191 },
192 };
193 PendingReloadVerdict { path, image }
194}
195
196#[derive(Clone)]
197struct DaemonProvenanceFacts {
198 build: DaemonBuildProvenance,
199 pid: Option<u32>,
200 started_at_ms: Option<u64>,
201 start_clock: Option<crate::clock::StartClock>,
202 executable_path: Option<PathBuf>,
203 executable_identity: Option<SpawnedFileIdentity>,
204 process_start_time: Option<u64>,
205 probe: ExecutableIdentityProbe,
206}
207
208impl Default for DaemonProvenanceFacts {
209 fn default() -> Self {
210 Self {
211 build: DaemonBuildProvenance {
212 build_git_sha: None,
213 build_lock_digest: None,
214 },
215 pid: None,
216 started_at_ms: None,
217 start_clock: None,
218 executable_path: None,
219 executable_identity: None,
220 process_start_time: None,
221 probe: ExecutableIdentityProbe::default(),
222 }
223 }
224}
225
226#[derive(Debug, Clone)]
227struct SupervisorRescanContext {
228 supervisor: Supervisor,
229 config_path: PathBuf,
230 configured_port: Option<u16>,
231 storage_config: Option<crate::daemon_config::StorageConfig>,
232 admission_facts_carrier_module_id: Option<String>,
233 admission_facts_targets: Option<Vec<String>>,
234 scope_authority_owners: Vec<String>,
235}
236
237const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
258 "reloading",
259 "supervisor_not_live",
260 "registration_not_active",
261 "no_forwarding_connection",
262 "relay_send_failed",
263];
264
265#[derive(Clone)]
267pub struct ControlHandler {
268 registry: Arc<Registry>,
269 forwarding: Arc<ForwardingTable>,
270 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
271 supervisor: SupervisorHandle,
272 subc_capabilities: Arc<[String]>,
273 route_bind_relay_timeout: Duration,
277 route_bind_relay_timeouts: BTreeMap<String, Duration>,
281 route_bind_breakers: RouteBindBreakers,
284 route_bind_concurrency: RouteBindConcurrency,
287 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
292 route_bind_breaker_threshold: u32,
294 route_bind_breaker_cooldown: Duration,
296 health_probe_timeout: Duration,
297 storage_config: Option<crate::daemon_config::StorageConfig>,
300 machine_id: Option<crate::machine_id::MachineId>,
304 admission_facts_carrier_module_id: Option<String>,
305 admission_facts_targets: Option<Vec<String>>,
306 scopes: Arc<RwLock<ScopeTable>>,
310 scope_authority_owners: Vec<String>,
313 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
316 rescan: Option<SupervisorRescanContext>,
317 connected_clients: ConnectedClients,
318 counters: DaemonCounters,
319 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
320 daemon_provenance: DaemonProvenanceFacts,
321 #[cfg(test)]
322 control_dispatch_delay: Option<Duration>,
323 #[cfg(test)]
324 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
325 #[cfg(test)]
329 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
330}
331
332impl fmt::Debug for ControlHandler {
333 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
334 f.debug_struct("ControlHandler")
335 .field("registry", &self.registry)
336 .field("forwarding", &self.forwarding)
337 .field("process_liveness", &self.process_liveness.is_some())
338 .field("supervisor", &self.supervisor)
339 .field("subc_capabilities", &self.subc_capabilities)
340 .finish()
341 }
342}
343
344struct RouteOpenRequest {
345 target: RouteTarget,
346 identity: BindIdentity,
347 consumer_identity: Option<ConsumerIdentity>,
348 consumer_capabilities: Option<Vec<String>>,
349 role_versions: Option<BTreeMap<String, String>>,
350 admission_facts: Option<serde_json::Value>,
351 scope: Option<ScopeSelector>,
352}
353
354struct RouteBindReservationGuard {
355 forwarding: Arc<ForwardingTable>,
356 endpoint: ModuleEndpointId,
357 relay_corr: u64,
358 armed: bool,
359}
360
361struct ModuleControlRpcGuard {
362 forwarding: Arc<ForwardingTable>,
363 endpoint: ModuleEndpointId,
364 corr: u64,
365 armed: bool,
366}
367
368impl ModuleControlRpcGuard {
369 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
370 Self {
371 forwarding,
372 endpoint,
373 corr,
374 armed: true,
375 }
376 }
377
378 fn disarm(&mut self) {
379 self.armed = false;
380 }
381}
382
383impl Drop for ModuleControlRpcGuard {
384 fn drop(&mut self) {
385 if self.armed {
386 let _ = self
387 .forwarding
388 .cancel_module_control_rpc(self.endpoint, self.corr);
389 }
390 }
391}
392
393impl RouteBindReservationGuard {
394 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
395 Self {
396 forwarding,
397 endpoint,
398 relay_corr,
399 armed: true,
400 }
401 }
402
403 fn release_and_disarm(&mut self) {
404 if !self.armed {
405 return;
406 }
407 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
408 self.endpoint,
409 self.relay_corr,
410 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
411 ) {
412 send_goodbye_target_best_effort(
413 &self.forwarding.counters(),
414 &target,
415 "canceled route.bind",
416 );
417 }
418 self.armed = false;
419 }
420
421 fn disarm(&mut self) {
422 self.armed = false;
423 }
424}
425
426impl Drop for RouteBindReservationGuard {
427 fn drop(&mut self) {
428 self.release_and_disarm();
429 }
430}
431
432#[derive(Debug, Clone, Default)]
462pub(crate) struct RouteBindBreakers {
463 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
464}
465
466#[derive(Debug, Clone, Default)]
467pub(crate) struct RouteBindConcurrency {
468 modules: Arc<Mutex<HashMap<String, usize>>>,
469}
470
471struct RouteBindConcurrencyGuard {
472 concurrency: RouteBindConcurrency,
473 module_id: String,
474}
475
476impl RouteBindConcurrency {
477 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
480 let mut modules = self
481 .modules
482 .lock()
483 .expect("route.bind concurrency mutex poisoned");
484 let in_flight = modules.entry(module_id.to_string()).or_default();
485 if *in_flight >= limit {
486 return Err(*in_flight);
487 }
488 *in_flight += 1;
489 Ok(RouteBindConcurrencyGuard {
490 concurrency: self.clone(),
491 module_id: module_id.to_string(),
492 })
493 }
494}
495
496impl Drop for RouteBindConcurrencyGuard {
497 fn drop(&mut self) {
498 let mut modules = self
499 .concurrency
500 .modules
501 .lock()
502 .expect("route.bind concurrency mutex poisoned");
503 let remove = {
504 let in_flight = modules
505 .get_mut(&self.module_id)
506 .expect("admitted route.bind has a concurrency entry");
507 *in_flight -= 1;
508 *in_flight == 0
509 };
510 if remove {
511 modules.remove(&self.module_id);
512 }
513 }
514}
515
516#[derive(Debug, Default)]
517struct ModuleBreakerState {
518 consecutive_timeouts: u32,
520 cooldown_until: Option<Instant>,
523 probe_in_flight: Option<Arc<()>>,
528}
529
530enum RouteBindAdmission<'a> {
532 Admitted {
533 guard: RouteBindBreakerGuard<'a>,
534 probe: bool,
537 },
538 Refused {
539 consecutive_timeouts: u32,
540 retry_in: Duration,
544 probe_in_flight: bool,
545 },
546}
547
548struct RouteBindBreakerGuard<'a> {
555 breakers: RouteBindBreakers,
556 module_id: &'a str,
557 probe_token: Option<Arc<()>>,
558 settled: bool,
559}
560
561impl RouteBindBreakerGuard<'_> {
562 fn record_accepted(&mut self) -> bool {
566 self.settled = true;
567 self.breakers.record_accepted(self.module_id)
568 }
569
570 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
573 self.settled = true;
574 self.breakers.record_timeout(
575 self.module_id,
576 self.probe_token.as_ref(),
577 threshold,
578 cooldown,
579 )
580 }
581
582 fn record_inconclusive(&mut self) {
591 self.settled = true;
592 self.breakers
593 .record_inconclusive(self.module_id, self.probe_token.as_ref());
594 }
595}
596
597impl Drop for RouteBindBreakerGuard<'_> {
598 fn drop(&mut self) {
599 if !self.settled {
600 self.breakers
601 .record_inconclusive(self.module_id, self.probe_token.as_ref());
602 }
603 }
604}
605
606struct BreakerOpened {
610 consecutive_timeouts: u32,
611 reopened_after_probe: bool,
614}
615
616impl RouteBindBreakers {
617 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
618 self.modules
619 .lock()
620 .expect("route.bind breaker mutex poisoned")
621 }
622
623 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
626 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
627 probe: probe_token.is_some(),
628 guard: RouteBindBreakerGuard {
629 breakers: self.clone(),
630 module_id,
631 probe_token,
632 settled: false,
633 },
634 };
635
636 let mut modules = self.lock();
637 let Some(state) = modules.get_mut(module_id) else {
638 return admitted(None);
639 };
640 let Some(cooldown_until) = state.cooldown_until else {
641 return admitted(None);
642 };
643 if state.probe_in_flight.is_some() {
644 return RouteBindAdmission::Refused {
645 consecutive_timeouts: state.consecutive_timeouts,
646 retry_in: Duration::ZERO,
647 probe_in_flight: true,
648 };
649 }
650 let now = Instant::now();
651 if now < cooldown_until {
652 return RouteBindAdmission::Refused {
653 consecutive_timeouts: state.consecutive_timeouts,
654 retry_in: cooldown_until - now,
655 probe_in_flight: false,
656 };
657 }
658 let token = Arc::new(());
659 state.probe_in_flight = Some(Arc::clone(&token));
660 admitted(Some(token))
661 }
662
663 fn record_accepted(&self, module_id: &str) -> bool {
664 self.lock()
665 .remove(module_id)
666 .is_some_and(|state| state.cooldown_until.is_some())
667 }
668
669 fn record_timeout(
670 &self,
671 module_id: &str,
672 probe_token: Option<&Arc<()>>,
673 threshold: u32,
674 cooldown: Duration,
675 ) -> Option<BreakerOpened> {
676 let mut modules = self.lock();
677 let state = modules.entry(module_id.to_string()).or_default();
678 let was_open = state.cooldown_until.is_some();
679 let was_probe = Self::owns_probe(state, probe_token);
680 if was_probe {
681 state.probe_in_flight = None;
682 }
683 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
684 if state.consecutive_timeouts < threshold {
685 return None;
686 }
687 state.cooldown_until = Some(Instant::now() + cooldown);
688 Some(BreakerOpened {
689 consecutive_timeouts: state.consecutive_timeouts,
690 reopened_after_probe: was_open && was_probe,
691 })
692 }
693
694 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
695 state
699 .probe_in_flight
700 .as_ref()
701 .zip(token)
702 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
703 }
704
705 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
706 if let Some(state) = self.lock().get_mut(module_id) {
707 if Self::owns_probe(state, probe_token) {
708 state.probe_in_flight = None;
709 }
710 }
711 }
712
713 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
728 self.lock()
729 .remove(module_id)
730 .map(|state| state.consecutive_timeouts)
731 .filter(|discarded| *discarded > 0)
732 }
733
734 fn open_snapshot(&self) -> Option<serde_json::Value> {
741 let now = Instant::now();
742 let modules = self.lock();
743 let open = modules
744 .iter()
745 .filter_map(|(module_id, state)| {
746 let cooldown_until = state.cooldown_until?;
747 Some((
748 module_id.clone(),
749 serde_json::json!({
750 "consecutive_timeouts": state.consecutive_timeouts,
751 "cooldown_remaining_ms":
752 cooldown_until.saturating_duration_since(now).as_millis() as u64,
753 "probe_in_flight": state.probe_in_flight.is_some(),
754 }),
755 ))
756 })
757 .collect::<serde_json::Map<String, serde_json::Value>>();
758 (!open.is_empty()).then_some(serde_json::Value::Object(open))
759 }
760}
761
762impl ControlHandler {
763 pub fn new(registry: Arc<Registry>) -> Self {
764 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
765 }
766
767 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
768 let counters = forwarding.counters();
769 let route_bind_breakers = forwarding.route_bind_breakers();
773 let route_bind_concurrency = forwarding.route_bind_concurrency();
774 let route_outages = forwarding.route_outages();
775 Self {
776 registry,
777 forwarding,
778 process_liveness: None,
779 supervisor: SupervisorHandle::new(),
780 subc_capabilities: Arc::from([
781 CAP_MANIFEST_REGISTRATION.to_string(),
782 CAP_CHANNEL_LIFECYCLE.to_string(),
783 CAP_PING_PONG.to_string(),
784 CAP_SESSION_ATTACH.to_string(),
785 CAP_ADMISSION_FACTS_RELAY.to_string(),
786 CAP_SCOPES_V1.to_string(),
787 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
788 ]),
789 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
790 route_bind_relay_timeouts: BTreeMap::new(),
791 route_bind_breakers,
792 route_bind_concurrency,
793 route_outages,
794 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
795 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
796 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
797 storage_config: None,
798 machine_id: None,
799 admission_facts_carrier_module_id: None,
800 admission_facts_targets: None,
801 scopes: Arc::new(RwLock::new(ScopeTable::new(
802 crate::daemon_config::default_scope_authority_owners(),
803 ))),
804 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
805 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
806 rescan: None,
807 connected_clients: ConnectedClients::new(),
808 counters,
809 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
810 daemon_provenance: DaemonProvenanceFacts::default(),
811 #[cfg(test)]
812 control_dispatch_delay: None,
813 #[cfg(test)]
814 provenance_probe_override: None,
815 #[cfg(test)]
816 route_bind_relay_sent_hold: None,
817 }
818 }
819
820 pub fn with_storage_config(
823 mut self,
824 storage_config: Option<crate::daemon_config::StorageConfig>,
825 ) -> Self {
826 self.storage_config = storage_config;
827 self
828 }
829
830 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
833 self.machine_id = machine_id;
834 self
835 }
836
837 pub fn with_admission_facts_config(
841 mut self,
842 carrier_module_id: Option<String>,
843 targets: Option<Vec<String>>,
844 ) -> Self {
845 self.admission_facts_carrier_module_id = carrier_module_id;
846 self.admission_facts_targets = targets;
847 self
848 }
849
850 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
854 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
855 self.scope_authority_owners = owners;
856 self
857 }
858
859 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
862 self.route_bind_relay_timeout = timeout;
863 self
864 }
865
866 pub fn with_route_bind_relay_timeouts(
872 mut self,
873 timeouts: impl IntoIterator<Item = (String, Duration)>,
874 ) -> Self {
875 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
876 self
877 }
878
879 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
885 self.route_bind_relay_timeouts
886 .get(module_id)
887 .copied()
888 .unwrap_or(self.route_bind_relay_timeout)
889 }
890
891 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
899 self.route_bind_breaker_threshold = threshold.max(1);
900 self.route_bind_breaker_cooldown = cooldown;
901 self
902 }
903
904 #[cfg(test)]
905 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
906 self.health_probe_timeout = timeout;
907 self
908 }
909
910 #[cfg(test)]
911 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
912 self.control_dispatch_delay = Some(delay);
913 self
914 }
915
916 #[cfg(test)]
920 pub(crate) fn with_route_bind_relay_sent_hold(
921 mut self,
922 hold: Arc<tokio::sync::Barrier>,
923 ) -> Self {
924 self.route_bind_relay_sent_hold = Some(hold);
925 self
926 }
927
928 pub fn with_process_liveness(
929 mut self,
930 process_liveness: Arc<dyn ModuleProcessLiveness>,
931 ) -> Self {
932 self.process_liveness = Some(process_liveness);
933 self
934 }
935
936 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
937 self.supervisor = supervisor;
938 self
939 }
940
941 pub fn with_daemon_provenance(
942 mut self,
943 pid: u32,
944 started_at_ms: u64,
945 executable_path: Option<PathBuf>,
946 build_git_sha: Option<String>,
947 build_lock_digest: Option<String>,
948 ) -> Self {
949 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
950 let process_start_time = process_start_time(pid);
951 self.daemon_provenance = DaemonProvenanceFacts {
952 build: DaemonBuildProvenance {
953 build_git_sha,
954 build_lock_digest,
955 },
956 pid: Some(pid),
957 started_at_ms: Some(started_at_ms),
958 start_clock: None,
959 executable_path,
960 executable_identity,
961 process_start_time,
962 probe: ExecutableIdentityProbe::default(),
963 };
964 self
965 }
966
967 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
968 self.daemon_provenance.start_clock = Some(clock);
969 self
970 }
971
972 #[cfg(test)]
973 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
974 self.provenance_probe_override = Some(result);
975 self
976 }
977
978 pub fn with_capability_config(
982 self,
983 modules: impl IntoIterator<Item = (String, bool)>,
984 reserved_capabilities: BTreeMap<String, String>,
985 ) -> Self {
986 self.capability_evaluator
987 .configure(modules, reserved_capabilities);
988 self
989 }
990
991 pub fn with_supervisor_rescan(
992 mut self,
993 supervisor: Supervisor,
994 config_path: impl Into<PathBuf>,
995 configured_port: Option<u16>,
996 ) -> Self {
997 self.rescan = Some(SupervisorRescanContext {
998 supervisor,
999 config_path: config_path.into(),
1000 configured_port,
1001 storage_config: self.storage_config.clone(),
1002 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1003 admission_facts_targets: self.admission_facts_targets.clone(),
1004 scope_authority_owners: self.scope_authority_owners.clone(),
1005 });
1006 self
1007 }
1008
1009 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1010 self.connected_clients = connected_clients;
1011 self
1012 }
1013
1014 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1015 Arc::clone(&self.forwarding)
1016 }
1017
1018 pub(crate) fn counters(&self) -> DaemonCounters {
1019 self.counters.clone()
1020 }
1021
1022 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1025 tokio::spawn(async move {
1026 loop {
1027 self.capability_evaluator
1028 .wait_for_change_or_deadline()
1029 .await;
1030 self.refresh_capability_requirements();
1031 }
1032 });
1033 }
1034
1035 fn runtime_capability_snapshot(
1036 &self,
1037 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1038 let runtime = self
1039 .supervisor
1040 .list()
1041 .into_iter()
1042 .map(|module| {
1043 let status = module.status().map_err(|err| {
1044 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1045 })?;
1046 Ok(RuntimeModule {
1047 module_id: status.module_id,
1048 state: status.state,
1049 enabled: status.enabled,
1050 })
1051 })
1052 .collect::<Result<Vec<_>, RouterError>>()?;
1053 let (_, registrations) = self.registry.list_modules().map_err(|err| {
1054 RouterError::backend(
1055 0,
1056 0,
1057 format!("failed to list capability registrations: {err}"),
1058 )
1059 })?;
1060 let registrations = registrations
1061 .into_iter()
1062 .map(|registration| RegisteredModule {
1063 module_id: registration.manifest.module_id,
1064 module_version: registration.manifest.module_version,
1065 capabilities: registration.manifest.capabilities,
1066 })
1067 .collect();
1068 Ok((runtime, registrations))
1069 }
1070
1071 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1078 let cached_registration = RegisteredModule {
1079 module_id: registration.manifest.module_id.clone(),
1080 module_version: registration.manifest.module_version.clone(),
1081 capabilities: registration.manifest.capabilities.clone(),
1082 };
1083 if self.capability_evaluator.record_hello(&cached_registration) {
1084 warn!(
1085 module_id = %cached_registration.module_id,
1086 "capability claims drifted from the cached manifest"
1087 );
1088 }
1089 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1090 self.enforce_capability_denies();
1091 }
1092 self.refresh_capability_requirements();
1093 }
1094
1095 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1099 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1100 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1101 self.supervisor.set_swap_promotion_observer(observer);
1102 }
1103
1104 pub fn refresh_capability_requirements(&self) {
1105 match self.runtime_capability_snapshot() {
1106 Ok((runtime, registrations)) => {
1107 log_requirement_events(
1108 self.capability_evaluator
1109 .evaluate_now(&runtime, ®istrations),
1110 );
1111 }
1112 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1113 }
1114 }
1115
1116 fn enforce_capability_denies(&self) {
1120 let (_, registrations) = match self.registry.list_modules() {
1121 Ok(snapshot) => snapshot,
1122 Err(err) => {
1123 warn!(error = %err, "failed to read registrations for capability deny census");
1124 return;
1125 }
1126 };
1127 let manifests = registrations
1128 .into_iter()
1129 .map(|registration| {
1130 (
1131 registration.manifest.module_id.clone(),
1132 registration.manifest,
1133 )
1134 })
1135 .collect::<BTreeMap<_, _>>();
1136 let census = match self.forwarding.route_census(None) {
1137 Ok(census) => census,
1138 Err(err) => {
1139 warn!(error = %err, "failed to read route census for capability deny enforcement");
1140 return;
1141 }
1142 };
1143
1144 for (target_module_id, routes) in census {
1145 let Some(target_manifest) = manifests.get(&target_module_id) else {
1146 continue;
1147 };
1148 let mut closed_routes = Vec::new();
1149 let mut module_goodbyes = Vec::new();
1150 for route in routes {
1151 let Principal::Reserved {
1152 module_id: opening_module_id,
1153 } = &route.principal
1154 else {
1155 continue;
1156 };
1157 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1158 continue;
1159 };
1160 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1161 continue;
1162 };
1163
1164 match self.forwarding.release_client_route(
1165 route.goodbye_target.connection_id,
1166 route.goodbye_target.channel,
1167 route.goodbye_target.epoch,
1168 ) {
1169 Ok(RouteRelease::Removed(module_goodbye)) => {
1170 warn!(
1171 opening_module_id,
1172 target_module_id,
1173 capability,
1174 "force-closing route because an attested capability deny edge now matches"
1175 );
1176 closed_routes.push(route);
1177 module_goodbyes.push(module_goodbye);
1178 }
1179 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1180 Err(err) => warn!(
1181 opening_module_id,
1182 target_module_id,
1183 capability,
1184 error = %err,
1185 "failed to force-close capability-denied route"
1186 ),
1187 }
1188 }
1189
1190 if closed_routes.is_empty() {
1191 continue;
1192 }
1193 send_route_control_pushes(
1194 &self.forwarding,
1195 closed_routes,
1196 ClientControlPush::RouteClosed {
1197 module_id: target_module_id,
1198 channels: Vec::new(),
1199 reason: RouteCloseReason::CapabilityDenied,
1200 drained: false,
1201 abandoned: 0,
1202 excluded_subscriptions: 0,
1203 terminal: Some(false),
1204 },
1205 );
1206 self.emit_route_goodbyes(module_goodbyes);
1207 }
1208 }
1209
1210 fn not_ready_reason(
1216 &self,
1217 registration: &crate::registry::ModuleRegistration,
1218 ) -> Option<NotReadyReason> {
1219 if !registration.ready {
1220 return Some(NotReadyReason {
1221 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1222 capability: None,
1223 });
1224 }
1225 self.first_unprovided_required_capability(registration)
1226 .map(|capability| NotReadyReason {
1227 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1228 capability: Some(capability),
1229 })
1230 }
1231
1232 fn first_unprovided_required_capability(
1250 &self,
1251 registration: &crate::registry::ModuleRegistration,
1252 ) -> Option<String> {
1253 let required = registration
1254 .manifest
1255 .capabilities
1256 .iter()
1257 .flat_map(|declarations| declarations.requires.iter())
1258 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1259 .map(|requirement| requirement.capability.as_str())
1260 .collect::<BTreeSet<_>>();
1261 if required.is_empty() {
1262 return None;
1263 }
1264 let module_id = registration.manifest.module_id.as_str();
1265 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1266 if required
1267 .iter()
1268 .any(|capability| verdict(capability).is_none())
1269 {
1270 self.refresh_capability_requirements();
1271 }
1272 required
1273 .into_iter()
1274 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1275 .map(str::to_string)
1276 }
1277
1278 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1279 self.capability_evaluator
1280 .statuses()
1281 .into_iter()
1282 .map(capability_requirement_status)
1283 .collect()
1284 }
1285
1286 fn deregister_connection(
1293 &self,
1294 connection_id: ConnectionId,
1295 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1296 self.registry.deregister_connection(connection_id)
1297 }
1298
1299 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1300 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1301 return None;
1302 }
1303 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1304 parse_client_control_request(&frame.body)
1305 else {
1306 return None;
1307 };
1308 Some(target_module_id(&target).to_string())
1309 }
1310
1311 pub(crate) fn route_open_capacity_refusal(
1312 &self,
1313 ctx: &RouteCtx,
1314 frame: &Frame,
1315 target_module_id: &str,
1316 in_flight: usize,
1317 limit: usize,
1318 ) -> Result<Frame, RouterError> {
1319 self.route_open_admission_refusal_frame(
1320 ctx,
1321 frame,
1322 target_module_id,
1323 "open_admission_full",
1324 (in_flight, limit),
1325 format!(
1326 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1327 ),
1328 )
1329 }
1330
1331 fn route_open_target_capacity_refusal(
1332 &self,
1333 ctx: &RouteCtx,
1334 frame: &Frame,
1335 target_module_id: &str,
1336 in_flight: usize,
1337 ) -> Result<Frame, RouterError> {
1338 self.route_open_admission_refusal_frame(
1339 ctx,
1340 frame,
1341 target_module_id,
1342 "target_binds_full",
1343 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1344 format!(
1345 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1346 ),
1347 )
1348 }
1349
1350 fn route_open_admission_refusal_frame(
1358 &self,
1359 ctx: &RouteCtx,
1360 frame: &Frame,
1361 target_module_id: &str,
1362 reason: &'static str,
1363 (in_flight, limit): (usize, usize),
1364 message: impl Into<String>,
1365 ) -> Result<Frame, RouterError> {
1366 let code = error_codes::TARGET_UNAVAILABLE;
1367 self.counters.increment_route_open_refused(code);
1368 info!(
1369 target: "control",
1370 code,
1371 reason,
1372 module_id = ?target_module_id,
1373 connection_id = ctx.connection_id.get(),
1374 in_flight,
1375 limit,
1376 "route.open refused"
1377 );
1378 control_error_frame(frame, code, message.into())
1379 }
1380
1381 #[cfg(test)]
1387 pub fn handle_control(
1388 &self,
1389 connection_id: ConnectionId,
1390 frame: Frame,
1391 ) -> Result<Vec<Frame>, RouterError> {
1392 match frame.header.ty {
1393 FrameType::Ping => Ok(vec![pong(&frame)?]),
1394 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1395 FrameType::Goodbye => self.handle_goodbye(connection_id),
1396 ty => Ok(vec![control_error_frame(
1397 &frame,
1398 "unsupported_control_frame",
1399 format!("unsupported channel-0 frame {ty:?}"),
1400 )?]),
1401 }
1402 }
1403
1404 pub async fn handle_control_frame(
1405 &self,
1406 ctx: &RouteCtx,
1407 frame: Frame,
1408 ) -> Result<Vec<Frame>, RouterError> {
1409 self.handle_control_frame_timed(ctx, frame, None).await
1410 }
1411
1412 pub(crate) async fn handle_control_frame_timed(
1413 &self,
1414 ctx: &RouteCtx,
1415 frame: Frame,
1416 dispatch_started_at: Option<StdInstant>,
1417 ) -> Result<Vec<Frame>, RouterError> {
1418 match frame.header.ty {
1419 FrameType::Ping => Ok(vec![pong(&frame)?]),
1420 FrameType::Hello => {
1421 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1422 }
1423 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1424 FrameType::Cancel => {
1425 if self
1426 .supervisor
1427 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1428 {
1429 Ok(Vec::new())
1430 } else {
1431 Ok(vec![control_error_frame(
1432 &frame,
1433 "unknown_subscription",
1434 "no supervisor spawn subscription has this correlation id",
1435 )?])
1436 }
1437 }
1438 FrameType::Request => {
1439 if self
1440 .forwarding
1441 .module_endpoint_for_connection(ctx.connection_id)
1442 .map_err(RouterError::Forwarding)?
1443 .is_some()
1444 {
1445 if !is_known_module_request_op(&frame.body) {
1446 return Ok(vec![control_error_frame(
1447 &frame,
1448 "unsupported_control_frame",
1449 "module-originated channel-0 REQUEST is not supported",
1450 )?]);
1451 }
1452 let request = match parse_module_control_request_from_module(&frame.body) {
1453 Ok(request) => request,
1454 Err((err, ControlRequestBodyError::UnknownOp)) => {
1455 return Ok(vec![control_error_frame(
1456 &frame,
1457 "unsupported_control_frame",
1458 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1459 )?])
1460 }
1461 Err((err, ControlRequestBodyError::InvalidBody)) => {
1462 return Ok(vec![control_error_frame(
1463 &frame,
1464 "invalid_control_body",
1465 format!("malformed module control body: {err}"),
1466 )?])
1467 }
1468 };
1469 let op = module_control_request_op(&request);
1470 let corr = frame.header.corr;
1471 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1472 let result =
1473 self.handle_module_control_request(ctx.connection_id, frame, request);
1474 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1475 return result;
1476 }
1477
1478 if is_known_module_request_op(&frame.body) {
1479 return Ok(vec![control_error_frame(
1480 &frame,
1481 "not_registered",
1482 "catalog.update requires an active module registration owned by this connection",
1483 )?]);
1484 }
1485
1486 let request = match parse_client_control_request(&frame.body) {
1487 Ok(request) => request,
1488 Err((err, ControlRequestBodyError::UnknownOp)) => {
1489 return Ok(vec![control_error_frame(
1490 &frame,
1491 "unknown_control_op",
1492 format!("unknown client control op: {err}"),
1493 )?])
1494 }
1495 Err((err, ControlRequestBodyError::InvalidBody)) => {
1496 return Ok(vec![control_error_frame(
1497 &frame,
1498 "invalid_control_body",
1499 format!("malformed client control body: {err}"),
1500 )?])
1501 }
1502 };
1503 let op = client_control_request_op(&request);
1504 let corr = frame.header.corr;
1505 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1506 #[cfg(test)]
1507 if let Some(delay) = self.control_dispatch_delay {
1508 tokio::time::sleep(delay).await;
1509 }
1510 let result = self
1511 .handle_client_control_request(ctx, frame, request)
1512 .await;
1513 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1514 result
1515 }
1516 FrameType::Push => {
1517 let Some(endpoint) = self
1518 .forwarding
1519 .module_endpoint_for_connection(ctx.connection_id)
1520 .map_err(RouterError::Forwarding)?
1521 else {
1522 return Ok(vec![control_error_frame(
1523 &frame,
1524 "unsupported_control_frame",
1525 "client-originated channel-0 PUSH is not supported",
1526 )?]);
1527 };
1528 self.handle_status_update(endpoint, frame)
1529 }
1530 FrameType::Response | FrameType::Error
1531 if self
1532 .forwarding
1533 .module_endpoint_for_connection(ctx.connection_id)
1534 .map_err(RouterError::Forwarding)?
1535 .is_some() =>
1536 {
1537 self.handle_module_relay_response(ctx.connection_id, frame)
1538 }
1539 ty => Ok(vec![control_error_frame(
1540 &frame,
1541 "unsupported_control_frame",
1542 format!("unsupported channel-0 frame {ty:?}"),
1543 )?]),
1544 }
1545 }
1546
1547 pub fn cleanup_connection(
1548 &self,
1549 connection_id: ConnectionId,
1550 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1551 let crash_closed = self
1552 .registry
1553 .get_module_by_connection(connection_id)?
1554 .and_then(|registration| {
1555 self.forwarding
1556 .module_endpoint_for_connection(connection_id)
1557 .ok()
1558 .flatten()
1559 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1560 .map(|routes| (registration.manifest.module_id, routes))
1561 });
1562 let crash_closed = crash_closed.map(|(module_id, routes)| {
1563 let terminal = match self.supervisor.get(&module_id) {
1564 None => false,
1565 Some(module) => match module.will_recover_after_connection_loss() {
1566 Ok(will_recover) => !will_recover,
1567 Err(err) => {
1568 warn!(
1569 %module_id,
1570 error = %err,
1571 "failed to read crash recovery verdict; reporting non-terminal conservatively"
1572 );
1573 false
1574 }
1575 },
1576 };
1577 let reason = match self.forwarding.is_daemon_draining() {
1581 Ok(true) => RouteCloseReason::Restart,
1582 Ok(false) => RouteCloseReason::Crash,
1583 Err(err) => {
1584 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
1585 RouteCloseReason::Crash
1586 }
1587 };
1588 (module_id, routes, reason, terminal)
1589 });
1590 let registrations = self.deregister_connection(connection_id);
1591 let cleanup = if crash_closed.is_some() {
1592 self.forwarding.cleanup_connection_counted(connection_id)
1593 } else {
1594 self.forwarding
1600 .cleanup_connection(connection_id)
1601 .map(|released| crate::forwarding::ConnectionCleanup {
1602 released,
1603 abandoned_relays: 0,
1604 })
1605 };
1606 if let Some((module_id, routes, reason, terminal)) = crash_closed {
1611 let abandoned = cleanup
1612 .as_ref()
1613 .map_or(0, |cleanup| cleanup.abandoned_relays);
1614 send_route_control_pushes(
1615 &self.forwarding,
1616 routes,
1617 ClientControlPush::RouteClosed {
1618 module_id,
1619 channels: Vec::new(),
1620 reason,
1621 drained: false,
1622 abandoned,
1623 excluded_subscriptions: 0,
1624 terminal: Some(terminal),
1625 },
1626 );
1627 }
1628 if let Ok(cleanup) = cleanup {
1629 self.emit_route_goodbyes(cleanup.released);
1630 }
1631 if matches!(®istrations, Ok(r) if !r.is_empty()) {
1635 crate::supervise::notify_registration_release();
1636 self.capability_evaluator.wake_deadline_loop();
1637 self.refresh_capability_requirements();
1638 }
1639 self.supervisor.remove_spawn_subscribers(connection_id);
1640 self.hello_launch_nonces
1643 .lock()
1644 .unwrap_or_else(|poisoned| poisoned.into_inner())
1645 .forget(connection_id);
1646 self.scopes
1647 .write()
1648 .unwrap_or_else(|poisoned| poisoned.into_inner())
1649 .release_connection(connection_id);
1650 registrations
1651 }
1652
1653 pub(crate) fn handle_route_goodbye(
1654 &self,
1655 connection_id: ConnectionId,
1656 route_channel: u16,
1657 route_epoch: u32,
1658 ) -> Result<bool, RouterError> {
1659 debug!(
1660 connection_id = connection_id.get(),
1661 route_channel, route_epoch, "handling route GOODBYE"
1662 );
1663 let RouteRelease::Removed(released_route) = self
1664 .forwarding
1665 .release_client_route(connection_id, route_channel, route_epoch)
1666 .map_err(RouterError::Forwarding)?
1667 else {
1668 return Ok(false);
1669 };
1670 self.emit_route_goodbyes(vec![released_route]);
1671 Ok(true)
1672 }
1673
1674 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
1675 for released in released_routes {
1676 let frame = match Frame::build_with_version(
1677 released.negotiated_ver,
1678 FrameType::Goodbye,
1679 control_flags(),
1680 released.channel,
1681 released.epoch,
1682 0,
1683 Vec::new(),
1684 ) {
1685 Ok(frame) => frame,
1686 Err(err) => {
1687 warn!(
1688 route_channel = released.channel,
1689 error = %err,
1690 "failed to build route GOODBYE frame"
1691 );
1692 continue;
1693 }
1694 };
1695 if !released.close_on_delivery_failure() {
1696 crate::forwarding::send_module_route_goodbye(
1697 &self.counters,
1698 &released.sink,
1699 frame,
1700 released.module_id.as_deref(),
1701 "client route released",
1702 );
1703 continue;
1704 }
1705 if let Err(err) = released.sink.try_send(frame) {
1706 warn!(
1707 target_connection_id = released.connection_id.get(),
1708 route_channel = released.channel,
1709 error = %err,
1710 "route GOODBYE was not delivered to client; closing target connection"
1711 );
1712 if self
1713 .forwarding
1714 .escalate_client_delivery_failure(
1715 released.connection_id,
1716 released.channel,
1717 released.epoch,
1718 CloseReason::new(
1719 "route_goodbye_delivery_failed",
1720 format!(
1721 "failed to enqueue route GOODBYE for channel {}: {err}",
1722 released.channel
1723 ),
1724 ),
1725 crate::forwarding::UndeliveredFrame {
1726 module_id: released.module_id.as_deref(),
1727 sink: &released.sink,
1728 },
1729 )
1730 .unwrap_or(false)
1731 {
1732 self.counters.increment_goodbye_relay_client_failed();
1733 }
1734 }
1735 }
1736 }
1737
1738 fn send_abandoned_route_bind_goodbye(
1750 &self,
1751 module_sink: &crate::FrameSink,
1752 negotiated_ver: u8,
1753 module_channel: u16,
1754 module_epoch: u32,
1755 ) {
1756 let frame = match Frame::build_with_version(
1757 negotiated_ver,
1758 FrameType::Goodbye,
1759 control_flags(),
1760 module_channel,
1761 module_epoch,
1762 0,
1763 Vec::new(),
1764 ) {
1765 Ok(frame) => frame,
1766 Err(err) => {
1767 warn!(
1768 route_channel = module_channel,
1769 error = %err,
1770 "failed to build GOODBYE for abandoned route.bind"
1771 );
1772 return;
1773 }
1774 };
1775 crate::forwarding::send_module_route_goodbye(
1776 &self.counters,
1777 module_sink,
1778 frame,
1779 None,
1780 "abandoned route.bind",
1781 );
1782 }
1783
1784 fn handle_hello(
1785 &self,
1786 connection_id: ConnectionId,
1787 sink: Option<crate::FrameSink>,
1788 frame: Frame,
1789 ) -> Result<Vec<Frame>, RouterError> {
1790 debug!(
1791 connection_id = connection_id.get(),
1792 corr = frame.header.corr,
1793 "handling HELLO"
1794 );
1795 if self
1799 .registry
1800 .get_module_by_connection(connection_id)
1801 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
1802 .is_some()
1803 {
1804 return Ok(vec![control_error_frame(
1805 &frame,
1806 "invalid_hello",
1807 "connection is already registered as a module",
1808 )?]);
1809 }
1810 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
1811 Ok(value) => value,
1812 Err(err) => {
1813 return Ok(vec![control_error_frame(
1814 &frame,
1815 "invalid_hello",
1816 format!("malformed HELLO body: {err}"),
1817 )?])
1818 }
1819 };
1820 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
1821 return Ok(vec![control_error_frame(
1822 &frame,
1823 "invalid_capability_grammar",
1824 err.to_string(),
1825 )?]);
1826 }
1827 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
1828 return Ok(vec![control_error_frame(
1829 &frame,
1830 "invalid_manifest",
1831 err.to_string(),
1832 )?]);
1833 }
1834 if let Some(provenance) = hello_value
1835 .get("manifest")
1836 .and_then(|manifest| manifest.get("provenance"))
1837 {
1838 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
1839 return Ok(vec![control_error_frame(
1840 &frame,
1841 "invalid_manifest",
1842 format!("malformed manifest provenance: {err}"),
1843 )?]);
1844 }
1845 }
1846 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
1847 Ok(hello) => hello,
1848 Err(err) => {
1849 return Ok(vec![control_error_frame(
1850 &frame,
1851 "invalid_hello",
1852 format!("malformed HELLO body: {err}"),
1853 )?])
1854 }
1855 };
1856
1857 if hello.protocol_ver != hello.manifest.protocol_ver {
1858 return Ok(vec![control_error_frame(
1859 &frame,
1860 "invalid_manifest",
1861 format!(
1862 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
1863 hello.protocol_ver, hello.manifest.protocol_ver
1864 ),
1865 )?]);
1866 }
1867
1868 if hello.manifest.module_id.trim().is_empty() {
1869 return Ok(vec![control_error_frame(
1870 &frame,
1871 "invalid_manifest",
1872 "manifest module_id must not be empty",
1873 )?]);
1874 }
1875
1876 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
1877 Ok(negotiated_ver) => negotiated_ver,
1878 Err(message) => {
1879 return Ok(vec![control_error_frame(
1880 &frame,
1881 "version_unsupported",
1882 message,
1883 )?])
1884 }
1885 };
1886
1887 let swap_admission = self
1895 .supervisor
1896 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
1897 if swap_admission == SwapHelloAdmission::Refused {
1898 warn!(
1899 module_id = %hello.manifest.module_id,
1900 connection_id = connection_id.get(),
1901 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
1902 );
1903 return Ok(vec![control_error_frame(
1904 &frame,
1905 "swap_token_invalid",
1906 format!(
1907 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
1908 hello.manifest.module_id
1909 ),
1910 )?]);
1911 }
1912 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
1913
1914 if let Some(rejection) = (!swap_candidate)
1922 .then(|| {
1923 self.supervisor.reserved_hello_rejection(
1924 &hello.manifest.module_id,
1925 hello.launch_nonce.as_deref(),
1926 )
1927 })
1928 .flatten()
1929 {
1930 let message = match rejection {
1931 ReservedHelloRejection::Exact { module_id } => format!(
1932 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
1933 ),
1934 ReservedHelloRejection::Prefix {
1935 prefix,
1936 owner_module_id,
1937 } => format!(
1938 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
1939 hello.manifest.module_id
1940 ),
1941 };
1942 return Ok(vec![control_error_frame(
1943 &frame,
1944 "reserved_module",
1945 message,
1946 )?]);
1947 }
1948
1949 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
1950 &hello.manifest.module_id,
1951 hello.manifest.capabilities.as_ref(),
1952 );
1953 if let Some(refusal) = reserved_capability_refusals.first() {
1954 let capability = refusal.capability.clone();
1955 let bound_module = refusal.claimants[0].clone();
1956 log_duplicate_claim_events(reserved_capability_refusals);
1957 return Ok(vec![control_error_frame(
1958 &frame,
1959 "reserved_capability",
1960 format!(
1961 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
1962 capability, bound_module, hello.manifest.module_id
1963 ),
1964 )?]);
1965 }
1966
1967 if self
1970 .forwarding
1971 .connection_has_client_routes(connection_id)
1972 .map_err(RouterError::Forwarding)?
1973 {
1974 return Ok(vec![control_error_frame(
1975 &frame,
1976 "invalid_hello",
1977 "connection has open client routes and cannot also register as a module",
1978 )?]);
1979 }
1980
1981 self.hello_launch_nonces
1986 .lock()
1987 .unwrap_or_else(|poisoned| poisoned.into_inner())
1988 .record(connection_id, hello.launch_nonce.as_deref());
1989 let control_ops = effective_module_control_ops(hello.control_ops);
1990 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
1993 if swap_candidate {
1994 return self.register_swap_candidate(
1995 connection_id,
1996 sink,
1997 &frame,
1998 hello.manifest,
1999 negotiated_ver,
2000 control_ops,
2001 hello_ack,
2002 );
2003 }
2004 let registration = match self.registry.register_with_control_ops(
2005 hello.manifest,
2006 negotiated_ver,
2007 connection_id,
2008 control_ops,
2009 ) {
2010 Ok(registration) => registration,
2011 Err(RegistryError::DuplicateModuleId { module_id }) => {
2012 return Ok(vec![control_error_frame(
2013 &frame,
2014 "duplicate_module_id",
2015 format!(
2016 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2017 ),
2018 )?])
2019 }
2020 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2021 return Ok(vec![control_error_frame(
2022 &frame,
2023 "invalid_module_id",
2024 err.to_string(),
2025 )?])
2026 }
2027 Err(err) => {
2028 return Ok(vec![control_error_frame(
2029 &frame,
2030 "registry_error",
2031 err.to_string(),
2032 )?])
2033 }
2034 };
2035
2036 let reply = if let Some(sink) = sink {
2037 let concurrency = manifest_concurrency(®istration.manifest);
2053 if let Err(err) = self.forwarding.register_module_connection_acked(
2054 connection_id,
2055 registration.manifest.module_id.clone(),
2056 negotiated_ver,
2057 concurrency,
2058 sink,
2059 hello_ack,
2060 ) {
2061 if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
2065 crate::supervise::notify_registration_release();
2066 }
2067 return Ok(vec![control_error_frame(
2068 &frame,
2069 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2070 "invalid_hello"
2071 } else {
2072 forwarding_error_code(&err)
2073 },
2074 err.to_string(),
2075 )?]);
2076 }
2077 Vec::new()
2078 } else {
2079 vec![hello_ack]
2082 };
2083
2084 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2093 info!(
2094 module_id = %registration.manifest.module_id,
2095 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2096 );
2097 }
2098
2099 self.apply_registration_capabilities(®istration);
2100
2101 info!(
2102 module_id = %registration.manifest.module_id,
2103 module_version = %registration.manifest.module_version,
2104 negotiated_ver,
2105 routable_provider = manifest_provides_routable_role(®istration.manifest),
2106 connection_id = connection_id.get(),
2107 "module registered"
2108 );
2109
2110 Ok(reply)
2111 }
2112
2113 #[allow(clippy::too_many_arguments)]
2123 fn register_swap_candidate(
2124 &self,
2125 connection_id: ConnectionId,
2126 sink: Option<crate::FrameSink>,
2127 frame: &Frame,
2128 manifest: ModuleManifest,
2129 negotiated_ver: u8,
2130 control_ops: Vec<String>,
2131 hello_ack: Frame,
2132 ) -> Result<Vec<Frame>, RouterError> {
2133 let module_id = manifest.module_id.clone();
2134 let registration = match self.registry.register_candidate_with_control_ops(
2135 manifest,
2136 negotiated_ver,
2137 connection_id,
2138 control_ops,
2139 ) {
2140 Ok(registration) => registration,
2141 Err(RegistryError::DuplicateModuleId { module_id }) => {
2142 return Ok(vec![control_error_frame(
2143 frame,
2144 "duplicate_module_id",
2145 format!(
2146 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2147 ),
2148 )?])
2149 }
2150 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2151 return Ok(vec![control_error_frame(
2152 frame,
2153 "invalid_module_id",
2154 err.to_string(),
2155 )?])
2156 }
2157 Err(err) => {
2158 return Ok(vec![control_error_frame(
2159 frame,
2160 "registry_error",
2161 err.to_string(),
2162 )?])
2163 }
2164 };
2165 let reply = if let Some(sink) = sink {
2166 let concurrency = manifest_concurrency(®istration.manifest);
2170 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2171 connection_id,
2172 module_id.clone(),
2173 negotiated_ver,
2174 concurrency,
2175 sink,
2176 hello_ack,
2177 ) {
2178 if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
2179 crate::supervise::notify_registration_release();
2180 }
2181 return Ok(vec![control_error_frame(
2182 frame,
2183 forwarding_error_code(&err),
2184 err.to_string(),
2185 )?]);
2186 }
2187 Vec::new()
2188 } else {
2189 vec![hello_ack]
2190 };
2191 self.supervisor.mark_swap_candidate_admitted(&module_id);
2192 info!(
2193 module_id = %module_id,
2194 module_version = %registration.manifest.module_version,
2195 negotiated_ver,
2196 ready = registration.ready,
2197 connection_id = connection_id.get(),
2198 "swap candidate registered; not routable until cutover"
2199 );
2200 Ok(reply)
2201 }
2202
2203 fn build_hello_ack(
2204 &self,
2205 frame: &Frame,
2206 negotiated_ver: u8,
2207 module_id: &str,
2208 ) -> Result<Frame, RouterError> {
2209 let ack = ModuleHelloAckBody {
2210 negotiated_ver,
2211 subc_ops: module_subc_ops(),
2212 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2213 storage: self
2214 .storage_config
2215 .as_ref()
2216 .map(|cfg| cfg.descriptor_for(module_id)),
2217 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2218 };
2219 let body = serde_json::to_vec(&ack).map_err(|err| {
2220 RouterError::backend(
2221 0,
2222 frame.header.corr,
2223 format!("failed to encode HELLO_ACK: {err}"),
2224 )
2225 })?;
2226
2227 Frame::build_with_version(
2228 negotiated_ver,
2229 FrameType::HelloAck,
2230 control_flags(),
2231 0,
2232 0,
2233 frame.header.corr,
2234 body,
2235 )
2236 .map_err(RouterError::FrameBuild)
2237 }
2238
2239 async fn handle_client_control_request(
2240 &self,
2241 ctx: &RouteCtx,
2242 frame: Frame,
2243 request: ClientControlRequest,
2244 ) -> Result<Vec<Frame>, RouterError> {
2245 match request {
2246 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2247 ClientControlRequest::CatalogList { module_id } => {
2248 self.handle_catalog_list(frame, module_id)
2249 }
2250 ClientControlRequest::RouteOpen {
2251 target,
2252 identity,
2253 consumer_identity,
2254 consumer_capabilities,
2255 role_versions,
2256 admission_facts,
2257 scope,
2258 } => {
2259 self.handle_route_open(
2260 ctx,
2261 frame,
2262 RouteOpenRequest {
2263 target,
2264 identity,
2265 consumer_identity,
2266 consumer_capabilities,
2267 role_versions,
2268 admission_facts,
2269 scope,
2270 },
2271 )
2272 .await
2273 }
2274 ClientControlRequest::RoutePoll {
2275 route_channel,
2276 route_epoch,
2277 kind,
2278 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2279 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2280 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2281 self.handle_supervisor_spawn_snapshot(frame)
2282 }
2283 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2284 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2285 }
2286 ClientControlRequest::SupervisorRestart {
2287 module_id,
2288 drain_timeout_ms,
2289 } => {
2290 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms)
2291 .await
2292 }
2293 ClientControlRequest::SupervisorSwap {
2294 module_id,
2295 ready_timeout_ms,
2296 } => {
2297 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms)
2298 .await
2299 }
2300 ClientControlRequest::SupervisorReload { module_id } => {
2301 self.handle_supervisor_reload(frame, module_id).await
2302 }
2303 ClientControlRequest::SupervisorRescan { preview } => {
2304 self.handle_supervisor_rescan(frame, preview).await
2305 }
2306 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2307 self.handle_supervisor_release_reserved(frame, module_id)
2308 .await
2309 }
2310 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2311 self.handle_supervisor_set_enabled(frame, module_id, enabled)
2312 .await
2313 }
2314 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2315 self.handle_supervisor_health_probe(frame, module_id).await
2316 }
2317 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2318 ClientControlRequest::SupervisorRoutes { module_id } => {
2319 self.handle_supervisor_routes(frame, module_id)
2320 }
2321 ClientControlRequest::SupervisorProvenance { module_id } => {
2322 self.handle_supervisor_provenance(frame, module_id).await
2323 }
2324 ClientControlRequest::SupervisorStderrTail {
2325 module_id,
2326 max_lines,
2327 max_bytes,
2328 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2329 ClientControlRequest::SupervisorTerminals { module_id } => {
2330 self.handle_supervisor_terminals(frame, module_id).await
2331 }
2332 }
2333 }
2334
2335 fn handle_module_control_request(
2336 &self,
2337 connection_id: ConnectionId,
2338 frame: Frame,
2339 request: ModuleControlRequestFromModule,
2340 ) -> Result<Vec<Frame>, RouterError> {
2341 match request {
2342 ModuleControlRequestFromModule::CatalogUpdate {
2343 provides,
2344 capabilities,
2345 ready,
2346 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2347 ModuleControlRequestFromModule::LiveRoots {} => {
2348 let registered = self
2349 .registry
2350 .get_module_by_connection(connection_id)
2351 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2352 let Some(registration) = registered else {
2353 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2354 };
2355 let response = self
2356 .forwarding
2357 .live_roots(®istration.manifest.module_id)
2358 .map_err(RouterError::Forwarding)?;
2359 Ok(vec![control_response_body_frame(
2360 &frame,
2361 &response,
2362 "ModuleControlResponseToModule::LiveRoots",
2363 )?])
2364 }
2365 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
2366 self.handle_scope_sync(connection_id, frame, generation, scopes)
2367 }
2368 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
2369 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
2370 }
2371 }
2372 }
2373
2374 fn handle_scope_sync(
2378 &self,
2379 connection_id: ConnectionId,
2380 frame: Frame,
2381 generation: u64,
2382 scopes: Vec<ScopeRecord>,
2383 ) -> Result<Vec<Frame>, RouterError> {
2384 let Some(registration) = self
2385 .registry
2386 .get_module_by_connection(connection_id)
2387 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2388 else {
2389 return Ok(vec![control_error_frame(
2390 &frame,
2391 "not_registered",
2392 "scope.sync requires an active module registration owned by this connection",
2393 )?]);
2394 };
2395 let owner = registration.manifest.module_id;
2396 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
2397 let is_current_launch = |connection: ConnectionId| {
2398 self.hello_launch_nonces
2399 .lock()
2400 .unwrap_or_else(|poisoned| poisoned.into_inner())
2401 .presented(connection, current_nonce.as_deref())
2402 };
2403 let mut table = self
2408 .scopes
2409 .write()
2410 .unwrap_or_else(|poisoned| poisoned.into_inner());
2411 let outcome = table.sync(&owner, connection_id, is_current_launch, generation, scopes);
2412 let drained = match &outcome {
2413 Ok(applied) => self
2414 .forwarding
2415 .publish_scope_changes(&applied.tag_changes)
2416 .map_err(RouterError::Forwarding)?,
2417 Err(_) => Vec::new(),
2418 };
2419 drop(table);
2420 match outcome {
2421 Ok(applied) => {
2422 let counts = ScopeOutcomeCounts::of(&applied.results);
2423 info!(
2424 owner = %owner,
2425 generation,
2426 records = applied.results.len(),
2427 created = counts.created,
2428 replaced = counts.replaced,
2429 updated = counts.updated,
2430 unchanged = counts.unchanged,
2431 refused = counts.refused,
2432 ended = applied.ended.len(),
2433 tag_changes = applied.tag_changes.len(),
2434 routes_closed = drained.len(),
2435 "scope sync accepted"
2436 );
2437 for refused in applied
2443 .results
2444 .iter()
2445 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
2446 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
2447 {
2448 warn!(
2449 owner = %owner,
2450 generation,
2451 scope_ref = %refused.scope_ref,
2452 scope_epoch = refused.scope_epoch,
2453 code = refused.code.as_deref().unwrap_or(""),
2454 "scope record refused"
2455 );
2456 }
2457 self.close_scope_drained_routes(drained);
2458 let response = ModuleControlResponseToModule::ScopeSync {
2459 generation,
2460 results: applied.results,
2461 ended: applied.ended,
2462 };
2463 Ok(vec![control_response_body_frame(
2464 &frame,
2465 &response,
2466 "ModuleControlResponseToModule::ScopeSync",
2467 )?])
2468 }
2469 Err(refusal) => {
2470 info!(
2471 owner = %owner,
2472 generation,
2473 code = refusal.code,
2474 "scope sync refused"
2475 );
2476 Ok(vec![control_error_frame(
2477 &frame,
2478 refusal.code,
2479 refusal.message,
2480 )?])
2481 }
2482 }
2483 }
2484
2485 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
2491 if drained.is_empty() {
2492 return;
2493 }
2494 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
2495 BTreeMap::new();
2496 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
2497 for route in drained {
2498 warn!(
2499 module_id = %route.module_id,
2500 reason = ?route.reason,
2501 client_connection_id = route.client.connection_id.get(),
2502 route_channel = route.client.channel,
2503 "closing route because its scope changed"
2504 );
2505 pushes
2506 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
2507 .or_insert_with(|| (route.reason, Vec::new()))
2508 .1
2509 .push(EndpointRoute {
2510 goodbye_target: route.client.clone(),
2511 principal: Principal::Unverified,
2512 bound_at: Instant::now(),
2513 draining: false,
2514 drain_reason: None,
2515 });
2516 goodbyes.push(route.module);
2517 goodbyes.push(route.client);
2518 }
2519 for ((module_id, _), (reason, routes)) in pushes {
2520 send_route_control_pushes(
2521 &self.forwarding,
2522 routes,
2523 ClientControlPush::RouteClosed {
2524 module_id,
2525 channels: Vec::new(),
2526 reason,
2527 drained: false,
2528 abandoned: 0,
2529 excluded_subscriptions: 0,
2530 terminal: Some(false),
2531 },
2532 );
2533 }
2534 self.emit_route_goodbyes(goodbyes);
2535 }
2536
2537 fn handle_scope_describe(
2540 &self,
2541 connection_id: ConnectionId,
2542 frame: Frame,
2543 owner: Principal,
2544 scope_ref: String,
2545 ) -> Result<Vec<Frame>, RouterError> {
2546 let registered = self
2547 .registry
2548 .get_module_by_connection(connection_id)
2549 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2550 if registered.is_none() {
2551 return Ok(vec![control_error_frame(
2552 &frame,
2553 "not_registered",
2554 "scope.describe requires an active module registration owned by this connection",
2555 )?]);
2556 }
2557 let description = self
2558 .scopes
2559 .read()
2560 .unwrap_or_else(|poisoned| poisoned.into_inner())
2561 .describe(&owner, &scope_ref);
2562 let owner_configured = match &owner {
2563 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
2568 _ => false,
2569 };
2570 let response = ModuleControlResponseToModule::ScopeDescribe {
2571 status: description.status,
2572 scope_epoch: description.scope_epoch,
2573 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
2574 owner_synced: description.owner_synced,
2575 owner_configured,
2576 scope: description.stamp,
2577 };
2578 Ok(vec![control_response_body_frame(
2579 &frame,
2580 &response,
2581 "ModuleControlResponseToModule::ScopeDescribe",
2582 )?])
2583 }
2584
2585 fn handle_catalog_update(
2586 &self,
2587 connection_id: ConnectionId,
2588 frame: Frame,
2589 provides: Vec<ProviderRole>,
2590 capabilities: Option<CapabilityDeclarations>,
2591 ready: Option<bool>,
2592 ) -> Result<Vec<Frame>, RouterError> {
2593 self.refresh_capability_requirements();
2594 let Some(registration) = self
2595 .registry
2596 .get_module_by_connection(connection_id)
2597 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2598 else {
2599 return Ok(vec![control_error_frame(
2600 &frame,
2601 "not_registered",
2602 "catalog.update requires an active module registration owned by this connection",
2603 )?]);
2604 };
2605
2606 if let Some(message) =
2607 catalog_update_frozen_field_message(®istration.manifest, &provides)
2608 {
2609 return Ok(vec![control_error_frame(
2610 &frame,
2611 "catalog_update_frozen_field",
2612 message,
2613 )?]);
2614 }
2615
2616 let mut candidate = registration.manifest.clone();
2617 candidate.provides = provides.clone();
2618 candidate.capabilities = capabilities
2619 .clone()
2620 .or_else(|| registration.manifest.capabilities.clone());
2621 if let Err(err) = candidate.validate_capability_grammar() {
2622 return Ok(vec![control_error_frame(
2623 &frame,
2624 "invalid_capability_grammar",
2625 err.to_string(),
2626 )?]);
2627 }
2628
2629 let mut conflicts = self
2632 .capability_evaluator
2633 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
2634 if let Some(conflict) = conflicts.first() {
2635 let message = format!(
2636 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2637 conflict.capability, conflict.claimants[0], candidate.module_id
2638 );
2639 for conflict in &mut conflicts {
2640 conflict.source = DuplicateClaimSource::CatalogUpdate;
2641 }
2642 log_duplicate_claim_events(conflicts);
2643 return Ok(vec![control_error_frame(
2644 &frame,
2645 "reserved_capability",
2646 message,
2647 )?]);
2648 }
2649
2650 let updated = self
2651 .registry
2652 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
2653 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2654 if updated.is_none() {
2655 return Ok(vec![control_error_frame(
2656 &frame,
2657 "not_registered",
2658 "catalog.update requires an active module registration owned by this connection",
2659 )?]);
2660 }
2661 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
2662 log_duplicate_claim_events(
2663 self.capability_evaluator
2664 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
2665 );
2666 }
2667 if capability_census_trigger(
2668 registration.manifest.capabilities.as_ref(),
2669 updated
2670 .as_ref()
2671 .and_then(|entry| entry.manifest.capabilities.as_ref()),
2672 ) {
2673 self.enforce_capability_denies();
2674 }
2675 self.refresh_capability_requirements();
2676
2677 let response = ModuleControlResponseToModule::CatalogUpdate {};
2678 control_response_body_frame(
2679 &frame,
2680 &response,
2681 "ModuleControlResponseToModule::CatalogUpdate",
2682 )
2683 .map(|frame| vec![frame])
2684 }
2685
2686 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
2687 self.refresh_capability_requirements();
2688 let mut counters = self.counters.snapshot();
2694 if let (Ok((connections_with_routes, max)), Some(obj)) = (
2695 self.forwarding.client_route_concentration(),
2696 counters.as_object_mut(),
2697 ) {
2698 obj.insert(
2699 "client_connections_with_routes".into(),
2700 connections_with_routes.into(),
2701 );
2702 obj.insert("max_routes_on_one_connection".into(), max.into());
2703 }
2704 if let (Some(open_breakers), Some(obj)) = (
2710 self.route_bind_breakers.open_snapshot(),
2711 counters.as_object_mut(),
2712 ) {
2713 obj.insert("route_bind_breakers_open".into(), open_breakers);
2714 }
2715 let response = ClientControlResponse::ServerDescribe {
2716 protocol_ver: PROTOCOL_VERSION,
2717 subc_ops: subc_ops(),
2718 capabilities: self.subc_capabilities.as_ref().to_vec(),
2719 connected_clients: self.connected_clients.count(),
2720 counters: Some(counters),
2721 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
2722 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
2723 capability_requirements: self.capability_requirement_statuses(),
2724 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2725 };
2726 Ok(vec![control_response_body_frame(
2727 &frame,
2728 &response,
2729 "ClientControlResponse::ServerDescribe",
2730 )?])
2731 }
2732
2733 fn handle_catalog_list(
2734 &self,
2735 frame: Frame,
2736 module_id: Option<String>,
2737 ) -> Result<Vec<Frame>, RouterError> {
2738 let (generation, modules) = self.registry.list_modules().map_err(|err| {
2739 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
2740 })?;
2741 let entries = modules
2742 .into_iter()
2743 .filter(|registration| {
2744 module_id
2745 .as_deref()
2746 .map(|wanted| registration.manifest.module_id == wanted)
2747 .unwrap_or(true)
2748 })
2749 .map(|registration| {
2750 let not_ready = self.not_ready_reason(®istration);
2751 let roles = registration.manifest.provides;
2752 CatalogEntry {
2753 module_id: registration.manifest.module_id,
2754 ready: not_ready.is_none(),
2755 not_ready,
2756 module_version: Some(registration.manifest.module_version),
2757 roles,
2758 control_ops: registration.control_ops,
2759 capabilities: registration.manifest.capabilities,
2760 self_signals: registration.manifest.self_signals,
2761 }
2762 })
2763 .collect();
2764 let response = ClientControlResponse::CatalogList {
2765 generation,
2766 modules: entries,
2767 subc_ops: subc_ops(),
2768 };
2769 Ok(vec![control_response_body_frame(
2770 &frame,
2771 &response,
2772 "ClientControlResponse::CatalogList",
2773 )?])
2774 }
2775
2776 fn route_open_principal(
2777 &self,
2778 frame: &Frame,
2779 consumer_identity: Option<ConsumerIdentity>,
2780 ) -> Result<Result<Principal, Frame>, RouterError> {
2781 let Some(consumer_identity) = consumer_identity else {
2782 return Ok(Ok(Principal::Direct));
2783 };
2784
2785 if self.supervisor.spawned_consumer_authorized(
2786 &consumer_identity.module_id,
2787 &consumer_identity.launch_nonce,
2788 ) {
2789 return Ok(Ok(Principal::Reserved {
2790 module_id: consumer_identity.module_id,
2791 }));
2792 }
2793
2794 Ok(Err(control_error_frame(
2795 frame,
2796 "bad_consumer_identity",
2797 format!(
2798 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
2799 consumer_identity.module_id
2800 ),
2801 )?))
2802 }
2803
2804 fn route_open_refusal_frame(
2811 &self,
2812 ctx: &RouteCtx,
2813 frame: &Frame,
2814 module_id: &str,
2815 reason: &'static str,
2816 code: &'static str,
2817 message: impl Into<String>,
2818 ) -> Result<Frame, RouterError> {
2819 self.observe_route_open_refusal(ctx, module_id, reason, code);
2820 control_error_frame(frame, code, message.into())
2821 }
2822
2823 fn route_open_breaker_refusal_frame(
2841 &self,
2842 ctx: &RouteCtx,
2843 frame: &Frame,
2844 module_id: &str,
2845 consecutive_timeouts: u32,
2846 retry_in: Duration,
2847 probe_in_flight: bool,
2848 ) -> Result<Frame, RouterError> {
2849 self.counters
2850 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
2851 debug!(
2852 target: "control",
2853 code = "module_timeout",
2854 module_id = ?module_id,
2855 connection_id = ctx.connection_id.get(),
2856 consecutive_timeouts,
2857 retry_in_ms = retry_in.as_millis() as u64,
2858 probe_in_flight,
2859 "route.open refused by open bind-relay breaker"
2860 );
2861 let detail = if probe_in_flight {
2868 "one test route is already being tried; retry once it settles".to_string()
2869 } else {
2870 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
2871 };
2872 control_error_frame(
2873 frame,
2874 "module_timeout",
2875 format!(
2876 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
2877 timed out in a row); {detail}; its established routes are unaffected"
2878 ),
2879 )
2880 }
2881
2882 fn observe_route_open_refusal(
2895 &self,
2896 ctx: &RouteCtx,
2897 module_id: &str,
2898 reason: &'static str,
2899 code: &'static str,
2900 ) {
2901 self.counters.increment_route_open_refused(code);
2902 info!(
2903 target: "control",
2904 code,
2905 reason,
2906 module_id = ?module_id,
2907 connection_id = ctx.connection_id.get(),
2908 "route.open refused"
2909 );
2910 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
2911 self.route_outages.record_not_serving(module_id, reason);
2912 }
2913 }
2914
2915 fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
2960 self.route_outages.record_accepted(module_id);
2961 self.counters.increment_route_open_accepted(principal);
2962 info!(
2963 target: "control",
2964 principal,
2965 module_id,
2966 connection_id = ctx.connection_id.get(),
2967 "route.open accepted"
2968 );
2969 }
2970
2971 fn supervised_absent_route_open_refusal_frame(
2972 &self,
2973 ctx: &RouteCtx,
2974 frame: &Frame,
2975 module_id: &str,
2976 code: &'static str,
2977 status: &crate::supervise::ModuleStatus,
2978 ) -> Result<Frame, RouterError> {
2979 self.counters.increment_route_open_refused(code);
2980 info!(
2981 target: "control",
2982 code,
2983 reason = "supervised_not_registered",
2984 module_id = ?module_id,
2985 connection_id = ctx.connection_id.get(),
2986 state = %status.state,
2987 enabled = status.enabled,
2988 live = status.live,
2989 "route.open refused"
2990 );
2991 self.route_outages
2995 .record_not_serving(module_id, "supervised_not_registered");
2996 control_error_frame(
2997 frame,
2998 code,
2999 format!(
3000 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
3001 status.state, status.enabled, status.live
3002 ),
3003 )
3004 }
3005
3006 async fn handle_route_open(
3007 &self,
3008 ctx: &RouteCtx,
3009 frame: Frame,
3010 request: RouteOpenRequest,
3011 ) -> Result<Vec<Frame>, RouterError> {
3012 let RouteOpenRequest {
3013 target,
3014 mut identity,
3015 consumer_identity,
3016 consumer_capabilities,
3017 role_versions,
3018 admission_facts,
3019 scope,
3020 } = request;
3021 let target_module_id = target_module_id(&target).to_string();
3022 if self
3023 .registry
3024 .get_module_by_connection(ctx.connection_id)
3025 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3026 .is_some()
3027 {
3028 return Ok(vec![control_error_frame(
3029 &frame,
3030 "invalid_request",
3031 "module connections cannot open client routes",
3032 )?]);
3033 }
3034 debug!(
3035 connection_id = ctx.connection_id.get(),
3036 corr = frame.header.corr,
3037 module_id = %target_module_id,
3038 "handling route.open"
3039 );
3040
3041 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
3047 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
3048 self.observe_route_open_refusal(
3049 ctx,
3050 &target_module_id,
3051 "invalid_role_versions",
3052 error_codes::INVALID_REQUEST,
3053 );
3054 return Ok(vec![control_error_body_frame(
3055 &frame,
3056 ErrorBody {
3057 code: error_codes::INVALID_REQUEST.to_string(),
3058 message: error.to_string(),
3059 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
3060 },
3061 )?]);
3062 }
3063
3064 let Some(registration) = self
3083 .registry
3084 .get_module(&target_module_id)
3085 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3086 else {
3087 if let Some((status, warming)) =
3088 self.supervisor_status(&target_module_id, frame.header.corr)?
3089 {
3090 if status.protocol == ModuleProtocol::None {
3097 return Ok(vec![self.route_open_refusal_frame(
3098 ctx,
3099 &frame,
3100 &target_module_id,
3101 "protocol_none",
3102 error_codes::MODULE_NO_PROTOCOL,
3103 format!(
3104 "module_id '{target_module_id}' is declared protocol: none; \
3105 it speaks no subc wire and serves no routes"
3106 ),
3107 )?]);
3108 }
3109 let code = if warming {
3110 "module_warming"
3111 } else {
3112 "target_unavailable"
3113 };
3114 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
3115 ctx,
3116 &frame,
3117 &target_module_id,
3118 code,
3119 &status,
3120 )?]);
3121 }
3122 if let Some(removed_ago_ms) =
3123 self.supervisor.removal_tombstone_age_ms(&target_module_id)
3124 {
3125 return Ok(vec![self.route_open_refusal_frame(
3126 ctx,
3127 &frame,
3128 &target_module_id,
3129 "removed",
3130 error_codes::MODULE_REMOVED,
3131 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
3132 )?]);
3133 }
3134 return Ok(vec![self.route_open_refusal_frame(
3135 ctx,
3136 &frame,
3137 &target_module_id,
3138 "not_registered",
3139 error_codes::UNKNOWN_MODULE,
3140 format!("module_id '{target_module_id}' is not registered"),
3141 )?]);
3142 };
3143
3144 if !registration.ready {
3148 self.counters
3149 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
3150 info!(
3151 target: "control",
3152 code = error_codes::MODULE_WARMING,
3153 module_id = ?target_module_id,
3154 connection_id = ctx.connection_id.get(),
3155 reason = "declared_not_ready",
3156 "route.open refused"
3157 );
3158 self.route_outages
3161 .record_not_serving(&target_module_id, "declared_not_ready");
3162 return Ok(vec![control_error_body_frame(
3163 &frame,
3164 ErrorBody {
3165 code: error_codes::MODULE_WARMING.to_string(),
3166 message: format!(
3167 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
3168 ),
3169 detail: Some(serde_json::json!({
3170 "reason": "declared_not_ready"
3171 })),
3172 },
3173 )?]);
3174 }
3175
3176 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
3195 self.counters
3196 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
3197 info!(
3198 target: "control",
3199 code = error_codes::MODULE_WARMING,
3200 module_id = ?target_module_id,
3201 connection_id = ctx.connection_id.get(),
3202 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3203 capability = %capability,
3204 "route.open refused"
3205 );
3206 return Ok(vec![control_error_body_frame(
3207 &frame,
3208 ErrorBody {
3209 code: error_codes::MODULE_WARMING.to_string(),
3210 message: format!(
3211 "module_id '{target_module_id}' requires capability '{capability}', \
3212 which no registered module provides; retry"
3213 ),
3214 detail: Some(serde_json::json!({
3215 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3216 "capability": capability,
3217 })),
3218 },
3219 )?]);
3220 }
3221
3222 if !target_has_required_role(&target, ®istration.manifest.provides) {
3223 return Ok(vec![self.route_open_refusal_frame(
3224 ctx,
3225 &frame,
3226 &target_module_id,
3227 "role_not_provided",
3228 "target_unavailable",
3229 format!("module_id '{target_module_id}' does not provide the requested target"),
3230 )?]);
3231 }
3232
3233 if registration.state != ChannelState::Active {
3234 return Ok(vec![self.route_open_refusal_frame(
3235 ctx,
3236 &frame,
3237 &target_module_id,
3238 "registration_not_active",
3239 "target_unavailable",
3240 format!("module_id '{target_module_id}' is not active"),
3241 )?]);
3242 }
3243
3244 if self
3245 .forwarding
3246 .module_is_draining(&target_module_id)
3247 .map_err(RouterError::Forwarding)?
3248 {
3249 return Ok(vec![self.route_open_refusal_frame(
3250 ctx,
3251 &frame,
3252 &target_module_id,
3253 "reloading",
3254 "module_reloading",
3255 format!("module_id '{target_module_id}' is reloading"),
3256 )?]);
3257 }
3258
3259 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
3260 process_liveness.process_live(&target_module_id) == Some(false)
3261 }) {
3262 if process_liveness.process_replacing(&target_module_id) {
3269 return Ok(vec![self.route_open_refusal_frame(
3270 ctx,
3271 &frame,
3272 &target_module_id,
3273 "reloading",
3274 "module_reloading",
3275 format!("module_id '{target_module_id}' is reloading"),
3276 )?]);
3277 }
3278 return Ok(vec![self.route_open_refusal_frame(
3279 ctx,
3280 &frame,
3281 &target_module_id,
3282 "supervisor_not_live",
3283 "target_unavailable",
3284 format!("module_id '{target_module_id}' is not live"),
3285 )?]);
3286 }
3287
3288 if !self
3289 .forwarding
3290 .has_live_module_connection(&target_module_id)
3291 .map_err(RouterError::Forwarding)?
3292 {
3293 return Ok(vec![self.route_open_refusal_frame(
3294 ctx,
3295 &frame,
3296 &target_module_id,
3297 "no_forwarding_connection",
3298 "target_unavailable",
3299 format!("module_id '{target_module_id}' has no live forwarding connection"),
3300 )?]);
3301 }
3302
3303 if let Some(error) =
3304 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
3305 {
3306 self.observe_route_open_refusal(
3307 ctx,
3308 &target_module_id,
3309 "op_not_allowed",
3310 "op_not_allowed",
3311 );
3312 return Ok(vec![error]);
3313 }
3314
3315 let principal = match self.route_open_principal(&frame, consumer_identity)? {
3316 Ok(principal) => principal,
3317 Err(error) => {
3318 self.observe_route_open_refusal(
3319 ctx,
3320 &target_module_id,
3321 "bad_consumer_identity",
3322 "bad_consumer_identity",
3323 );
3324 return Ok(vec![error]);
3325 }
3326 };
3327
3328 if let Principal::Reserved {
3332 module_id: opening_module_id,
3333 } = &principal
3334 {
3335 if let Some(opening_registration) = self
3336 .registry
3337 .get_module(opening_module_id)
3338 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3339 {
3340 if let Some(capability) =
3341 denied_capability(&opening_registration.manifest, ®istration.manifest)
3342 {
3343 warn!(
3344 opening_module_id,
3345 target_module_id,
3346 capability,
3347 "refusing route.open because an attested capability deny edge matches"
3348 );
3349 return Ok(vec![self.route_open_refusal_frame(
3350 ctx,
3351 &frame,
3352 &target_module_id,
3353 "capability_deny_edge",
3354 "capability_forbidden",
3355 format!(
3356 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
3357 ),
3358 )?]);
3359 }
3360 }
3361 }
3362
3363 if admission_facts.is_some() {
3364 let carrier_matches = matches!(
3365 &principal,
3366 Principal::Reserved { module_id }
3367 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
3368 );
3369 if !carrier_matches {
3370 return Ok(vec![self.route_open_refusal_frame(
3371 ctx,
3372 &frame,
3373 &target_module_id,
3374 "admission_facts_carrier_not_permitted",
3375 "admission_facts_not_permitted",
3376 "admission facts may only be carried by the configured reserved module",
3377 )?]);
3378 }
3379
3380 let target_allowed = self
3381 .admission_facts_targets
3382 .as_ref()
3383 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
3384 if !target_allowed {
3385 return Ok(vec![self.route_open_refusal_frame(
3386 ctx,
3387 &frame,
3388 &target_module_id,
3389 "admission_facts_target_not_listed",
3390 "admission_facts_target_not_allowed",
3391 format!(
3392 "admission facts are not permitted for target module_id '{target_module_id}'"
3393 ),
3394 )?]);
3395 }
3396
3397 }
3401
3402 let (bound_scope, scope_stamp) = match scope {
3408 None => (None, None),
3409 Some(selector) => {
3410 let owner_configured = match &selector.owner {
3411 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3417 _ => false,
3418 };
3419 let admitted = self
3420 .scopes
3421 .read()
3422 .unwrap_or_else(|poisoned| poisoned.into_inner())
3423 .admit(&principal, &target_module_id, &selector, owner_configured)
3424 .and_then(|admission| {
3425 crate::scopes::check_target_flow_support(
3426 &admission.stamp,
3427 &target_module_id,
3428 registration.manifest.capabilities.as_ref(),
3429 )?;
3430 Ok(admission)
3431 });
3432 match admitted {
3433 Ok(admission) => (
3434 Some(BoundScope {
3435 owner: admission.owner,
3436 scope_ref: admission.stamp.scope_ref.clone(),
3437 tag: admission.tag,
3438 }),
3439 Some(admission.stamp),
3440 ),
3441 Err(refusal) => {
3442 return Ok(vec![self.route_open_refusal_frame(
3443 ctx,
3444 &frame,
3445 &target_module_id,
3446 refusal.code,
3447 refusal.code,
3448 refusal.message,
3449 )?]);
3450 }
3451 }
3452 }
3453 };
3454
3455 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
3477 Ok(project_root) => project_root,
3478 Err(err) => {
3479 return Ok(vec![control_error_frame(
3480 &frame,
3481 "invalid_project_root",
3482 err.to_string(),
3483 )?])
3484 }
3485 };
3486 identity.project_root = project_root.as_path().to_path_buf();
3487
3488 let _concurrency_guard = match self
3499 .route_bind_concurrency
3500 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
3501 {
3502 Ok(guard) => guard,
3503 Err(in_flight) => {
3504 return Ok(vec![self.route_open_target_capacity_refusal(
3505 ctx,
3506 &frame,
3507 &target_module_id,
3508 in_flight,
3509 )?]);
3510 }
3511 };
3512
3513 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
3516 RouteBindAdmission::Admitted { guard, probe } => {
3517 if probe {
3518 info!(
3519 module_id = %target_module_id,
3520 connection_id = ctx.connection_id.get(),
3521 "route.bind breaker half-open: admitting one probe"
3522 );
3523 }
3524 guard
3525 }
3526 RouteBindAdmission::Refused {
3527 consecutive_timeouts,
3528 retry_in,
3529 probe_in_flight,
3530 } => {
3531 return Ok(vec![self.route_open_breaker_refusal_frame(
3532 ctx,
3533 &frame,
3534 &target_module_id,
3535 consecutive_timeouts,
3536 retry_in,
3537 probe_in_flight,
3538 )?]);
3539 }
3540 };
3541
3542 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
3547 let relay_deadline = Instant::now() + route_bind_relay_timeout;
3548 let pending = match self
3549 .forwarding
3550 .begin_route_bind_relay_for(
3551 ctx.connection_id,
3552 ctx.egress.clone(),
3553 response_version(&frame),
3554 frame.header.corr,
3555 &target_module_id,
3556 principal.clone(),
3557 bound_scope,
3558 Some(project_root),
3559 relay_deadline,
3560 )
3561 .await
3562 {
3563 Ok(pending) => pending,
3564 Err(err) => {
3565 return Ok(vec![self.route_open_refusal_frame(
3566 ctx,
3567 &frame,
3568 &target_module_id,
3569 "relay_reservation_failed",
3570 forwarding_error_code(&err),
3571 err.to_string(),
3572 )?])
3573 }
3574 };
3575 let crate::forwarding::PendingRouteBindRelay {
3576 endpoint,
3577 module_sink,
3578 negotiated_ver,
3579 client_channel,
3580 client_epoch,
3581 module_channel,
3582 module_epoch,
3583 corr: relay_corr,
3584 receiver,
3585 } = pending;
3586 let mut reservation =
3587 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
3588
3589 if let Some(stamp) = scope_stamp
3594 .as_ref()
3595 .filter(|stamp| stamp.attributes.flow_id.is_some())
3596 {
3597 let relay_registration = self
3598 .registry
3599 .get_module_by_connection(endpoint.connection_id)
3600 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3601 if let Err(refusal) = crate::scopes::check_target_flow_support(
3602 stamp,
3603 &target_module_id,
3604 relay_registration
3605 .as_ref()
3606 .and_then(|registration| registration.manifest.capabilities.as_ref()),
3607 ) {
3608 reservation.release_and_disarm();
3609 return Ok(vec![self.route_open_refusal_frame(
3610 ctx,
3611 &frame,
3612 &target_module_id,
3613 refusal.code,
3614 refusal.code,
3615 refusal.message,
3616 )?]);
3617 }
3618 }
3619
3620 debug!(
3621 connection_id = ctx.connection_id.get(),
3622 client_channel,
3623 client_epoch,
3624 module_channel,
3625 module_epoch,
3626 "reserved route handle pair"
3627 );
3628 let principal_label = match &principal {
3631 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
3632 Principal::Direct => "direct".to_string(),
3633 other => format!("{other:?}"),
3634 };
3635 let relay = ModuleControlRequest::RouteBind {
3636 route_channel: module_channel,
3637 epoch: module_epoch,
3638 target,
3639 identity,
3640 principal: Some(principal),
3641 consumer_capabilities,
3642 role_versions,
3643 admission_facts,
3644 scope: scope_stamp,
3645 };
3646 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
3647 RouterError::backend(
3648 0,
3649 frame.header.corr,
3650 format!("failed to encode route.bind request: {err}"),
3651 )
3652 })?;
3653 let relay_frame = Frame::build_with_version(
3654 negotiated_ver,
3655 FrameType::Request,
3656 control_flags(),
3657 0,
3658 0,
3659 relay_corr,
3660 relay_body,
3661 )
3662 .map_err(RouterError::FrameBuild)?;
3663
3664 if let Err(err) = module_sink.send(relay_frame).await {
3665 reservation.release_and_disarm();
3666 return Ok(vec![self.route_open_refusal_frame(
3667 ctx,
3668 &frame,
3669 &target_module_id,
3670 "relay_send_failed",
3671 "target_unavailable",
3672 err.to_string(),
3673 )?]);
3674 }
3675
3676 #[cfg(test)]
3677 if let Some(hold) = &self.route_bind_relay_sent_hold {
3678 hold.wait().await;
3682 hold.wait().await;
3683 }
3684
3685 if self
3690 .forwarding
3691 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
3692 .map_err(RouterError::Forwarding)?
3693 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
3694 {
3695 self.send_abandoned_route_bind_goodbye(
3696 &module_sink,
3697 negotiated_ver,
3698 module_channel,
3699 module_epoch,
3700 );
3701 }
3702
3703 match timeout_at(relay_deadline, receiver).await {
3704 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
3705 reservation.disarm();
3706 if breaker.record_accepted() {
3707 info!(
3708 module_id = %target_module_id,
3709 "route.bind breaker closed: the probe was accepted"
3710 );
3711 }
3712 self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
3713 Ok(Vec::new())
3714 }
3715 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
3716 reservation.release_and_disarm();
3717 breaker.record_inconclusive();
3721 let scope_code = match body.code.as_str() {
3725 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
3726 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
3727 _ => None,
3728 };
3729 if let Some(code) = scope_code {
3730 self.observe_route_open_refusal(
3731 ctx,
3732 &target_module_id,
3733 "scope_changed_before_commit",
3734 code,
3735 );
3736 return Ok(vec![control_error_body_frame(&frame, body)?]);
3737 }
3738 self.counters
3739 .increment_route_open_refused("module_rejected");
3740 info!(
3741 target: "control",
3742 code = "module_rejected",
3743 module_code = ?body.code,
3744 module_id = ?target_module_id,
3745 connection_id = ctx.connection_id.get(),
3746 "route.open refused"
3747 );
3748 Ok(vec![control_error_body_frame(&frame, body)?])
3749 }
3750 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
3751 reservation.release_and_disarm();
3752 breaker.record_inconclusive();
3753 tracing::warn!(
3760 module_id = %target_module_id,
3761 "route.bind relay abandoned: {message}"
3762 );
3763 Ok(vec![self.route_open_refusal_frame(
3764 ctx,
3765 &frame,
3766 &target_module_id,
3767 "relay_abandoned",
3768 "target_unavailable",
3769 message,
3770 )?])
3771 }
3772 Ok(Err(_)) => {
3773 reservation.release_and_disarm();
3774 breaker.record_inconclusive();
3775 Ok(vec![self.route_open_refusal_frame(
3776 ctx,
3777 &frame,
3778 &target_module_id,
3779 "relay_waiter_canceled",
3780 "target_unavailable",
3781 "route.bind relay waiter was canceled before the module responded",
3782 )?])
3783 }
3784 Err(_) => {
3785 reservation.release_and_disarm();
3786 if let Some(opened) = breaker.record_timeout(
3790 self.route_bind_breaker_threshold,
3791 self.route_bind_breaker_cooldown,
3792 ) {
3793 warn!(
3794 module_id = %target_module_id,
3795 consecutive_timeouts = opened.consecutive_timeouts,
3796 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
3797 reopened_after_probe = opened.reopened_after_probe,
3798 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
3799 );
3800 }
3801 tracing::warn!(
3809 module_id = %target_module_id,
3810 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
3811 "route.bind relay timed out: module did not ack within budget"
3812 );
3813 Ok(vec![self.route_open_refusal_frame(
3814 ctx,
3815 &frame,
3816 &target_module_id,
3817 "relay_timed_out",
3818 "module_timeout",
3819 format!(
3820 "module_id '{target_module_id}' did not answer route.bind within {:?}",
3821 route_bind_relay_timeout
3822 ),
3823 )?])
3824 }
3825 }
3826 }
3827
3828 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3829 let response = ClientControlResponse::SupervisorSpawnSnapshot {
3830 snapshot: self.supervisor.spawn_snapshot(),
3831 };
3832 Ok(vec![control_response_body_frame(
3833 &frame,
3834 &response,
3835 "ClientControlResponse::SupervisorSpawnSnapshot",
3836 )?])
3837 }
3838
3839 fn handle_supervisor_spawn_subscribe(
3840 &self,
3841 ctx: &RouteCtx,
3842 frame: Frame,
3843 since: Option<SpawnCursor>,
3844 ) -> Result<Vec<Frame>, RouterError> {
3845 match self.supervisor.subscribe_spawns(
3846 ctx.connection_id,
3847 frame.header.corr,
3848 response_version(&frame),
3849 since,
3850 ctx.egress.clone(),
3851 ) {
3852 Ok(()) => Ok(Vec::new()),
3853 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
3854 Ok(vec![control_error_body_frame(
3855 &frame,
3856 ErrorBody {
3857 code: "spawn_cursor_incarnation_mismatch".to_string(),
3858 message: "spawn cursor belongs to a different daemon incarnation"
3859 .to_string(),
3860 detail: Some(serde_json::json!({
3861 "current_daemon_incarnation": current
3862 })),
3863 },
3864 )?])
3865 }
3866 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
3867 &frame,
3868 ErrorBody {
3869 code: "spawn_cursor_too_old".to_string(),
3870 message: "spawn cursor predates the retained event ring".to_string(),
3871 detail: Some(serde_json::json!({
3872 "oldest_retained_cursor": oldest
3873 })),
3874 },
3875 )?]),
3876 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
3877 0,
3878 frame.header.corr,
3879 format!("failed to open supervisor spawn subscription: {error}"),
3880 )),
3881 }
3882 }
3883
3884 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3885 let generation = self
3886 .registry
3887 .generation()
3888 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3889 let mut modules = Vec::new();
3890 for module in self.supervisor.list() {
3891 let status = module.status_for_control("list").map_err(|err| {
3892 RouterError::backend(
3893 0,
3894 frame.header.corr,
3895 format!("failed to read supervisor status: {err}"),
3896 )
3897 })?;
3898 let (configured, _) = module.configuration().map_err(|err| {
3899 RouterError::backend(
3900 0,
3901 frame.header.corr,
3902 format!("failed to read module configuration: {err}"),
3903 )
3904 })?;
3905 let image = module.running_image_agreement().await;
3907 let resources = Some(module.child_resource_usage());
3910 let pending_reload = Some(reload_verdict(
3911 &configured.program,
3912 status.spawned_from.as_deref(),
3913 image,
3914 ));
3915 modules.push(SupervisorEntry {
3916 launch_nonce_env: Some(!cfg!(unix)),
3919 module_id: status.module_id,
3920 state: status.state.to_string(),
3921 enabled: status.enabled,
3922 live: status.live,
3923 protocol: status.protocol,
3924 health: status.health.status,
3925 pending_reload,
3926 last_probe_ms: status.health.last_probe_ms,
3927 last_exit_code: status.last_exit.as_ref().and_then(|e| e.code),
3928 last_exit_signal: status.last_exit.as_ref().and_then(|e| e.signal),
3929 last_exit_ms: status.last_exit.as_ref().map(|e| e.at_ms),
3930 last_exit_kind: status.last_exit.as_ref().map(|e| e.kind.into()),
3931 restart_count: Some(status.restart_count),
3932 max_restarts: Some(status.max_restarts),
3933 lifetime_restarts: Some(status.lifetime_restarts),
3934 spawn_generation: Some(status.spawn_generation),
3935 restart_window_secs: Some(status.restart_window.as_secs()),
3936 drain_timeout_ms: Some(status.drain_timeout.as_millis() as u64),
3937 restart_backoff_ms: Some(status.restart_backoff.as_millis() as u64),
3938 restart_max_backoff_ms: Some(status.restart_max_backoff.as_millis() as u64),
3939 resources,
3940 });
3941 }
3942 let response = ClientControlResponse::SupervisorList {
3943 generation,
3944 modules,
3945 };
3946 Ok(vec![control_response_body_frame(
3947 &frame,
3948 &response,
3949 "ClientControlResponse::SupervisorList",
3950 )?])
3951 }
3952
3953 fn handle_supervisor_stderr_tail(
3954 &self,
3955 frame: Frame,
3956 module_id: String,
3957 max_lines: Option<u32>,
3958 max_bytes: Option<u32>,
3959 ) -> Result<Vec<Frame>, RouterError> {
3960 let Some(module) = self.supervisor.get(&module_id) else {
3961 return Ok(vec![control_error_frame(
3962 &frame,
3963 "unknown_module",
3964 format!("module_id '{module_id}' is not supervised"),
3965 )?]);
3966 };
3967
3968 let snapshot = module.stderr_tail(
3969 max_lines.map(|value| value as usize),
3970 max_bytes.map(|value| value as usize),
3971 );
3972
3973 let response = ClientControlResponse::SupervisorStderrTail {
3974 module_id,
3975 tail: StderrTail {
3976 capture: match snapshot.capture {
3977 CaptureState::Captured => StderrCaptureState::Captured,
3978 CaptureState::Incomplete { reason } => {
3979 StderrCaptureState::Incomplete { reason }
3980 }
3981 CaptureState::NotCaptured { reason } => {
3982 StderrCaptureState::NotCaptured { reason }
3983 }
3984 },
3985 entries: snapshot
3986 .entries
3987 .into_iter()
3988 .map(|entry| match entry {
3989 TailEntry::Line {
3990 text,
3991 truncated,
3992 at_ms,
3993 } => StderrTailEntry::Line {
3994 text,
3995 truncated,
3996 at_ms,
3997 },
3998 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
3999 })
4000 .collect(),
4001 dropped_lines: snapshot.dropped_lines,
4002 },
4003 };
4004 Ok(vec![control_response_body_frame(
4005 &frame,
4006 &response,
4007 "ClientControlResponse::SupervisorStderrTail",
4008 )?])
4009 }
4010
4011 async fn handle_supervisor_terminals(
4012 &self,
4013 frame: Frame,
4014 module_id: String,
4015 ) -> Result<Vec<Frame>, RouterError> {
4016 let Some(module) = self.supervisor.get(&module_id) else {
4017 return Ok(vec![control_error_frame(
4018 &frame,
4019 "unknown_module",
4020 format!("module_id '{module_id}' is not supervised"),
4021 )?]);
4022 };
4023
4024 let terminals = module
4027 .read_durable_terminal_history()
4028 .await
4029 .map_err(|error| {
4030 RouterError::backend(
4031 0,
4032 frame.header.corr,
4033 format!("failed to read terminal history: {error}"),
4034 )
4035 })?;
4036 let response = ClientControlResponse::SupervisorTerminals {
4037 module_id,
4038 terminals,
4039 };
4040 Ok(vec![control_response_body_frame(
4041 &frame,
4042 &response,
4043 "ClientControlResponse::SupervisorTerminals",
4044 )?])
4045 }
4046
4047 fn handle_supervisor_routes(
4048 &self,
4049 frame: Frame,
4050 module_id: Option<String>,
4051 ) -> Result<Vec<Frame>, RouterError> {
4052 let modules = self
4053 .forwarding
4054 .route_census(module_id.as_deref())
4055 .map_err(RouterError::Forwarding)?
4056 .into_iter()
4057 .map(|(module_id, routes)| SupervisorRouteModule {
4058 module_id,
4059 routes: routes
4060 .into_iter()
4061 .map(|route| SupervisorRoute {
4062 consumer: match route.principal {
4063 Principal::Reserved { module_id } => {
4064 SupervisorRouteConsumer::Reserved { module_id }
4065 }
4066 Principal::Direct | Principal::Unverified => {
4067 SupervisorRouteConsumer::Direct {
4068 connection_id: route.goodbye_target.connection_id.get(),
4069 }
4070 }
4071 },
4072 age_ms: Instant::now()
4073 .saturating_duration_since(route.bound_at)
4074 .as_millis()
4075 .try_into()
4076 .unwrap_or(u64::MAX),
4077 draining: route.draining,
4078 drain_reason: route.drain_reason,
4079 })
4080 .collect(),
4081 })
4082 .collect();
4083 let response = ClientControlResponse::SupervisorRoutes { modules };
4084 Ok(vec![control_response_body_frame(
4085 &frame,
4086 &response,
4087 "ClientControlResponse::SupervisorRoutes",
4088 )?])
4089 }
4090
4091 async fn handle_supervisor_provenance(
4092 &self,
4093 frame: Frame,
4094 module_id: Option<String>,
4095 ) -> Result<Vec<Frame>, RouterError> {
4096 let mut selected = if let Some(module_id) = module_id {
4097 let Some(module) = self.supervisor.get(&module_id) else {
4098 return Ok(vec![control_error_frame(
4099 &frame,
4100 "unknown_module",
4101 format!("module_id '{module_id}' is not supervised"),
4102 )?]);
4103 };
4104 vec![module]
4105 } else {
4106 self.supervisor.list()
4107 };
4108
4109 let mut modules = Vec::with_capacity(selected.len());
4110 for module in selected.drain(..) {
4111 let status = module.status().map_err(|err| {
4112 RouterError::backend(
4113 0,
4114 frame.header.corr,
4115 format!("failed to read supervisor status: {err}"),
4116 )
4117 })?;
4118 let module_declared = self
4119 .registry
4120 .get_module(&status.module_id)
4121 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4122 .and_then(|registration| registration.manifest.provenance)
4123 .map(|build| ModuleDeclaredProvenance::Reported { build })
4124 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
4125 #[cfg(test)]
4126 let running_image = match &self.provenance_probe_override {
4127 Some(result) => result.clone(),
4128 None => module.running_image_agreement().await,
4129 };
4130 #[cfg(not(test))]
4131 let running_image = module.running_image_agreement().await;
4132 modules.push(SupervisorModuleProvenance {
4133 module_id: status.module_id,
4134 module_declared,
4135 daemon_observed: SupervisorObservedProcess {
4136 pid: status.pid,
4137 spawned_at_ms: status.spawned_at_ms,
4138 spawned_from: status.spawned_from,
4139 running_image,
4140 },
4141 });
4142 }
4143 let daemon = SupervisorDaemonProvenance {
4144 daemon_build: self.daemon_provenance.build.clone(),
4145 daemon_observed: DaemonObservedProcess {
4146 pid: self.daemon_provenance.pid,
4147 started_at_ms: self
4148 .daemon_provenance
4149 .start_clock
4150 .map(|clock| clock.started_at_ms())
4151 .or(self.daemon_provenance.started_at_ms),
4152 running_image: self
4153 .daemon_provenance
4154 .probe
4155 .observe(
4156 self.daemon_provenance.pid,
4157 self.daemon_provenance.executable_path.as_deref(),
4158 self.daemon_provenance.executable_identity,
4159 self.daemon_provenance.process_start_time,
4160 )
4161 .await,
4162 },
4163 };
4164 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
4165 Ok(vec![control_response_body_frame(
4166 &frame,
4167 &response,
4168 "ClientControlResponse::SupervisorProvenance",
4169 )?])
4170 }
4171
4172 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4173 self.refresh_capability_requirements();
4174 let generation = self
4175 .registry
4176 .generation()
4177 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4178 let modules = self
4179 .supervisor
4180 .list()
4181 .into_iter()
4182 .map(|module| {
4183 let status = module.status_for_control("health").map_err(|err| {
4184 RouterError::backend(
4185 0,
4186 frame.header.corr,
4187 format!("failed to read supervisor health: {err}"),
4188 )
4189 })?;
4190 let module_id = status.module_id;
4191 let capability_detail = self
4192 .capability_evaluator
4193 .required_problem_detail(&module_id);
4194 Ok(SupervisorHealthEntry {
4195 module_id,
4196 status: status.health.status,
4197 detail: append_capability_problem_detail(
4198 status.health.detail,
4199 capability_detail,
4200 ),
4201 metrics: status.health.metrics,
4202 consecutive_failures: status.health.consecutive_failures,
4203 late_answer_count: status.health.late_answer_count,
4204 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
4205 last_action: status.health.last_action,
4206 last_action_ms: status.health.last_action_ms,
4207 last_probe_ms: status.health.last_probe_ms,
4208 })
4209 })
4210 .collect::<Result<Vec<_>, RouterError>>()?;
4211 let response = ClientControlResponse::SupervisorHealth {
4212 generation,
4213 modules,
4214 };
4215 Ok(vec![control_response_body_frame(
4216 &frame,
4217 &response,
4218 "ClientControlResponse::SupervisorHealth",
4219 )?])
4220 }
4221
4222 async fn handle_supervisor_restart(
4223 &self,
4224 frame: Frame,
4225 module_id: String,
4226 drain_timeout_ms: Option<u64>,
4227 ) -> Result<Vec<Frame>, RouterError> {
4228 let operation_lock = self.supervisor.operation_lock();
4229 let _operation_guard = operation_lock.lock().await;
4230 let Some(module) = self.supervisor.get(&module_id) else {
4231 return Ok(vec![control_error_frame(
4232 &frame,
4233 "unknown_module",
4234 format!("module_id '{module_id}' is not supervised"),
4235 )?]);
4236 };
4237
4238 self.route_outages.mark_operator_action(&module_id);
4239 if let Err(err) = module.restart(drain_timeout_ms).await {
4240 self.route_outages
4241 .operator_action_ended_unrefused(&module_id);
4242 let (code, message) = match err {
4243 crate::supervise::SuperviseError::Disabled { .. } => {
4244 ("module_disabled", err.to_string())
4245 }
4246 crate::supervise::SuperviseError::SwapInProgress { .. } => {
4247 ("swap_in_progress", err.to_string())
4248 }
4249 _ => (
4250 "target_unavailable",
4251 format!("failed to restart module_id '{module_id}': {err}"),
4252 ),
4253 };
4254 return Ok(vec![control_error_frame(&frame, code, message)?]);
4255 }
4256
4257 let response = ClientControlResponse::SupervisorAck {
4258 module_id,
4259 applied: true,
4260 };
4261 Ok(vec![control_response_body_frame(
4262 &frame,
4263 &response,
4264 "ClientControlResponse::SupervisorAck",
4265 )?])
4266 }
4267
4268 async fn handle_supervisor_swap(
4272 &self,
4273 frame: Frame,
4274 module_id: String,
4275 ready_timeout_ms: Option<u64>,
4276 ) -> Result<Vec<Frame>, RouterError> {
4277 let module = {
4284 let operation_lock = self.supervisor.operation_lock();
4285 let _operation_guard = operation_lock.lock().await;
4286 self.supervisor.get(&module_id)
4287 };
4288 let Some(module) = module else {
4289 return Ok(vec![control_error_frame(
4290 &frame,
4291 "unknown_module",
4292 format!("module_id '{module_id}' is not supervised"),
4293 )?]);
4294 };
4295
4296 self.route_outages.mark_operator_action(&module_id);
4297 if let Err(err) = module
4298 .swap(ready_timeout_ms.map(Duration::from_millis))
4299 .await
4300 {
4301 self.route_outages
4302 .operator_action_ended_unrefused(&module_id);
4303 use crate::supervise::SuperviseError;
4304 let message = err.to_string();
4305 let error = match err {
4306 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
4307 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
4308 code: "swap_refused".to_string(),
4309 message,
4310 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
4311 },
4312 SuperviseError::SwapFailed {
4313 arm,
4314 candidate_exit,
4315 ..
4316 } => ErrorBody {
4317 code: "swap_failed".to_string(),
4318 message,
4319 detail: Some(serde_json::json!({
4320 "arm": arm.as_str(),
4321 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
4322 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
4323 })),
4324 },
4325 _ => ErrorBody::new(
4326 "target_unavailable",
4327 format!("failed to swap module_id '{module_id}': {message}"),
4328 ),
4329 };
4330 return Ok(vec![control_error_body_frame(&frame, error)?]);
4331 }
4332 self.route_outages
4336 .operator_action_ended_unrefused(&module_id);
4337
4338 let response = ClientControlResponse::SupervisorAck {
4339 module_id,
4340 applied: true,
4341 };
4342 Ok(vec![control_response_body_frame(
4343 &frame,
4344 &response,
4345 "ClientControlResponse::SupervisorAck",
4346 )?])
4347 }
4348
4349 async fn handle_supervisor_reload(
4350 &self,
4351 frame: Frame,
4352 module_id: String,
4353 ) -> Result<Vec<Frame>, RouterError> {
4354 let operation_lock = self.supervisor.operation_lock();
4355 let _operation_guard = operation_lock.lock().await;
4356 let Some(module) = self.supervisor.get(&module_id) else {
4357 return Ok(vec![control_error_frame(
4358 &frame,
4359 "unknown_module",
4360 format!("module_id '{module_id}' is not supervised"),
4361 )?]);
4362 };
4363
4364 self.route_outages.mark_operator_action(&module_id);
4365 if let Err(err) = module.reload().await {
4366 self.route_outages
4367 .operator_action_ended_unrefused(&module_id);
4368 let (code, message) = match err {
4369 crate::supervise::SuperviseError::Disabled { .. } => {
4370 ("module_disabled", err.to_string())
4371 }
4372 crate::supervise::SuperviseError::SwapInProgress { .. } => {
4373 ("swap_in_progress", err.to_string())
4374 }
4375 _ => (
4376 "reload_failed",
4377 format!("failed to reload module_id '{module_id}': {err}"),
4378 ),
4379 };
4380 return Ok(vec![control_error_frame(&frame, code, message)?]);
4381 }
4382
4383 let response = ClientControlResponse::SupervisorAck {
4384 module_id,
4385 applied: true,
4386 };
4387 Ok(vec![control_response_body_frame(
4388 &frame,
4389 &response,
4390 "ClientControlResponse::SupervisorAck",
4391 )?])
4392 }
4393
4394 async fn handle_supervisor_rescan(
4395 &self,
4396 frame: Frame,
4397 preview: bool,
4398 ) -> Result<Vec<Frame>, RouterError> {
4399 let Some(context) = self.rescan.clone() else {
4400 return Ok(vec![control_error_frame(
4401 &frame,
4402 "rescan_unavailable",
4403 "the daemon was not started with a reloadable config path".to_string(),
4404 )?]);
4405 };
4406
4407 let operation_lock = self.supervisor.operation_lock();
4408 let _operation_guard = operation_lock.lock().await;
4409 let loaded = match crate::daemon_config::load(&context.config_path) {
4410 Ok(config) => config,
4411 Err(err) => {
4412 return Ok(vec![control_error_frame(
4413 &frame,
4414 "invalid_daemon_config",
4415 format!("supervisor rescan rejected daemon config: {err}"),
4416 )?])
4417 }
4418 };
4419 let Some(config) = loaded else {
4427 return Ok(vec![control_error_frame(
4428 &frame,
4429 "invalid_daemon_config",
4430 format!(
4431 "daemon config not found at {}; refusing to rescan (an absent config would \
4432 retire every supervised module)",
4433 context.config_path.display()
4434 ),
4435 )?]);
4436 };
4437 let (
4438 configured_port,
4439 storage_config,
4440 admission_facts_carrier_module_id,
4441 admission_facts_targets,
4442 scope_authority_owners,
4443 modules,
4444 reserved_capabilities,
4445 ) = (
4446 config.port,
4447 config.storage,
4448 config.admission_facts_carrier_module_id,
4449 config.admission_facts_targets,
4450 config.scope_authority_owners,
4451 config.modules,
4452 config.reserved_capabilities,
4453 );
4454
4455 let mut restart_required = Vec::new();
4464 for section in RestartRequiredSection::ALL {
4465 let changed = match section {
4466 RestartRequiredSection::Port => configured_port != context.configured_port,
4467 RestartRequiredSection::Storage => storage_config != context.storage_config,
4468 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
4469 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
4470 }
4471 RestartRequiredSection::AdmissionFactsTargets => {
4472 admission_facts_targets != context.admission_facts_targets
4473 }
4474 RestartRequiredSection::ScopeAuthorityOwners => {
4475 scope_authority_owners != context.scope_authority_owners
4476 }
4477 };
4478 if changed {
4479 restart_required.push(section.label().to_string());
4480 }
4481 }
4482 if !restart_required.is_empty() {
4483 warn!(
4484 config_path = %context.config_path.display(),
4485 sections = %restart_required.join(", "),
4486 "daemon config changed outside the modules section; restart the daemon to apply those changes"
4487 );
4488 }
4489
4490 for configured in &modules {
4491 if let Err(err) = validate_spec(&configured.module_spec()) {
4492 return Ok(vec![control_error_frame(
4493 &frame,
4494 "invalid_daemon_config",
4495 format!("supervisor rescan rejected daemon config: {err}"),
4496 )?]);
4497 }
4498 }
4499
4500 let configured_capabilities = modules
4501 .iter()
4502 .map(|module| (module.module_id.clone(), module.enabled))
4503 .collect::<Vec<_>>();
4504 let preview_capability_warnings = if preview {
4505 let (_, registrations) = self.runtime_capability_snapshot()?;
4506 let current_modules = self
4507 .supervisor
4508 .list()
4509 .into_iter()
4510 .map(|module| module.module_id().to_string())
4511 .collect::<BTreeSet<_>>();
4512 let resulting_modules = configured_capabilities.clone();
4513 let removed = current_modules
4514 .into_iter()
4515 .filter(|module_id| {
4516 !resulting_modules
4517 .iter()
4518 .any(|(configured_id, _)| configured_id == module_id)
4519 })
4520 .collect::<Vec<_>>();
4521 self.capability_evaluator.preview_removal_warnings(
4522 resulting_modules,
4523 &removed,
4524 ®istrations,
4525 )
4526 } else {
4527 Vec::new()
4528 };
4529 let result = match self
4530 .reconcile_supervised_modules(&context.supervisor, modules, preview)
4531 .await
4532 {
4533 Ok(result) => result,
4534 Err(message) => {
4535 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
4536 }
4537 };
4538 if !preview {
4539 self.capability_evaluator
4540 .configure(configured_capabilities, reserved_capabilities);
4541 self.capability_evaluator.wake_deadline_loop();
4542 self.refresh_capability_requirements();
4543 }
4544 let mut result = result;
4545 result.restart_required = restart_required;
4546 result.capability_warnings = preview_capability_warnings;
4547 let response = ClientControlResponse::SupervisorRescan { result };
4548 Ok(vec![control_response_body_frame(
4549 &frame,
4550 &response,
4551 "ClientControlResponse::SupervisorRescan",
4552 )?])
4553 }
4554
4555 async fn handle_supervisor_release_reserved(
4556 &self,
4557 frame: Frame,
4558 module_id: String,
4559 ) -> Result<Vec<Frame>, RouterError> {
4560 let Some(context) = self.rescan.clone() else {
4561 return Ok(vec![control_error_frame(
4562 &frame,
4563 "release_unavailable",
4564 "reserved-id release requires a daemon started with a reloadable config path",
4565 )?]);
4566 };
4567 let operation_lock = self.supervisor.operation_lock();
4568 let _operation_guard = operation_lock.lock().await;
4569 let loaded = match crate::daemon_config::load(&context.config_path) {
4570 Ok(Some(config)) => config,
4571 Ok(None) => {
4572 return Ok(vec![control_error_frame(
4573 &frame,
4574 "invalid_daemon_config",
4575 format!(
4576 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
4577 context.config_path.display()
4578 ),
4579 )?])
4580 }
4581 Err(err) => {
4582 return Ok(vec![control_error_frame(
4583 &frame,
4584 "invalid_daemon_config",
4585 format!("unable to verify reserved-id release against daemon config: {err}"),
4586 )?])
4587 }
4588 };
4589 if loaded
4590 .modules
4591 .iter()
4592 .any(|configured| configured.module_id == module_id)
4593 {
4594 return Ok(vec![control_error_frame(
4595 &frame,
4596 "reserved_module_configured",
4597 format!(
4598 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
4599 ),
4600 )?]);
4601 }
4602 if !self.supervisor.release_retained_reserved_gate(&module_id) {
4603 return Ok(vec![control_error_frame(
4604 &frame,
4605 "reserved_gate_not_retained",
4606 format!(
4607 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
4608 ),
4609 )?]);
4610 }
4611
4612 let response = ClientControlResponse::SupervisorAck {
4613 module_id,
4614 applied: true,
4615 };
4616 Ok(vec![control_response_body_frame(
4617 &frame,
4618 &response,
4619 "ClientControlResponse::SupervisorAck",
4620 )?])
4621 }
4622
4623 async fn reconcile_supervised_modules(
4632 &self,
4633 supervisor: &Supervisor,
4634 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
4635 preview: bool,
4636 ) -> Result<SupervisorRescanResult, String> {
4637 let mut current = BTreeMap::new();
4638 for module in self.supervisor.list() {
4639 let (spec, health) = module.configuration().map_err(|err| {
4640 format!(
4641 "failed to read configuration for module_id '{}': {err}",
4642 module.module_id()
4643 )
4644 })?;
4645 let enabled = module
4646 .status()
4647 .map_err(|err| {
4648 format!(
4649 "failed to read status for module_id '{}': {err}",
4650 module.module_id()
4651 )
4652 })?
4653 .enabled;
4654 current.insert(
4655 module.module_id().to_string(),
4656 (module, spec, health, enabled),
4657 );
4658 }
4659 let configured = configured_modules
4660 .into_iter()
4661 .map(|module| (module.module_id.clone(), module))
4662 .collect::<BTreeMap<_, _>>();
4663
4664 let added = configured
4665 .keys()
4666 .filter(|module_id| !current.contains_key(*module_id))
4667 .cloned()
4668 .collect::<Vec<_>>();
4669 let removed = current
4670 .keys()
4671 .filter(|module_id| !configured.contains_key(*module_id))
4672 .cloned()
4673 .collect::<Vec<_>>();
4674 let mut changed_pending_reload = Vec::new();
4675 let mut configuration_changes = BTreeSet::new();
4676 let mut enabled_changes = BTreeSet::new();
4677 let mut unchanged = 0_u32;
4678
4679 for (module_id, configured_module) in &configured {
4680 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
4681 else {
4682 continue;
4683 };
4684 let launch_changed = *current_spec != configured_module.module_spec();
4688 let configuration_changed =
4689 launch_changed || *current_health != configured_module.health;
4690 let enabled_changed = *current_enabled != configured_module.enabled;
4691 if configuration_changed {
4692 configuration_changes.insert(module_id.clone());
4693 }
4694 if launch_changed {
4695 changed_pending_reload.push(module_id.clone());
4696 }
4697 if enabled_changed {
4698 enabled_changes.insert(module_id.clone());
4699 }
4700 if !configuration_changed && !enabled_changed {
4701 unchanged = unchanged.saturating_add(1);
4702 }
4703 }
4704
4705 if preview {
4711 return Ok(SupervisorRescanResult {
4712 added,
4713 removed,
4714 changed_pending_reload,
4715 enabled_changes: enabled_changes.iter().cloned().collect(),
4716 unchanged,
4717 preview: true,
4718 restart_required: Vec::new(),
4722 capability_warnings: Vec::new(),
4723 });
4724 }
4725
4726 for module_id in &removed {
4727 let module = ¤t
4728 .get(module_id)
4729 .expect("removed module came from current supervisor state")
4730 .0;
4731 module.retire().await.map_err(|err| {
4732 format!("failed to retire module_id '{module_id}' during rescan: {err}")
4733 })?;
4734 self.supervisor.record_rescan_removal(module_id);
4768 self.supervisor.retire(module_id);
4769 self.route_outages.forget(module_id);
4770 }
4771
4772 for module_id in configured.keys() {
4773 let Some((module, _, _, _)) = current.get(module_id) else {
4774 continue;
4775 };
4776 let configured_module = configured
4777 .get(module_id)
4778 .expect("configured module id came from configured map");
4779 if configuration_changes.contains(module_id) {
4780 module
4781 .update_configuration(
4782 configured_module.module_spec(),
4783 configured_module.health.clone(),
4784 configured_module.drain_timeout_ms,
4785 )
4786 .await
4787 .map_err(|err| {
4788 format!(
4789 "failed to update module_id '{module_id}' configuration during rescan: {err}"
4790 )
4791 })?;
4792 }
4793 if enabled_changes.contains(module_id) {
4794 self.route_outages.mark_operator_action(module_id);
4797 module
4798 .set_enabled(configured_module.enabled)
4799 .await
4800 .map_err(|err| {
4801 self.route_outages.operator_action_ended_unrefused(module_id);
4802 format!(
4803 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
4804 configured_module.enabled
4805 )
4806 })?;
4807 }
4808 }
4809
4810 for module_id in &added {
4811 let configured_module = configured
4812 .get(module_id)
4813 .expect("added module id came from configured map");
4814 supervisor
4815 .supervise_configured_with_health(
4816 configured_module.module_spec(),
4817 configured_module.enabled,
4818 configured_module.health.clone(),
4819 configured_module.drain_timeout_ms,
4820 configured_module.restart,
4821 )
4822 .map_err(|err| {
4823 format!("failed to add module_id '{module_id}' during rescan: {err}")
4824 })?;
4825 }
4826
4827 Ok(SupervisorRescanResult {
4828 added,
4829 removed,
4830 changed_pending_reload,
4831 enabled_changes: enabled_changes.iter().cloned().collect(),
4832 unchanged,
4833 preview: false,
4834 restart_required: Vec::new(),
4837 capability_warnings: Vec::new(),
4838 })
4839 }
4840
4841 async fn handle_supervisor_set_enabled(
4842 &self,
4843 frame: Frame,
4844 module_id: String,
4845 enabled: bool,
4846 ) -> Result<Vec<Frame>, RouterError> {
4847 let operation_lock = self.supervisor.operation_lock();
4848 let _operation_guard = operation_lock.lock().await;
4849 let Some(module) = self.supervisor.get(&module_id) else {
4850 return Ok(vec![control_error_frame(
4851 &frame,
4852 "unknown_module",
4853 format!("module_id '{module_id}' is not supervised"),
4854 )?]);
4855 };
4856
4857 self.route_outages.mark_operator_action(&module_id);
4860 let applied = match module.set_enabled(enabled).await {
4861 Ok(applied) => applied,
4862 Err(err) => {
4863 self.route_outages
4864 .operator_action_ended_unrefused(&module_id);
4865 return Ok(vec![control_error_frame(
4866 &frame,
4867 "target_unavailable",
4868 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
4869 )?]);
4870 }
4871 };
4872 if !applied {
4873 self.route_outages
4876 .operator_action_ended_unrefused(&module_id);
4877 }
4878
4879 self.capability_evaluator.wake_deadline_loop();
4880 self.refresh_capability_requirements();
4881 let response = ClientControlResponse::SupervisorAck { module_id, applied };
4882 Ok(vec![control_response_body_frame(
4883 &frame,
4884 &response,
4885 "ClientControlResponse::SupervisorAck",
4886 )?])
4887 }
4888
4889 async fn handle_supervisor_health_probe(
4890 &self,
4891 frame: Frame,
4892 module_id: String,
4893 ) -> Result<Vec<Frame>, RouterError> {
4894 self.refresh_capability_requirements();
4895 let Some(registration) = self
4896 .registry
4897 .get_module(&module_id)
4898 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4899 else {
4900 return Ok(vec![control_error_frame(
4901 &frame,
4902 "unknown_module",
4903 format!("module_id '{module_id}' is not registered"),
4904 )?]);
4905 };
4906
4907 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
4920 {
4921 return Ok(vec![control_error_frame(
4922 &frame,
4923 "health_not_advertised",
4924 format!("module_id '{module_id}' did not advertise health.check"),
4925 )?]);
4926 }
4927
4928 let deadline = Instant::now() + self.health_probe_timeout;
4929 let pending = match self.forwarding.begin_module_control_rpc_for(
4930 &module_id,
4931 MODULE_CONTROL_OP_HEALTH_CHECK,
4932 deadline,
4933 ) {
4934 Ok(pending) => pending,
4935 Err(err) => {
4936 return Ok(vec![control_error_frame(
4937 &frame,
4938 forwarding_error_code(&err),
4939 err.to_string(),
4940 )?])
4941 }
4942 };
4943
4944 let PendingModuleControlRpc {
4945 endpoint,
4946 module_sink,
4947 negotiated_ver,
4948 corr: probe_corr,
4949 receiver,
4950 } = pending;
4951 let mut guard =
4952 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
4953 let probe_body =
4954 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
4955 RouterError::backend(
4956 0,
4957 frame.header.corr,
4958 format!("failed to encode health.check request: {err}"),
4959 )
4960 })?;
4961 let probe_frame = Frame::build_with_version(
4962 negotiated_ver,
4963 FrameType::Request,
4964 control_flags(),
4965 0,
4966 0,
4967 probe_corr,
4968 probe_body,
4969 )
4970 .map_err(RouterError::FrameBuild)?;
4971
4972 if let Err(err) = module_sink.send(probe_frame).await {
4973 return Ok(vec![control_error_frame(
4974 &frame,
4975 "target_unavailable",
4976 err.to_string(),
4977 )?]);
4978 }
4979
4980 match timeout_at(deadline, receiver).await {
4981 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
4982 guard.disarm();
4983 let Some(report) = response.health_report() else {
4984 return Ok(vec![control_error_frame(
4985 &frame,
4986 "invalid_control_body",
4987 "health.check RPC returned a non-health response",
4988 )?]);
4989 };
4990 let HealthReport {
4995 status,
4996 detail,
4997 metrics,
4998 } = report;
4999 let capability_detail = self
5000 .capability_evaluator
5001 .required_problem_detail(&module_id);
5002 let response = ClientControlResponse::SupervisorHealthProbe {
5003 module_id,
5004 status,
5005 detail: append_capability_problem_detail(detail, capability_detail),
5006 metrics,
5007 };
5008 Ok(vec![control_response_body_frame(
5009 &frame,
5010 &response,
5011 "ClientControlResponse::SupervisorHealthProbe",
5012 )?])
5013 }
5014 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
5015 guard.disarm();
5016 Ok(vec![control_error_body_frame(&frame, body)?])
5017 }
5018 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5019 guard.disarm();
5020 Ok(vec![control_error_frame(
5021 &frame,
5022 "target_unavailable",
5023 message,
5024 )?])
5025 }
5026 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5027 guard.disarm();
5028 Ok(vec![control_error_frame(
5029 &frame,
5030 "invalid_control_body",
5031 message,
5032 )?])
5033 }
5034 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5035 guard.disarm();
5036 Ok(vec![control_error_frame(
5037 &frame,
5038 "invalid_control_body",
5039 format!("expected module-control op '{expected}', got '{actual}'"),
5040 )?])
5041 }
5042 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
5043 guard.disarm();
5044 Ok(vec![control_error_frame(
5045 &frame,
5046 "module_timeout",
5047 format!(
5048 "module_id '{module_id}' answered health.check after {:?}",
5049 self.health_probe_timeout
5050 ),
5051 )?])
5052 }
5053 Ok(Err(_)) => Ok(vec![control_error_frame(
5054 &frame,
5055 "target_unavailable",
5056 "health.check waiter was canceled before the module responded",
5057 )?]),
5058 Err(_) => Ok(vec![control_error_frame(
5059 &frame,
5060 "module_timeout",
5061 format!(
5062 "module_id '{module_id}' did not answer health.check within {:?}",
5063 self.health_probe_timeout
5064 ),
5065 )?]),
5066 }
5067 }
5068
5069 fn supervisor_status(
5070 &self,
5071 module_id: &str,
5072 corr: u64,
5073 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
5074 self.supervisor
5075 .get(module_id)
5076 .map(|module| {
5077 let warming = module.is_warming_for_control("status").map_err(|err| {
5078 RouterError::backend(
5079 0,
5080 corr,
5081 format!(
5082 "failed to read supervisor warming state for module_id '{module_id}': {err}"
5083 ),
5084 )
5085 })?;
5086 module.status_for_control("status").map_err(|err| {
5087 RouterError::backend(
5088 0,
5089 corr,
5090 format!(
5091 "failed to read supervisor status for module_id '{module_id}': {err}"
5092 ),
5093 )
5094 }).map(|status| (status, warming))
5095 })
5096 .transpose()
5097 }
5098
5099 fn guard_module_control_op(
5100 &self,
5101 frame: &Frame,
5102 module_id: &str,
5103 op: &str,
5104 ) -> Result<Option<Frame>, RouterError> {
5105 if self.module_grants_op(module_id, op, frame.header.corr)? {
5106 return Ok(None);
5107 }
5108
5109 Ok(Some(control_error_frame(
5110 frame,
5111 "op_not_allowed",
5112 format!("module_id '{module_id}' did not grant control op '{op}'"),
5113 )?))
5114 }
5115
5116 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
5117 let Some(registration) = self
5118 .registry
5119 .get_module(module_id)
5120 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
5121 else {
5122 return Ok(false);
5123 };
5124 Ok(module_registration_grants_op(®istration.control_ops, op))
5125 }
5126
5127 fn handle_status_update(
5128 &self,
5129 endpoint: ModuleEndpointId,
5130 frame: Frame,
5131 ) -> Result<Vec<Frame>, RouterError> {
5132 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
5133 Ok(update) => update,
5134 Err(err) => {
5135 if is_known_module_push_op(&frame.body) {
5140 return Ok(vec![control_error_frame(
5141 &frame,
5142 "invalid_control_body",
5143 format!("malformed module control push body: {err}"),
5144 )?]);
5145 }
5146 return Ok(Vec::new());
5147 }
5148 };
5149
5150 match update {
5151 ModuleControlPush::RouteStatus {
5152 route_channel,
5153 route_epoch,
5154 status,
5155 } => {
5156 self.forwarding
5157 .cache_status(endpoint, route_channel, route_epoch, status)
5158 .map_err(RouterError::Forwarding)?;
5159 }
5160 }
5161 Ok(Vec::new())
5162 }
5163
5164 fn handle_route_poll(
5165 &self,
5166 ctx: &RouteCtx,
5167 frame: Frame,
5168 route_channel: u16,
5169 route_epoch: u32,
5170 kind: PollKind,
5171 ) -> Result<Vec<Frame>, RouterError> {
5172 let snapshot = self
5173 .forwarding
5174 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
5175 .map_err(RouterError::Forwarding)?;
5176 let response = match (kind, snapshot) {
5177 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
5178 ClientControlResponse::RoutePoll {
5179 route_channel,
5180 route_epoch,
5181 status,
5182 live: None,
5183 }
5184 }
5185 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
5186 route_channel,
5187 route_epoch,
5188 status: None,
5189 live: None,
5190 },
5191 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
5192 let live = self
5218 .process_liveness
5219 .as_ref()
5220 .and_then(|source| source.process_live(&module_id))
5221 .unwrap_or(true);
5222 ClientControlResponse::RoutePoll {
5223 route_channel,
5224 route_epoch,
5225 status: None,
5226 live: Some(live),
5227 }
5228 }
5229 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
5230 route_channel,
5231 route_epoch,
5232 status: None,
5233 live: Some(false),
5234 },
5235 };
5236
5237 Ok(vec![control_response_body_frame(
5238 &frame,
5239 &response,
5240 "ClientControlResponse::RoutePoll",
5241 )?])
5242 }
5243
5244 pub(crate) fn observe_module_control_completion(
5245 &self,
5246 completion: ModuleControlRpcCompletion,
5247 ) -> bool {
5248 match completion {
5249 ModuleControlRpcCompletion::Unknown => false,
5250 ModuleControlRpcCompletion::Settled => true,
5251 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
5252 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
5253 info!(
5254 module_id = %module_id,
5255 latency_ms,
5256 "late health.check answer proves the module is alive"
5257 );
5258 match self
5259 .supervisor
5260 .record_late_health_answer(&module_id, latency_ms)
5261 {
5262 Ok(true) => {}
5263 Ok(false) => debug!(
5264 module_id = %module_id,
5265 latency_ms,
5266 "late health.check answer has no active supervisor snapshot"
5267 ),
5268 Err(err) => warn!(
5269 module_id = %module_id,
5270 latency_ms,
5271 error = %err,
5272 "failed to record late health.check answer"
5273 ),
5274 }
5275 true
5276 }
5277 }
5278 }
5279
5280 fn refuse_to_end_module_connection_for_a_client(
5301 &self,
5302 module_connection_id: ConnectionId,
5303 corr: u64,
5304 err: ForwardingError,
5305 ) -> Result<(), RouterError> {
5306 if let ForwardingError::ConnectionClosing { connection_id } = err {
5307 if connection_id != module_connection_id {
5308 warn!(
5309 module_connection_id = module_connection_id.get(),
5310 client_connection_id = connection_id.get(),
5311 corr,
5312 "dropping a route.bind response for a closing client; the module connection keeps serving"
5313 );
5314 return Ok(());
5315 }
5316 }
5317 Err(RouterError::Forwarding(err))
5318 }
5319
5320 fn handle_module_relay_response(
5321 &self,
5322 connection_id: ConnectionId,
5323 frame: Frame,
5324 ) -> Result<Vec<Frame>, RouterError> {
5325 let mut secondary_error = None;
5326 let outcome = match frame.header.ty {
5327 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
5328 Ok(probe) if probe.op == "route.bind" => {
5329 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
5330 Ok(ModuleControlResponse::RouteBindAck {}) => {
5331 RouteBindRelayOutcome::Accepted
5332 }
5333 Ok(other) => {
5334 let message =
5335 format!("route.bind response carried unexpected body: {other:?}");
5336 secondary_error = Some(control_error_frame(
5337 &frame,
5338 "invalid_control_body",
5339 message.clone(),
5340 )?);
5341 RouteBindRelayOutcome::ModuleGone(message)
5342 }
5343 Err(err) => {
5344 let message = format!("malformed route.bind response body: {err}");
5345 secondary_error = Some(control_error_frame(
5346 &frame,
5347 "invalid_control_body",
5348 message.clone(),
5349 )?);
5350 RouteBindRelayOutcome::ModuleGone(message)
5351 }
5352 }
5353 }
5354 Ok(probe) => {
5355 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
5356 {
5357 Ok(response) => ModuleControlRpcOutcome::Response(response),
5358 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
5359 "malformed {} response body: {err}",
5360 probe.op
5361 )),
5362 };
5363 let completion = self
5364 .forwarding
5365 .complete_module_control_rpc(
5366 connection_id,
5367 frame.header.corr,
5368 Some(&probe.op),
5369 outcome,
5370 )
5371 .map_err(RouterError::Forwarding)?;
5372 if !self.observe_module_control_completion(completion) {
5373 debug!(
5374 connection_id = connection_id.get(),
5375 corr = frame.header.corr,
5376 op = %probe.op,
5377 "dropping late or unknown module-control RPC response"
5378 );
5379 }
5380 return Ok(Vec::new());
5381 }
5382 Err(err) => {
5383 if let Some(expected_op) = self
5384 .forwarding
5385 .pending_module_control_op(connection_id, frame.header.corr)
5386 .map_err(RouterError::Forwarding)?
5387 {
5388 let completion = self
5389 .forwarding
5390 .complete_module_control_rpc(
5391 connection_id,
5392 frame.header.corr,
5393 None,
5394 ModuleControlRpcOutcome::MalformedResponse(format!(
5395 "malformed {expected_op} response body: {err}"
5396 )),
5397 )
5398 .map_err(RouterError::Forwarding)?;
5399 if !self.observe_module_control_completion(completion) {
5400 debug!(
5401 connection_id = connection_id.get(),
5402 corr = frame.header.corr,
5403 "dropping late malformed module-control RPC response"
5404 );
5405 }
5406 return Ok(Vec::new());
5407 }
5408 let message = format!("malformed route.bind response body: {err}");
5409 secondary_error = Some(control_error_frame(
5410 &frame,
5411 "invalid_control_body",
5412 message.clone(),
5413 )?);
5414 RouteBindRelayOutcome::ModuleGone(message)
5415 }
5416 },
5417 FrameType::Error => {
5418 if self
5419 .forwarding
5420 .pending_module_control_op(connection_id, frame.header.corr)
5421 .map_err(RouterError::Forwarding)?
5422 .is_some()
5423 {
5424 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
5425 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
5426 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
5427 "malformed module-control ERROR body: {err}"
5428 )),
5429 };
5430 let completion = self
5431 .forwarding
5432 .complete_module_control_rpc(
5433 connection_id,
5434 frame.header.corr,
5435 None,
5436 outcome,
5437 )
5438 .map_err(RouterError::Forwarding)?;
5439 if !self.observe_module_control_completion(completion) {
5440 debug!(
5441 connection_id = connection_id.get(),
5442 corr = frame.header.corr,
5443 "dropping late or unknown module-control RPC error"
5444 );
5445 }
5446 return Ok(Vec::new());
5447 }
5448 match serde_json::from_slice::<ErrorBody>(&frame.body) {
5449 Ok(body) => RouteBindRelayOutcome::Rejected(body),
5450 Err(err) => {
5451 let message = format!("malformed route.bind ERROR body: {err}");
5452 secondary_error = Some(control_error_frame(
5453 &frame,
5454 "invalid_control_body",
5455 message.clone(),
5456 )?);
5457 RouteBindRelayOutcome::ModuleGone(message)
5458 }
5459 }
5460 }
5461 ty => {
5462 return Ok(vec![control_error_frame(
5463 &frame,
5464 "unsupported_control_frame",
5465 format!("unsupported module channel-0 frame {ty:?}"),
5466 )?])
5467 }
5468 };
5469
5470 let settled =
5471 self.forwarding
5472 .complete_pending_relay(connection_id, frame.header.corr, outcome);
5473 let completion = match settled {
5474 Ok(completion) => completion,
5475 Err(err) => {
5476 self.refuse_to_end_module_connection_for_a_client(
5477 connection_id,
5478 frame.header.corr,
5479 err,
5480 )?;
5481 return Ok(secondary_error.into_iter().collect());
5482 }
5483 };
5484 if let Some(target) = completion.abandoned.as_ref() {
5485 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
5486 }
5487 if !completion.settled {
5488 debug!(
5489 connection_id = connection_id.get(),
5490 corr = frame.header.corr,
5491 frame_type = ?frame.header.ty,
5492 "dropping late or unknown route.bind relay response"
5493 );
5494 }
5495 Ok(secondary_error.into_iter().collect())
5496 }
5497
5498 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
5499 debug!(connection_id = connection_id.get(), "handling GOODBYE");
5500 self.cleanup_connection(connection_id)
5504 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
5505 Ok(Vec::new())
5506 }
5507}
5508
5509impl Default for ControlHandler {
5510 fn default() -> Self {
5511 Self::new(Arc::new(Registry::default()))
5512 }
5513}
5514
5515impl crate::supervise::SwapPromotionObserver for ControlHandler {
5516 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
5517 self.apply_registration_capabilities(registration);
5518 }
5519}
5520
5521fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
5522 CapabilityRequirementStatus {
5523 consumer: status.consumer,
5524 capability: status.capability,
5525 need: match status.need {
5526 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
5527 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
5528 },
5529 verdict: status.verdict.as_str().to_string(),
5530 episode_seq: status.episode_seq,
5531 config_satisfiable: status.config_satisfiable,
5532 runtime_available: status.runtime_available,
5533 detail: status.detail,
5534 }
5535}
5536
5537fn append_capability_problem_detail(
5538 detail: Option<String>,
5539 capability_detail: Option<String>,
5540) -> Option<String> {
5541 match (detail, capability_detail) {
5542 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
5543 (Some(detail), None) => Some(detail),
5544 (None, Some(capability_detail)) => Some(capability_detail),
5545 (None, None) => None,
5546 }
5547}
5548
5549fn subc_ops() -> Vec<String> {
5550 SUBC_CONTROL_OPS
5551 .iter()
5552 .map(|op| (*op).to_string())
5553 .collect()
5554}
5555
5556fn module_subc_ops() -> Vec<String> {
5557 SUBC_CONTROL_OPS
5558 .iter()
5559 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
5560 .map(|op| (*op).to_string())
5561 .collect()
5562}
5563
5564#[cfg(test)]
5565fn module_baseline_control_ops() -> Vec<String> {
5566 MODULE_BASELINE_CONTROL_OPS
5567 .iter()
5568 .map(|op| (*op).to_string())
5569 .collect()
5570}
5571
5572fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
5573 let mut seen = HashSet::new();
5574 let mut effective = Vec::new();
5575 for op in MODULE_BASELINE_CONTROL_OPS {
5576 if seen.insert((*op).to_string()) {
5577 effective.push((*op).to_string());
5578 }
5579 }
5580 for op in declared.unwrap_or_default() {
5581 if seen.insert(op.clone()) {
5582 effective.push(op);
5583 }
5584 }
5585 effective
5586}
5587
5588fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
5589 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
5590}
5591
5592fn target_module_id(target: &RouteTarget) -> &str {
5593 match target {
5594 RouteTarget::ToolProvider { module_id }
5595 | RouteTarget::ManagementSurface { module_id }
5596 | RouteTarget::InternalService { module_id, .. } => module_id,
5597 }
5598}
5599
5600fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
5601 roles.iter().any(|role| match (target, role) {
5602 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
5603 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
5604 (
5605 RouteTarget::InternalService { service_id, .. },
5606 ProviderRole::InternalService {
5607 service_id: provided,
5608 ..
5609 },
5610 ) => service_id == provided,
5611 _ => false,
5612 })
5613}
5614
5615fn is_routable_role(role: &ProviderRole) -> bool {
5616 matches!(
5617 role,
5618 ProviderRole::ToolProvider { .. }
5619 | ProviderRole::ManagementSurface { .. }
5620 | ProviderRole::InternalService { .. }
5621 )
5622}
5623
5624#[derive(Debug, Clone, Copy, PartialEq, Eq)]
5625enum ControlRequestBodyError {
5626 UnknownOp,
5627 InvalidBody,
5628}
5629
5630#[derive(Debug, Deserialize)]
5631struct ControlOpProbe {
5632 op: String,
5633}
5634
5635const MODULE_PUSH_OPS: &[&str] = &["route.status"];
5638
5639fn is_known_module_push_op(body: &[u8]) -> bool {
5640 serde_json::from_slice::<ControlOpProbe>(body)
5641 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
5642 .unwrap_or(false)
5643}
5644
5645fn is_known_module_request_op(body: &[u8]) -> bool {
5646 serde_json::from_slice::<ControlOpProbe>(body)
5647 .map(|probe| is_module_to_subc_op(&probe.op))
5648 .unwrap_or(false)
5649}
5650
5651fn is_module_to_subc_op(op: &str) -> bool {
5652 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
5653}
5654
5655fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
5656 debug!(
5657 op = %op,
5658 connection_id = connection_id.get(),
5659 corr,
5660 "control dispatch"
5661 );
5662}
5663
5664fn log_slow_control_dispatch(
5665 dispatch_started_at: Option<StdInstant>,
5666 op: &'static str,
5667 connection_id: ConnectionId,
5668 corr: u64,
5669) {
5670 let Some(dispatch_started_at) = dispatch_started_at else {
5671 return;
5672 };
5673 let elapsed = dispatch_started_at.elapsed();
5674 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
5675 warn!(
5676 op = %op,
5677 connection_id = connection_id.get(),
5678 corr,
5679 elapsed_ms = elapsed.as_millis() as u64,
5680 "slow control dispatch"
5681 );
5682 }
5683}
5684
5685fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
5686 match request {
5687 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
5688 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
5689 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
5690 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
5691 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
5692 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
5693 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
5694 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
5695 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
5696 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
5697 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
5698 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
5699 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
5700 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
5701 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
5702 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
5703 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
5704 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
5705 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
5706 }
5707}
5708
5709fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
5710 match request {
5711 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
5712 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
5713 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
5714 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
5715 }
5716}
5717
5718fn parse_client_control_request(
5719 body: &[u8],
5720) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
5721 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
5722 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5723 Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
5724 ControlRequestBodyError::InvalidBody
5725 }
5726 Ok(_) => ControlRequestBodyError::UnknownOp,
5727 Err(_) => ControlRequestBodyError::InvalidBody,
5728 };
5729 (err, classification)
5730 })
5731}
5732
5733fn parse_module_control_request_from_module(
5734 body: &[u8],
5735) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
5736 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
5737 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5738 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
5739 Ok(_) => ControlRequestBodyError::UnknownOp,
5740 Err(_) => ControlRequestBodyError::InvalidBody,
5741 };
5742 (err, classification)
5743 })
5744}
5745
5746#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
5747enum ProviderRoleKind {
5748 ToolProvider,
5749 PipelineStage,
5750 ManagementSurface,
5751 InternalService,
5752}
5753
5754fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
5755 match role {
5756 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
5757 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
5758 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
5759 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
5760 }
5761}
5762
5763fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
5764 roles.iter().map(provider_role_kind).collect()
5765}
5766
5767const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
5770
5771#[derive(Debug, Default, PartialEq, Eq)]
5773struct ScopeOutcomeCounts {
5774 created: usize,
5775 replaced: usize,
5776 updated: usize,
5777 unchanged: usize,
5778 refused: usize,
5779}
5780
5781impl ScopeOutcomeCounts {
5782 fn of(results: &[ScopeRecordResult]) -> Self {
5783 let mut counts = Self::default();
5784 for result in results {
5785 let slot = match result.outcome {
5786 ScopeRecordOutcome::Created => &mut counts.created,
5787 ScopeRecordOutcome::Replaced => &mut counts.replaced,
5788 ScopeRecordOutcome::Updated => &mut counts.updated,
5789 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
5790 ScopeRecordOutcome::Refused => &mut counts.refused,
5791 };
5792 *slot += 1;
5793 }
5794 counts
5795 }
5796}
5797
5798#[cfg(test)]
5799mod scope_outcome_count_tests {
5800 use super::*;
5801
5802 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
5803 ScopeRecordResult {
5804 scope_ref: "r".to_string(),
5805 scope_epoch: 1,
5806 outcome,
5807 code: None,
5808 message: None,
5809 version: None,
5810 parent_state: None,
5811 }
5812 }
5813
5814 #[test]
5817 fn every_outcome_is_counted_in_its_own_field() {
5818 let results = [
5819 result(ScopeRecordOutcome::Created),
5820 result(ScopeRecordOutcome::Created),
5821 result(ScopeRecordOutcome::Replaced),
5822 result(ScopeRecordOutcome::Updated),
5823 result(ScopeRecordOutcome::Unchanged),
5824 result(ScopeRecordOutcome::Refused),
5825 result(ScopeRecordOutcome::Refused),
5826 result(ScopeRecordOutcome::Refused),
5827 ];
5828 assert_eq!(
5829 ScopeOutcomeCounts::of(&results),
5830 ScopeOutcomeCounts {
5831 created: 2,
5832 replaced: 1,
5833 updated: 1,
5834 unchanged: 1,
5835 refused: 3,
5836 }
5837 );
5838 }
5839}
5840
5841fn capability_census_trigger(
5845 old: Option<&CapabilityDeclarations>,
5846 new: Option<&CapabilityDeclarations>,
5847) -> bool {
5848 let old_provides = old
5849 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
5850 .unwrap_or_default();
5851 let old_denies = old
5852 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
5853 .unwrap_or_default();
5854 let new = new.cloned().unwrap_or(CapabilityDeclarations {
5855 provides: Vec::new(),
5856 requires: Vec::new(),
5857 must_never_reach: Vec::new(),
5858 });
5859
5860 new.provides
5861 .iter()
5862 .any(|capability| !old_provides.contains(capability))
5863 || new
5864 .must_never_reach
5865 .iter()
5866 .any(|capability| !old_denies.contains(capability))
5867}
5868
5869fn denied_capability<'a>(
5872 opening_manifest: &'a ModuleManifest,
5873 target_manifest: &ModuleManifest,
5874) -> Option<&'a str> {
5875 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
5876 let target_capabilities = target_manifest.capabilities.as_ref()?;
5877 opening_capabilities
5878 .must_never_reach
5879 .iter()
5880 .find(|denied| {
5881 target_capabilities
5882 .provides
5883 .iter()
5884 .any(|provided| provided == *denied)
5885 })
5886 .map(String::as_str)
5887}
5888
5889fn catalog_update_frozen_field_message(
5890 registered: &ModuleManifest,
5891 provides: &[ProviderRole],
5892) -> Option<String> {
5893 let old_has_provides = !registered.provides.is_empty();
5894 let new_has_provides = !provides.is_empty();
5895 if old_has_provides != new_has_provides {
5896 return Some(format!(
5897 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
5898 registered.module_id
5899 ));
5900 }
5901
5902 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
5903 return Some(format!(
5904 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
5905 registered.module_id
5906 ));
5907 }
5908
5909 let registered_concurrency = manifest_concurrency(registered);
5910 let mut candidate = registered.clone();
5911 candidate.provides = provides.to_vec();
5912 let candidate_concurrency = manifest_concurrency(&candidate);
5913 if candidate_concurrency != registered_concurrency {
5914 return Some(format!(
5915 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
5916 registered.module_id, registered_concurrency, candidate_concurrency
5917 ));
5918 }
5919
5920 None
5923}
5924
5925fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
5926 manifest.provides.iter().any(is_routable_role)
5927}
5928
5929fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
5935 manifest
5936 .provides
5937 .iter()
5938 .find_map(|provider| match provider {
5939 ProviderRole::ToolProvider { concurrency, .. }
5940 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
5941 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
5942 })
5943 .unwrap_or(Concurrency::ModuleManaged)
5944}
5945
5946fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
5953 let has_management_surface = manifest
5954 .provides
5955 .iter()
5956 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
5957 if !has_management_surface {
5958 return false;
5959 }
5960 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
5961 return false;
5962 };
5963 let Some(provides) = raw
5964 .get("manifest")
5965 .and_then(|manifest| manifest.get("provides"))
5966 .and_then(serde_json::Value::as_array)
5967 else {
5968 return false;
5969 };
5970 provides.iter().any(|role| {
5975 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
5976 && role.get("concurrency").is_none()
5977 })
5978}
5979
5980fn negotiate_version(peer_version: u8) -> Result<u8, String> {
5981 if peer_version != PROTOCOL_VERSION {
5982 return Err(format!(
5983 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
5984 ));
5985 }
5986 Ok(PROTOCOL_VERSION)
5987}
5988
5989fn pong(frame: &Frame) -> Result<Frame, RouterError> {
5990 Frame::build_with_version(
5991 response_version(frame),
5992 FrameType::Pong,
5993 frame.header.flags,
5994 0,
5995 0,
5996 frame.header.corr,
5997 Vec::new(),
5998 )
5999 .map_err(RouterError::FrameBuild)
6000}
6001
6002fn control_error_frame(
6003 frame: &Frame,
6004 code: &'static str,
6005 message: impl Into<String>,
6006) -> Result<Frame, RouterError> {
6007 control_error_body_frame(
6008 frame,
6009 ErrorBody {
6010 code: code.to_string(),
6011 message: message.into(),
6012 detail: None,
6013 },
6014 )
6015}
6016
6017fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
6018 let body = serde_json::to_vec(&error).map_err(|err| {
6019 RouterError::backend(
6020 0,
6021 frame.header.corr,
6022 format!("failed to encode control ERROR: {err}"),
6023 )
6024 })?;
6025
6026 Frame::build_with_version(
6027 response_version(frame),
6028 FrameType::Error,
6029 control_flags(),
6030 0,
6031 0,
6032 frame.header.corr,
6033 body,
6034 )
6035 .map_err(RouterError::FrameBuild)
6036}
6037
6038fn control_response_body_frame<T: Serialize>(
6039 frame: &Frame,
6040 reply: &T,
6041 label: &'static str,
6042) -> Result<Frame, RouterError> {
6043 let body = serde_json::to_vec(reply).map_err(|err| {
6044 RouterError::backend(
6045 0,
6046 frame.header.corr,
6047 format!("failed to encode {label}: {err}"),
6048 )
6049 })?;
6050
6051 Frame::build_with_version(
6052 response_version(frame),
6053 FrameType::Response,
6054 control_flags(),
6055 0,
6056 0,
6057 frame.header.corr,
6058 body,
6059 )
6060 .map_err(RouterError::FrameBuild)
6061}
6062
6063fn forwarding_error_code(err: &ForwardingError) -> &'static str {
6093 match err {
6094 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
6095 ForwardingError::NoModuleConnection => "target_unavailable",
6096 ForwardingError::ModuleReloading { .. } => "module_reloading",
6097 ForwardingError::ClientRouteChannelExhausted { .. }
6098 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
6099 ForwardingError::StaleModuleEndpoint
6100 | ForwardingError::UnknownReservation { .. }
6101 | ForwardingError::ConnectionClosing { .. }
6102 | ForwardingError::ClientEgressClosed { .. }
6103 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
6104 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
6107 ForwardingError::RelayCorrelationExhausted
6108 | ForwardingError::RouteOpenBuild(_)
6109 | ForwardingError::Poisoned => "forwarding_error",
6110 }
6111}
6112
6113fn response_version(frame: &Frame) -> u8 {
6114 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
6115 frame.header.ver
6116 } else {
6117 PROTOCOL_VERSION
6118 }
6119}
6120
6121fn control_flags() -> Flags {
6122 Flags::new(false, Priority::Passive, false)
6123}
6124
6125fn send_goodbye_target_best_effort(
6130 counters: &DaemonCounters,
6131 target: &GoodbyeTarget,
6132 context: &'static str,
6133) {
6134 let Ok(frame) = Frame::build_with_version(
6135 target.negotiated_ver,
6136 FrameType::Goodbye,
6137 control_flags(),
6138 target.channel,
6139 target.epoch,
6140 0,
6141 Vec::new(),
6142 ) else {
6143 return;
6144 };
6145 crate::forwarding::send_module_route_goodbye(
6146 counters,
6147 &target.sink,
6148 frame,
6149 target.module_id.as_deref(),
6150 context,
6151 );
6152}
6153
6154pub(crate) fn send_route_control_pushes(
6155 forwarding: &ForwardingTable,
6156 routes: Vec<EndpointRoute>,
6157 push: ClientControlPush,
6158) {
6159 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
6160 for route in routes {
6161 let target = route.goodbye_target;
6162 if let Some((existing, channels)) = targets
6163 .iter_mut()
6164 .find(|(existing, _)| existing.connection_id == target.connection_id)
6165 {
6166 debug_assert_eq!(
6167 existing.negotiated_ver, target.negotiated_ver,
6168 "one connection cannot negotiate multiple frame versions"
6169 );
6170 if !channels.contains(&target.channel) {
6171 channels.push(target.channel);
6172 }
6173 continue;
6174 }
6175 let channel = target.channel;
6176 targets.push((target, vec![channel]));
6177 }
6178 for (target, mut channels) in targets {
6179 channels.sort_unstable();
6180 let mut push = push.clone();
6181 match &mut push {
6182 ClientControlPush::RouteClosing {
6183 channels: covered, ..
6184 }
6185 | ClientControlPush::RouteClosed {
6186 channels: covered, ..
6187 } => *covered = channels,
6188 }
6189 let body = match serde_json::to_vec(&push) {
6190 Ok(body) => body,
6191 Err(err) => {
6192 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
6193 continue;
6194 }
6195 };
6196 let frame = match Frame::build_with_version(
6197 target.negotiated_ver,
6198 FrameType::Push,
6199 control_flags(),
6200 0,
6201 0,
6202 0,
6203 body.clone(),
6204 ) {
6205 Ok(frame) => frame,
6206 Err(err) => {
6207 warn!(
6208 route_channel = target.channel,
6209 error = %err,
6210 "failed to build route lifecycle control PUSH frame"
6211 );
6212 continue;
6213 }
6214 };
6215 if let Err(err) = target.sink.try_send(frame) {
6216 if target.close_on_delivery_failure() {
6217 warn!(
6218 target_connection_id = target.connection_id.get(),
6219 route_channel = target.channel,
6220 error = %err,
6221 "route lifecycle control PUSH was not delivered to client; closing target connection"
6222 );
6223 let _ = forwarding.escalate_client_delivery_failure(
6224 target.connection_id,
6225 target.channel,
6226 target.epoch,
6227 CloseReason::new(
6228 "route_lifecycle_push_delivery_failed",
6229 format!(
6230 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
6231 target.channel
6232 ),
6233 ),
6234 crate::forwarding::UndeliveredFrame {
6235 module_id: target.module_id.as_deref(),
6236 sink: &target.sink,
6237 },
6238 );
6239 }
6240 }
6241 }
6242}
6243
6244#[cfg(test)]
6245mod tests {
6246 #[cfg(unix)]
6247 #[tokio::test]
6248 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
6249 let dir = subc_test_support::TestTempDir::new("rescan-live-health");
6250 let path = dir.join("subc.jsonc");
6251 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
6252 "program":"/bin/sleep","args":["60"],"protocol":"none",
6253 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
6254 }}}).to_string()).unwrap();
6255 let mut configured = crate::daemon_config::load(&path)
6256 .unwrap()
6257 .unwrap()
6258 .modules
6259 .pop()
6260 .unwrap();
6261 let registry = std::sync::Arc::new(crate::Registry::default());
6262 let handle = crate::SupervisorHandle::new();
6263 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
6264 .with_handle(handle.clone());
6265 let module = supervisor
6266 .supervise_configured_with_health(
6267 configured.module_spec(),
6268 true,
6269 configured.health.clone(),
6270 None,
6271 configured.restart,
6272 )
6273 .unwrap();
6274 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
6275 let before = module.status().unwrap().pid;
6276 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
6277 configured.health.cadence = std::time::Duration::from_secs(3600);
6278 let health_only = handler
6279 .reconcile_supervised_modules(&supervisor, vec![configured.clone()], false)
6280 .await
6281 .unwrap();
6282 assert!(
6283 health_only.changed_pending_reload.is_empty(),
6284 "health policy is already applied live"
6285 );
6286 assert_eq!(module.status().unwrap().pid, before);
6287 assert_eq!(
6288 module.configuration().unwrap().1.http,
6289 configured.health.http
6290 );
6291 configured.args = vec!["61".into()];
6292 let launch = handler
6293 .reconcile_supervised_modules(&supervisor, vec![configured], false)
6294 .await
6295 .unwrap();
6296 assert_eq!(launch.changed_pending_reload, ["stock"]);
6297 assert_eq!(
6298 module.status().unwrap().pid,
6299 before,
6300 "a launch edit is stored until reload"
6301 );
6302 module.drain().await.unwrap();
6303 }
6304 use std::{
6305 collections::BTreeMap,
6306 fmt,
6307 path::PathBuf,
6308 sync::{Arc, Mutex},
6309 time::Duration,
6310 };
6311 use subc_test_support::TestTempDir;
6312
6313 use serde_json::{json, Value};
6314 use subc_protocol::{
6315 manifest::{
6316 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
6317 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
6318 },
6319 session::HealthStatus,
6320 FrameType,
6321 };
6322
6323 use super::*;
6324 use crate::{
6325 forwarding::{DataRoute, DataRouteState},
6326 registry::ChannelState,
6327 router::FrameSink,
6328 stderr_tail::DEFAULT_MAX_LINE_BYTES,
6329 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
6330 RouteCtx, Router,
6331 };
6332 use tokio::{
6333 sync::mpsc,
6334 time::{sleep, Instant},
6335 };
6336 use tracing::{
6337 field::{Field, Visit},
6338 Event, Subscriber,
6339 };
6340 use tracing_subscriber::{layer::Context, prelude::*, Layer};
6341
6342 fn fake_aft_stub_path() -> PathBuf {
6361 let mut path = std::env::current_exe().expect("current_exe available in tests");
6362 path.pop(); path.pop(); path.push(if cfg!(windows) {
6365 "fake-aft-stub.exe"
6366 } else {
6367 "fake-aft-stub"
6368 });
6369 assert!(
6370 path.exists(),
6371 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
6372 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
6373 path.display()
6374 );
6375 path
6376 }
6377
6378 fn client_retries(code: &str) -> bool {
6383 subc_protocol::error_codes::is_retryable_route_open(code)
6384 }
6385
6386 #[test]
6397 fn retryability_of_forwarding_codes_matches_the_failure() {
6398 let transient = [
6401 ForwardingError::NoModuleConnection,
6402 ForwardingError::ModuleReloading {
6403 module_id: "m".into(),
6404 },
6405 ForwardingError::StaleModuleEndpoint,
6406 ForwardingError::UnknownReservation {
6407 client_channel: 1,
6408 module_channel: 1,
6409 },
6410 ForwardingError::ConnectionClosing {
6411 connection_id: ConnectionId::new(1),
6412 },
6413 ForwardingError::ClientEgressClosed {
6414 connection_id: ConnectionId::new(1),
6415 },
6416 ForwardingError::ModuleEgressUnavailable {
6417 connection_id: ConnectionId::new(1),
6418 },
6419 ];
6420 for err in transient {
6421 let code = forwarding_error_code(&err);
6422 assert!(
6423 client_retries(code),
6424 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
6425 );
6426 }
6427
6428 let permanent = [
6433 ForwardingError::ConnectionRoleConflict {
6434 connection_id: ConnectionId::new(1),
6435 },
6436 ForwardingError::ClientRouteChannelExhausted {
6437 connection_id: ConnectionId::new(1),
6438 },
6439 ForwardingError::ModuleRouteChannelExhausted {
6440 endpoint: ModuleEndpointId {
6441 connection_id: ConnectionId::new(1),
6442 generation: 1,
6443 },
6444 },
6445 ForwardingError::RelayCorrelationExhausted,
6446 ForwardingError::RouteOpenBuild("x".into()),
6447 ForwardingError::Poisoned,
6448 ];
6449 for err in permanent {
6450 let code = forwarding_error_code(&err);
6451 assert!(
6452 !client_retries(code),
6453 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
6454 );
6455 }
6456 }
6457
6458 #[tokio::test]
6486 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
6487 let handler = ControlHandler::default();
6488 let frame =
6489 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
6490
6491 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
6495 assert_eq!(
6496 stamped,
6497 Principal::Direct,
6498 "a caller that proved nothing must not be stamped as a supervised module"
6499 );
6500
6501 let forged = handler
6506 .route_open_principal(
6507 &frame,
6508 Some(ConsumerIdentity {
6509 module_id: "aft".to_string(),
6510 launch_nonce: "not-a-real-nonce".to_string(),
6511 }),
6512 )
6513 .unwrap();
6514 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
6515 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
6516 }
6517
6518 #[test]
6535 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
6536 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
6537 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
6538 let ClientControlRequest::RouteOpen {
6539 consumer_identity, ..
6540 } = parsed
6541 else {
6542 panic!("route.open body must parse as RouteOpen");
6543 };
6544 assert_eq!(
6545 consumer_identity,
6546 Some(ConsumerIdentity {
6547 module_id: "aft".to_string(),
6548 launch_nonce: "n".to_string(),
6549 }),
6550 "the wire field name must reach the value route_open_principal reads"
6551 );
6552 }
6553
6554 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
6555 ModuleManifest::builder(module_id, "0.1.0")
6556 .protocol_ver(protocol_ver)
6557 .provides(vec![ProviderRole::ToolProvider {
6558 tools: vec![Tool {
6559 name: "read".to_string(),
6560 description: None,
6561 execution_mode: ExecutionMode::Pure,
6562 schema: json!({"type": "object"}),
6563 }],
6564 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
6565 concurrency: Concurrency::ModuleManaged,
6566 emits_push: true,
6567 sub_supervises: true,
6568 }])
6569 .build()
6570 }
6571
6572 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
6573 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
6574 }
6575
6576 fn hello_frame_with_control_ops(
6577 module_id: &str,
6578 protocol_ver: u8,
6579 corr: u64,
6580 control_ops: Option<Vec<String>>,
6581 ) -> Frame {
6582 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
6583 }
6584
6585 fn hello_frame_with_nonce(
6586 module_id: &str,
6587 protocol_ver: u8,
6588 corr: u64,
6589 launch_nonce: Option<&str>,
6590 ) -> Frame {
6591 hello_frame_full(
6592 module_id,
6593 protocol_ver,
6594 corr,
6595 None,
6596 launch_nonce.map(ToOwned::to_owned),
6597 )
6598 }
6599
6600 fn hello_frame_full(
6601 module_id: &str,
6602 protocol_ver: u8,
6603 corr: u64,
6604 control_ops: Option<Vec<String>>,
6605 launch_nonce: Option<String>,
6606 ) -> Frame {
6607 let body = serde_json::to_vec(&ModuleHelloBody {
6608 manifest: manifest(module_id, protocol_ver),
6609 protocol_ver,
6610 control_ops,
6611 launch_nonce,
6612 })
6613 .unwrap();
6614 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
6615 }
6616
6617 fn non_routable_hello_frame_with_control_ops(
6618 module_id: &str,
6619 corr: u64,
6620 control_ops: Option<Vec<String>>,
6621 ) -> Frame {
6622 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
6623 manifest.provides.clear();
6624 let body = serde_json::to_vec(&ModuleHelloBody {
6625 manifest,
6626 protocol_ver: PROTOCOL_VERSION,
6627 control_ops,
6628 launch_nonce: None,
6629 })
6630 .unwrap();
6631 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
6632 }
6633
6634 fn capability_grammar_hello_frame(
6635 capabilities: Value,
6636 runtime_computed: Option<Value>,
6637 corr: u64,
6638 ) -> Frame {
6639 let mut body = serde_json::to_value(ModuleHelloBody {
6640 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
6641 protocol_ver: PROTOCOL_VERSION,
6642 control_ops: None,
6643 launch_nonce: None,
6644 })
6645 .expect("HELLO body serializes");
6646 body["manifest"]["capabilities"] = capabilities;
6647 if let Some(runtime_computed) = runtime_computed {
6648 body["runtime_computed"] = runtime_computed;
6649 }
6650 Frame::build(
6651 FrameType::Hello,
6652 control_flags(),
6653 0,
6654 0,
6655 corr,
6656 serde_json::to_vec(&body).expect("HELLO body reserializes"),
6657 )
6658 .expect("HELLO frame builds")
6659 }
6660
6661 fn channel_request(channel: u16, corr: u64) -> Frame {
6662 Frame::build(
6663 FrameType::Request,
6664 Flags::new(true, Priority::Interactive, false),
6665 channel,
6666 0,
6667 corr,
6668 b"opaque".to_vec(),
6669 )
6670 .unwrap()
6671 }
6672
6673 fn route_ctx(
6674 connection_id: ConnectionId,
6675 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
6676 let (tx, rx) = mpsc::channel(8);
6677 (
6678 RouteCtx {
6679 connection_id,
6680 egress: FrameSink::new(tx),
6681 },
6682 rx,
6683 )
6684 }
6685
6686 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
6687 serde_json::from_slice(&frame.body).unwrap()
6688 }
6689
6690 async fn hello_via_sink(
6695 handler: &ControlHandler,
6696 ctx: &RouteCtx,
6697 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
6698 hello: Frame,
6699 ) -> Frame {
6700 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
6701 assert!(
6702 replies.is_empty(),
6703 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
6704 );
6705 let ack = rx
6706 .try_recv()
6707 .expect("HELLO_ACK is queued on the module sink")
6708 .frame;
6709 assert_eq!(ack.header.ty, FrameType::HelloAck);
6710 ack
6711 }
6712
6713 fn parse_error(frame: &Frame) -> Value {
6714 serde_json::from_slice(&frame.body).unwrap()
6715 }
6716
6717 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
6718 serde_json::from_slice(&frame.body).unwrap()
6719 }
6720
6721 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
6722 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
6723 route_channel,
6724 route_epoch: 0,
6725 kind,
6726 })
6727 .unwrap();
6728 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6729 }
6730
6731 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
6732 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
6733 module_id: module_id.to_string(),
6734 })
6735 .unwrap();
6736 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6737 }
6738
6739 fn route_open_frame(corr: u64, module_id: &str, project_root: TestTempDir) -> Frame {
6740 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
6741 }
6742
6743 fn route_open_frame_with_consumer_capabilities(
6744 corr: u64,
6745 module_id: &str,
6746 project_root: TestTempDir,
6747 consumer_capabilities: Option<Vec<String>>,
6748 ) -> Frame {
6749 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6750 target: RouteTarget::ToolProvider {
6751 module_id: module_id.to_string(),
6752 },
6753 identity: BindIdentity::new(
6754 project_root.path().to_path_buf(),
6755 "unit".to_string(),
6756 "session".to_string(),
6757 ),
6758 consumer_identity: None,
6759 consumer_capabilities,
6760 role_versions: None,
6761 admission_facts: None,
6762 scope: None,
6763 })
6764 .unwrap();
6765 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6766 }
6767
6768 fn route_open_frame_with_role_versions(
6769 corr: u64,
6770 module_id: &str,
6771 project_root: TestTempDir,
6772 role_versions: Option<BTreeMap<String, String>>,
6773 ) -> Frame {
6774 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6775 target: RouteTarget::ToolProvider {
6776 module_id: module_id.to_string(),
6777 },
6778 identity: BindIdentity::new(
6779 project_root.path().to_path_buf(),
6780 "unit".to_string(),
6781 format!("session-{corr}"),
6782 ),
6783 consumer_identity: None,
6784 consumer_capabilities: None,
6785 role_versions,
6786 admission_facts: None,
6787 scope: None,
6788 })
6789 .unwrap();
6790 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6791 }
6792
6793 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
6794 entries
6795 .iter()
6796 .map(|(role, version)| (role.to_string(), version.to_string()))
6797 .collect()
6798 }
6799
6800 fn route_open_frame_with_admission_facts(
6801 corr: u64,
6802 module_id: &str,
6803 project_root: TestTempDir,
6804 consumer_identity: Option<subc_control::ConsumerIdentity>,
6805 facts: Option<Value>,
6806 ) -> Frame {
6807 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6808 target: RouteTarget::ToolProvider {
6809 module_id: module_id.to_string(),
6810 },
6811 identity: BindIdentity::new(
6812 project_root.path().to_path_buf(),
6813 "unit".to_string(),
6814 format!("session-{corr}"),
6815 ),
6816 consumer_identity,
6817 consumer_capabilities: None,
6818 role_versions: None,
6819 admission_facts: facts,
6820 scope: None,
6821 })
6822 .unwrap();
6823 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6824 }
6825
6826 #[derive(Clone, Default)]
6827 struct EventCapture {
6828 events: Arc<Mutex<Vec<CapturedEvent>>>,
6829 }
6830
6831 #[derive(Clone, Debug)]
6832 struct CapturedEvent {
6833 target: String,
6834 level: tracing::Level,
6835 fields: BTreeMap<String, String>,
6836 }
6837
6838 impl EventCapture {
6839 fn events(&self) -> Vec<CapturedEvent> {
6840 self.events.lock().unwrap().clone()
6841 }
6842 }
6843
6844 impl<S> Layer<S> for EventCapture
6845 where
6846 S: Subscriber,
6847 {
6848 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
6849 let mut visitor = EventFieldVisitor::default();
6850 event.record(&mut visitor);
6851 self.events.lock().unwrap().push(CapturedEvent {
6852 target: event.metadata().target().to_string(),
6853 level: *event.metadata().level(),
6854 fields: visitor.fields,
6855 });
6856 }
6857 }
6858
6859 #[derive(Default)]
6860 struct EventFieldVisitor {
6861 fields: BTreeMap<String, String>,
6862 }
6863
6864 impl Visit for EventFieldVisitor {
6865 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
6866 self.fields
6867 .insert(field.name().to_string(), format!("{value:?}"));
6868 }
6869 }
6870
6871 fn health_response(corr: u64, status: HealthStatus) -> Frame {
6872 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
6873 status,
6874 detail: Some("warming".to_string()),
6875 metrics: Some(json!({"queue_depth": 3})),
6876 })
6877 .unwrap();
6878 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
6879 }
6880
6881 fn route_bind_ack(corr: u64) -> Frame {
6882 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
6883 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
6884 }
6885
6886 fn unique_project_root(label: &str) -> TestTempDir {
6887 TestTempDir::new(label)
6888 }
6889
6890 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
6891 match parse_route_poll(frame) {
6892 ClientControlResponse::RoutePoll {
6893 status: None,
6894 live: Some(live),
6895 ..
6896 } => assert_eq!(live, expected_live),
6897 other => panic!("unexpected route.poll response: {other:?}"),
6898 }
6899 }
6900
6901 fn bind_liveness_route(
6902 registry: &Registry,
6903 forwarding: &ForwardingTable,
6904 module_id: &str,
6905 ) -> (RouteCtx, u16, u32) {
6906 let module_connection = ConnectionId::new(101);
6907 let client_connection = ConnectionId::new(202);
6908 let registration = registry
6909 .register_with_control_ops(
6910 manifest(module_id, PROTOCOL_VERSION),
6911 PROTOCOL_VERSION,
6912 module_connection,
6913 module_baseline_control_ops(),
6914 )
6915 .unwrap();
6916 let (module_tx, _module_rx) = mpsc::channel(8);
6917 let endpoint = forwarding
6918 .register_module_connection(
6919 module_connection,
6920 module_id.to_string(),
6921 PROTOCOL_VERSION,
6922 manifest_concurrency(®istration.manifest),
6923 FrameSink::new(module_tx),
6924 )
6925 .unwrap();
6926 let (client_ctx, _client_rx) = route_ctx(client_connection);
6927 let pending = forwarding
6928 .begin_route_bind_relay_for_test(
6929 client_connection,
6930 client_ctx.egress.clone(),
6931 1,
6932 module_id,
6933 )
6934 .unwrap();
6935 assert_eq!(pending.endpoint, endpoint);
6936 let route_channel = pending.client_channel;
6937 let route_epoch = pending.client_epoch;
6938 forwarding
6939 .complete_pending_relay(
6940 module_connection,
6941 pending.corr,
6942 RouteBindRelayOutcome::Accepted,
6943 )
6944 .unwrap();
6945 (client_ctx, route_channel, route_epoch)
6946 }
6947
6948 struct FakeProcessLiveness {
6949 live: Option<bool>,
6950 }
6951
6952 impl ModuleProcessLiveness for FakeProcessLiveness {
6953 fn process_live(&self, _module_id: &str) -> Option<bool> {
6954 self.live
6955 }
6956 }
6957
6958 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6959 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
6960 {
6961 let registry = Arc::new(Registry::default());
6962 let supervisor_handle = SupervisorHandle::new();
6963 let supervisor = Supervisor::new(
6964 Arc::clone(®istry),
6965 RestartPolicy::new(1, Duration::from_millis(10)),
6966 )
6967 .with_handle(supervisor_handle.clone());
6968 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
6969 let module = supervisor
6970 .spawn(ModuleSpec {
6971 module_id: "stderr-tail-wire".to_string(),
6972 program: fake_aft_stub_path(),
6973 args: Vec::new(),
6974 env: vec![
6975 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
6976 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
6977 ],
6978 reserved: false,
6979 reserved_prefixes: Vec::new(),
6980 protocol: ModuleProtocol::Subc,
6981 overlap: Default::default(),
6982 })
6983 .unwrap();
6984
6985 let deadline = Instant::now() + Duration::from_secs(5);
6986 loop {
6987 let tail = module.stderr_tail(None, None);
6988 if tail
6989 .entries
6990 .iter()
6991 .any(|entry| matches!(entry, TailEntry::ProcessStart))
6992 && tail.entries.iter().any(|entry| {
6993 matches!(
6994 entry,
6995 TailEntry::Line {
6996 truncated: true,
6997 ..
6998 }
6999 )
7000 })
7001 {
7002 break;
7003 }
7004 assert!(
7005 Instant::now() < deadline,
7006 "module did not produce a truncated line and restart boundary: {tail:?}"
7007 );
7008 sleep(Duration::from_millis(10)).await;
7009 }
7010
7011 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
7012 let request = ClientControlRequest::SupervisorStderrTail {
7013 module_id: "stderr-tail-wire".to_string(),
7014 max_lines: None,
7015 max_bytes: None,
7016 };
7017 let frame = Frame::build(
7018 FrameType::Request,
7019 control_flags(),
7020 0,
7021 0,
7022 1,
7023 serde_json::to_vec(&request).unwrap(),
7024 )
7025 .unwrap();
7026 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7027 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
7028 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
7029 serde_json::from_slice(&responses[0].body).unwrap()
7030 else {
7031 panic!("expected supervisor.stderr_tail response");
7032 };
7033
7034 assert!(
7035 tail.entries
7036 .iter()
7037 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
7038 "the control response lost the restart boundary"
7039 );
7040 let Some(StderrTailEntry::Line {
7041 text,
7042 truncated,
7043 at_ms,
7044 }) = tail.entries.iter().find(|entry| {
7045 matches!(
7046 entry,
7047 StderrTailEntry::Line {
7048 truncated: true,
7049 ..
7050 }
7051 )
7052 })
7053 else {
7054 panic!("the control response lost the truncated line");
7055 };
7056 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
7057 assert!(*truncated);
7058 assert!(
7059 at_ms.is_some(),
7060 "the control response lost the line's capture time"
7061 );
7062 }
7063
7064 #[tokio::test(flavor = "current_thread")]
7069 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
7070 let dir = TestTempDir::new("terminals-off-worker");
7071 let journal_path = dir.join("terminals.jsonl");
7072 let registry = Arc::new(Registry::default());
7073 let supervisor_handle = SupervisorHandle::new();
7074 let supervisor =
7075 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
7076 .with_handle(supervisor_handle.clone())
7077 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
7078 let module = supervisor
7079 .spawn(ModuleSpec {
7080 module_id: "terminal-off-worker".to_string(),
7081 program: fake_aft_stub_path(),
7082 args: Vec::new(),
7083 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
7084 reserved: false,
7085 reserved_prefixes: Vec::new(),
7086 protocol: ModuleProtocol::Subc,
7087 overlap: Default::default(),
7088 })
7089 .unwrap();
7090 let deadline = Instant::now() + Duration::from_secs(5);
7091 while module.terminal_history().entries.len() != 2 {
7092 assert!(Instant::now() < deadline, "module did not record two exits");
7093 sleep(Duration::from_millis(10)).await;
7094 }
7095
7096 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
7097 let handler =
7098 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
7099 let frame = Frame::build(
7100 FrameType::Request,
7101 control_flags(),
7102 0,
7103 0,
7104 1,
7105 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
7106 module_id: "terminal-off-worker".to_string(),
7107 })
7108 .unwrap(),
7109 )
7110 .unwrap();
7111 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7112 let spawned_at = std::time::Instant::now();
7113 let read = tokio::spawn({
7114 let handler = Arc::clone(&handler);
7115 async move { handler.handle_control_frame(&ctx, frame).await }
7116 });
7117 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
7120 .await
7121 .unwrap()
7122 .expect("the history read reached its pause");
7123 let elapsed = spawned_at.elapsed();
7124 assert!(
7125 elapsed < Duration::from_secs(2) && !read.is_finished(),
7126 "this task could not run while the history read was paused \
7127 (resumed after {elapsed:?}, read finished: {})",
7128 read.is_finished()
7129 );
7130
7131 drop(release);
7132 let responses = read.await.unwrap().unwrap();
7133 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
7134 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
7135 panic!("expected supervisor.terminals response");
7136 };
7137 assert_eq!(terminals.entries.len(), 2);
7138 assert_eq!(terminals.journal_skipped_lines, 0);
7139 assert_eq!(terminals.journal_read_errors, 0);
7140 }
7141
7142 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7143 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
7144 let registry = Arc::new(Registry::default());
7145 let supervisor_handle = SupervisorHandle::new();
7146 let supervisor =
7147 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
7148 .with_handle(supervisor_handle.clone());
7149 let module = supervisor
7150 .spawn(ModuleSpec {
7151 module_id: "terminal-golden".to_string(),
7152 program: fake_aft_stub_path(),
7153 args: Vec::new(),
7154 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
7155 reserved: false,
7156 reserved_prefixes: Vec::new(),
7157 protocol: ModuleProtocol::Subc,
7158 overlap: Default::default(),
7159 })
7160 .unwrap();
7161
7162 let deadline = Instant::now() + Duration::from_secs(5);
7163 while module.terminal_history().entries.len() != 2 {
7164 assert!(
7165 Instant::now() < deadline,
7166 "module did not retain two terminal exits: {:?}",
7167 module.terminal_history()
7168 );
7169 sleep(Duration::from_millis(10)).await;
7170 }
7171
7172 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
7173 let request = ClientControlRequest::SupervisorTerminals {
7174 module_id: "terminal-golden".to_string(),
7175 };
7176 let frame = Frame::build(
7177 FrameType::Request,
7178 control_flags(),
7179 0,
7180 0,
7181 1,
7182 serde_json::to_vec(&request).unwrap(),
7183 )
7184 .unwrap();
7185 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7186 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
7187 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
7188 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
7189 panic!("expected supervisor.terminals response");
7190 };
7191 assert_eq!(terminals.entries.len(), 2);
7192 assert_eq!(terminals.dropped, 0);
7193
7194 let mut rendered = serde_json::to_value(response).unwrap();
7195 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
7198 for (index, entry) in rendered["entries"]
7199 .as_array_mut()
7200 .expect("terminal response entries array")
7201 .iter_mut()
7202 .enumerate()
7203 {
7204 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
7205 }
7206
7207 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
7208 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
7209 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
7210 if std::env::var_os("UPDATE_GOLDEN").is_some() {
7211 std::fs::write(&golden_path, &serialized).unwrap();
7212 }
7213 let expected: Value =
7214 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
7215 assert_eq!(rendered, expected);
7216 }
7217
7218 #[test]
7219 fn hello_registers_manifest_and_returns_ack() {
7220 let registry = Arc::new(Registry::default());
7221 let handler = ControlHandler::new(Arc::clone(®istry));
7222 let conn = ConnectionId::new(1);
7223
7224 let responses = handler
7225 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
7226 .unwrap();
7227
7228 assert_eq!(responses.len(), 1);
7229 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
7230 assert_eq!(responses[0].header.channel, 0);
7231 assert_eq!(responses[0].header.corr, 7);
7232 let ack = parse_ack(&responses[0]);
7233 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
7234 assert!(ack
7235 .subc_capabilities
7236 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
7237 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
7238 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
7239 assert!(ack
7240 .subc_ops
7241 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
7242 assert!(ack
7243 .subc_ops
7244 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
7245
7246 let registration = registry.get_module("aft").unwrap().unwrap();
7247 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
7248 assert_eq!(registration.state, ChannelState::Active);
7249 assert_eq!(registration.connection_id, conn);
7250 assert_eq!(registration.control_ops, module_baseline_control_ops());
7251 }
7252
7253 #[test]
7254 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
7255 let invalid_identifiers = [
7256 ("case_change", "credentials-Provider/v1"),
7257 ("leading_zero", "credentials-provider/v01"),
7258 ("trailing_hyphen", "credentials-provider-/v1"),
7259 ("consecutive_hyphens", "credentials--provider/v1"),
7260 ("uppercase", "Credentials-provider/v1"),
7261 ("missing_v", "credentials-provider/1"),
7262 ("whitespace", "credentials provider/v1"),
7263 ("zero_version", "credentials-provider/v0"),
7264 ("out_of_range_version", "credentials-provider/v4294967296"),
7265 (
7266 "overlength_name",
7267 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
7268 ),
7269 ];
7270 let mut cases = invalid_identifiers
7271 .into_iter()
7272 .map(|(name, identifier)| {
7273 (
7274 format!("identifier_{name}"),
7275 "capabilities.provides[0]".to_string(),
7276 identifier.to_string(),
7277 json!({ "provides": [identifier] }),
7278 None,
7279 )
7280 })
7281 .collect::<Vec<_>>();
7282 cases.extend([
7283 (
7284 "unknown_need".to_string(),
7285 "capabilities.requires[0].need".to_string(),
7286 "deferred".to_string(),
7287 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
7288 None,
7289 ),
7290 (
7291 "duplicate_provides".to_string(),
7292 "capabilities.provides[1]".to_string(),
7293 "credentials-provider/v1".to_string(),
7294 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
7295 None,
7296 ),
7297 (
7298 "duplicate_must_never_reach".to_string(),
7299 "capabilities.must_never_reach[1]".to_string(),
7300 "credentials-provider/v1".to_string(),
7301 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
7302 None,
7303 ),
7304 (
7305 "duplicate_requires_same_need".to_string(),
7306 "capabilities.requires[1]".to_string(),
7307 "credentials-provider/v1".to_string(),
7308 json!({ "requires": [
7309 { "capability": "credentials-provider/v1", "need": "required" },
7310 { "capability": "credentials-provider/v1", "need": "required" }
7311 ] }),
7312 None,
7313 ),
7314 (
7315 "duplicate_requires_conflicting_need".to_string(),
7316 "capabilities.requires[1]".to_string(),
7317 "credentials-provider/v1".to_string(),
7318 json!({ "requires": [
7319 { "capability": "credentials-provider/v1", "need": "required" },
7320 { "capability": "credentials-provider/v1", "need": "optional" }
7321 ] }),
7322 None,
7323 ),
7324 (
7325 "capabilities_root_pointer".to_string(),
7326 "runtime_computed[0]".to_string(),
7327 "/capabilities".to_string(),
7328 json!({}),
7329 Some(json!(["/capabilities"])),
7330 ),
7331 (
7332 "capabilities_descendant_pointer".to_string(),
7333 "runtime_computed[0]".to_string(),
7334 "/capabilities/provides".to_string(),
7335 json!({}),
7336 Some(json!(["/capabilities/provides"])),
7337 ),
7338 (
7339 "malformed_pointer_without_leading_slash".to_string(),
7340 "runtime_computed[0]".to_string(),
7341 "capabilities".to_string(),
7342 json!({}),
7343 Some(json!(["capabilities"])),
7344 ),
7345 (
7346 "malformed_pointer_escape".to_string(),
7347 "runtime_computed[0]".to_string(),
7348 "/roles/~2/tools".to_string(),
7349 json!({}),
7350 Some(json!(["/roles/~2/tools"])),
7351 ),
7352 (
7353 "unknown_capabilities_field".to_string(),
7354 "capabilities.future".to_string(),
7355 "<array>".to_string(),
7356 json!({ "future": [] }),
7357 None,
7358 ),
7359 ]);
7360
7361 for (index, (name, field, value, capabilities, runtime_computed)) in
7362 cases.into_iter().enumerate()
7363 {
7364 let registry = Arc::new(Registry::default());
7365 let handler = ControlHandler::new(Arc::clone(®istry));
7366 let response = handler
7367 .handle_control(
7368 ConnectionId::new((index + 1) as u64),
7369 capability_grammar_hello_frame(
7370 capabilities,
7371 runtime_computed,
7372 index as u64 + 1,
7373 ),
7374 )
7375 .expect("invalid HELLO returns a refusal");
7376
7377 assert_eq!(response.len(), 1, "{name} must emit one refusal");
7378 let error = parse_error(&response[0]);
7379 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
7380 let message = error["message"]
7381 .as_str()
7382 .expect("error message is a string");
7383 assert!(
7384 message.contains(&field),
7385 "{name}: field missing from {message}"
7386 );
7387 assert!(
7388 message.contains(&value),
7389 "{name}: value missing from {message}"
7390 );
7391 assert_eq!(
7392 registry
7393 .active_registration_count()
7394 .expect("registry reads"),
7395 0,
7396 "{name}: refused HELLO must not create a catalog entry"
7397 );
7398 }
7399 }
7400
7401 #[test]
7402 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
7403 let registry = Arc::new(Registry::default());
7404 let handler = ControlHandler::new(Arc::clone(®istry));
7405 let capabilities = json!({
7406 "provides": ["credentials-provider/v1"],
7407 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
7408 "must_never_reach": ["federation-transport/v1"]
7409 });
7410 let response = handler
7411 .handle_control(
7412 ConnectionId::new(99),
7413 capability_grammar_hello_frame(
7414 capabilities.clone(),
7415 Some(json!(["/roles/0/tools"])),
7416 99,
7417 ),
7418 )
7419 .expect("valid HELLO registers");
7420 assert_eq!(response[0].header.ty, FrameType::HelloAck);
7421
7422 let request = Frame::build(
7423 FrameType::Request,
7424 control_flags(),
7425 0,
7426 0,
7427 100,
7428 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7429 .expect("catalog request serializes"),
7430 )
7431 .expect("catalog request frame builds");
7432 let response = handler
7433 .handle_catalog_list(request, None)
7434 .expect("catalog list succeeds");
7435 let ClientControlResponse::CatalogList { modules, .. } =
7436 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
7437 else {
7438 panic!("catalog request must return catalog.list");
7439 };
7440 assert_eq!(modules.len(), 1);
7441 assert_eq!(
7442 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
7443 capabilities
7444 );
7445 }
7446
7447 #[test]
7448 fn catalog_list_mirrors_management_operation_description() {
7449 let registry = Arc::new(Registry::default());
7450 let handler = ControlHandler::new(Arc::clone(®istry));
7451 let description = "List managed records and return their identifiers and metadata.";
7452 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
7453 manifest.provides = vec![ProviderRole::ManagementSurface {
7454 operations: vec![ManagementOperation {
7455 name: "records.list".to_string(),
7456 kind: ManagementOperationKind::Query,
7457 description: Some(description.to_string()),
7458 }],
7459 config_schema: json!({"type": "object"}),
7460 observability: vec![ObservabilitySurface {
7461 name: "records.stats".to_string(),
7462 kind: ObservabilityKind::Snapshot,
7463 }],
7464 identity_scope: vec![IdentityScope::Project],
7465 concurrency: Concurrency::ModuleManaged,
7466 }];
7467 registry
7468 .register_with_control_ops(
7469 manifest,
7470 PROTOCOL_VERSION,
7471 ConnectionId::new(99),
7472 Vec::new(),
7473 )
7474 .expect("described management manifest registers");
7475
7476 let request = Frame::build(
7477 FrameType::Request,
7478 control_flags(),
7479 0,
7480 0,
7481 100,
7482 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7483 .expect("catalog request serializes"),
7484 )
7485 .expect("catalog request frame builds");
7486 let response = handler
7487 .handle_catalog_list(request, None)
7488 .expect("catalog list succeeds");
7489 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
7490 assert_eq!(
7491 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
7492 "catalog.list must preserve the declared operation description verbatim"
7493 );
7494 }
7495
7496 #[test]
7497 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
7498 let registry = Arc::new(Registry::default());
7499 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
7500 [("vault".to_string(), true), ("squatter".to_string(), true)],
7501 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
7502 );
7503 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
7504 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
7505 provides: vec!["credentials-provider/v1".to_string()],
7506 requires: Vec::new(),
7507 must_never_reach: Vec::new(),
7508 });
7509 let frame = Frame::build(
7510 FrameType::Hello,
7511 control_flags(),
7512 0,
7513 0,
7514 77,
7515 serde_json::to_vec(&ModuleHelloBody {
7516 manifest: squatter,
7517 protocol_ver: PROTOCOL_VERSION,
7518 control_ops: None,
7519 launch_nonce: None,
7520 })
7521 .expect("HELLO serializes"),
7522 )
7523 .expect("HELLO frame builds");
7524 let response = handler
7525 .handle_control(ConnectionId::new(77), frame)
7526 .expect("reserved claim receives a typed refusal");
7527 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
7528 assert_eq!(
7529 registry
7530 .active_registration_count()
7531 .expect("registry reads"),
7532 0,
7533 "a reserved capability refusal must not leave a catalog entry"
7534 );
7535 }
7536
7537 #[test]
7538 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
7539 for settlement in ["timeout", "inconclusive", "drop"] {
7540 let breakers = RouteBindBreakers::default();
7541 let RouteBindAdmission::Admitted {
7542 guard: mut old,
7543 probe: false,
7544 } = breakers.admit("prov")
7545 else {
7546 panic!("ordinary relay admitted")
7547 };
7548 let RouteBindAdmission::Admitted {
7549 guard: mut opener, ..
7550 } = breakers.admit("prov")
7551 else {
7552 panic!("second relay admitted")
7553 };
7554 assert!(
7555 !opener
7556 .record_timeout(1, Duration::ZERO)
7557 .unwrap()
7558 .reopened_after_probe
7559 );
7560 let RouteBindAdmission::Admitted {
7561 guard: mut probe,
7562 probe: true,
7563 } = breakers.admit("prov")
7564 else {
7565 panic!("one cooldown probe admitted")
7566 };
7567 match settlement {
7568 "timeout" => assert!(
7569 !old.record_timeout(1, Duration::ZERO)
7570 .unwrap()
7571 .reopened_after_probe
7572 ),
7573 "inconclusive" => old.record_inconclusive(),
7574 "drop" => drop(old),
7575 _ => unreachable!(),
7576 }
7577 assert!(
7578 matches!(
7579 breakers.admit("prov"),
7580 RouteBindAdmission::Refused {
7581 probe_in_flight: true,
7582 ..
7583 }
7584 ),
7585 "{settlement} of a pre-open relay cannot release the real probe"
7586 );
7587 assert!(
7588 probe
7589 .record_timeout(1, Duration::ZERO)
7590 .unwrap()
7591 .reopened_after_probe
7592 );
7593 assert!(matches!(
7594 breakers.admit("prov"),
7595 RouteBindAdmission::Admitted { probe: true, .. }
7596 ));
7597 }
7598 let breakers = RouteBindBreakers::default();
7599 let admit = || match breakers.admit("prov") {
7600 RouteBindAdmission::Admitted { guard, .. } => guard,
7601 _ => panic!("relay admitted"),
7602 };
7603 admit().record_timeout(1, Duration::ZERO);
7604 let mut old_probe = admit();
7605 breakers.reset_for_new_module_connection("prov");
7606 admit().record_timeout(1, Duration::ZERO);
7607 let _new_probe = admit();
7608 old_probe.record_inconclusive();
7609 assert!(matches!(
7610 breakers.admit("prov"),
7611 RouteBindAdmission::Refused {
7612 probe_in_flight: true,
7613 ..
7614 }
7615 ));
7616 }
7617
7618 #[tokio::test]
7619 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
7620 for candidate in [false, true] {
7621 let registry = Arc::new(Registry::default());
7622 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
7623 [("vault".to_string(), true), ("squatter".to_string(), true)],
7624 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
7625 );
7626 let conn = ConnectionId::new(77);
7627 let (ctx, mut rx) = route_ctx(conn);
7628 let initial = capability_manifest("squatter", &[], &[]);
7629 if candidate {
7630 registry
7631 .register_candidate_with_control_ops(
7632 initial.clone(),
7633 PROTOCOL_VERSION,
7634 conn,
7635 module_baseline_control_ops(),
7636 )
7637 .unwrap();
7638 handler
7639 .forwarding
7640 .register_candidate_module_connection(
7641 conn,
7642 "squatter".to_string(),
7643 PROTOCOL_VERSION,
7644 manifest_concurrency(&initial),
7645 ctx.egress.clone(),
7646 )
7647 .unwrap();
7648 } else {
7649 hello_via_sink(
7650 &handler,
7651 &ctx,
7652 &mut rx,
7653 hello_frame_with_manifest(initial.clone(), 1),
7654 )
7655 .await;
7656 }
7657 let response = handler
7658 .handle_control_frame(
7659 &ctx,
7660 catalog_update_with_capabilities_frame(
7661 2,
7662 capability_manifest("squatter", &["credentials-provider/v1"], &[])
7663 .capabilities
7664 .unwrap(),
7665 ),
7666 )
7667 .await
7668 .unwrap();
7669 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
7670 assert_eq!(
7671 registry
7672 .get_module_by_connection(conn)
7673 .unwrap()
7674 .unwrap()
7675 .manifest,
7676 initial
7677 );
7678 }
7679 }
7680
7681 #[test]
7682 fn server_describe_surfaces_required_capability_verdict_fields() {
7683 let registry = Arc::new(Registry::default());
7684 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
7685 [
7686 ("consumer".to_string(), true),
7687 ("provider".to_string(), false),
7688 ],
7689 BTreeMap::new(),
7690 );
7691 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
7692 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
7693 provides: Vec::new(),
7694 requires: vec![subc_protocol::manifest::CapabilityRequirement {
7695 capability: "credentials-provider/v1".to_string(),
7696 need: subc_protocol::manifest::CapabilityNeed::Required,
7697 }],
7698 must_never_reach: Vec::new(),
7699 });
7700 let hello = Frame::build(
7701 FrameType::Hello,
7702 control_flags(),
7703 0,
7704 0,
7705 78,
7706 serde_json::to_vec(&ModuleHelloBody {
7707 manifest: consumer,
7708 protocol_ver: PROTOCOL_VERSION,
7709 control_ops: None,
7710 launch_nonce: None,
7711 })
7712 .expect("HELLO serializes"),
7713 )
7714 .expect("HELLO frame builds");
7715 handler
7716 .handle_control(ConnectionId::new(78), hello)
7717 .expect("consumer registers");
7718 let describe = Frame::build(
7719 FrameType::Request,
7720 control_flags(),
7721 0,
7722 0,
7723 79,
7724 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
7725 .expect("request serializes"),
7726 )
7727 .expect("describe frame builds");
7728 let response = handler
7729 .handle_server_describe(describe)
7730 .expect("server.describe succeeds");
7731 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
7732 let requirement = &rendered["capability_requirements"][0];
7733 assert_eq!(requirement["consumer"], "consumer");
7734 assert_eq!(requirement["verdict"], "never_provided");
7735 assert_eq!(requirement["episode_seq"], 1);
7736 assert_eq!(requirement["config_satisfiable"], false);
7737 assert_eq!(requirement["runtime_available"], false);
7738 assert!(requirement["detail"]
7739 .as_str()
7740 .expect("detail string")
7741 .contains("credentials-provider/v1"));
7742 }
7743
7744 #[test]
7745 fn catalog_list_omits_capabilities_for_legacy_manifest() {
7746 let registry = Arc::new(Registry::default());
7747 let handler = ControlHandler::new(Arc::clone(®istry));
7748 let hello = handler
7749 .handle_control(
7750 ConnectionId::new(101),
7751 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
7752 )
7753 .expect("legacy HELLO registers");
7754 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
7755
7756 let request = Frame::build(
7757 FrameType::Request,
7758 control_flags(),
7759 0,
7760 0,
7761 102,
7762 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7763 .expect("catalog request serializes"),
7764 )
7765 .expect("catalog request frame builds");
7766 let response = handler
7767 .handle_catalog_list(request, None)
7768 .expect("catalog list succeeds");
7769 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
7770 assert!(
7771 body["modules"][0].get("capabilities").is_none(),
7772 "legacy manifest must retain an absent capabilities field on catalog.list"
7773 );
7774 }
7775
7776 #[test]
7777 fn hello_ack_omits_storage_when_no_storage_config() {
7778 let registry = Arc::new(Registry::default());
7779 let handler = ControlHandler::new(Arc::clone(®istry));
7780 let responses = handler
7781 .handle_control(
7782 ConnectionId::new(1),
7783 hello_frame("aft", PROTOCOL_VERSION, 7),
7784 )
7785 .unwrap();
7786 let ack = parse_ack(&responses[0]);
7787 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
7788 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
7789 }
7790
7791 #[tokio::test]
7792 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
7793 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
7794 let registry = Arc::new(Registry::default());
7795 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
7796 let responses = handler
7797 .handle_control(
7798 ConnectionId::new(1),
7799 hello_frame("aft", PROTOCOL_VERSION, 7),
7800 )
7801 .unwrap();
7802 let ack = parse_ack(&responses[0]);
7803 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
7804
7805 let described = handler
7806 .handle_control_frame(
7807 &route_ctx(ConnectionId::new(2)).0,
7808 Frame::build(
7809 FrameType::Request,
7810 control_flags(),
7811 0,
7812 0,
7813 9,
7814 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
7815 )
7816 .unwrap(),
7817 )
7818 .await
7819 .unwrap();
7820 let ClientControlResponse::ServerDescribe { machine_id, .. } =
7821 serde_json::from_slice(&described[0].body).unwrap()
7822 else {
7823 panic!("server.describe answered with another shape");
7824 };
7825 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
7826 }
7827
7828 #[test]
7829 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
7830 let registry = Arc::new(Registry::default());
7833 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
7834 crate::daemon_config::StorageConfig::Sqlite {
7835 data_home: std::path::PathBuf::from("/data"),
7836 },
7837 ));
7838
7839 let responses = handler
7840 .handle_control(
7841 ConnectionId::new(1),
7842 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
7843 )
7844 .unwrap();
7845 let ack = parse_ack(&responses[0]);
7846 assert_eq!(
7847 ack.storage,
7848 Some(serde_json::json!({
7849 "module_id": "alfonso-routing",
7850 "storage_namespace": "default",
7851 "isolation": { "kind": "module" },
7852 "backend": {
7853 "backend": "sqlite",
7854 "path": "/data/cortexkit/alfonso-routing/store.db"
7855 }
7856 })),
7857 "the delivered descriptor is the module's own sqlite store path"
7858 );
7859 }
7860
7861 #[test]
7862 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
7863 let registry = Arc::new(Registry::default());
7864 let handler = ControlHandler::new(Arc::clone(®istry));
7865 let conn = ConnectionId::new(1);
7866 let responses = handler
7867 .handle_control(
7868 conn,
7869 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
7870 )
7871 .unwrap();
7872 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
7873 let registration = registry.get_module("aft").unwrap().unwrap();
7874 assert_eq!(registration.control_ops, module_baseline_control_ops());
7875
7876 let frame =
7877 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
7878 assert!(handler
7879 .guard_module_control_op(&frame, "aft", "route.bind")
7880 .unwrap()
7881 .is_none());
7882 let error = handler
7883 .guard_module_control_op(&frame, "aft", "test.synthetic")
7884 .unwrap()
7885 .expect("synthetic ungranted op should be rejected");
7886 assert_eq!(error.header.ty, FrameType::Error);
7887 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
7888 }
7889
7890 #[test]
7891 fn hello_control_ops_some_adds_optional_grants() {
7892 let registry = Arc::new(Registry::default());
7893 let handler = ControlHandler::new(Arc::clone(®istry));
7894 handler
7895 .handle_control(
7896 ConnectionId::new(1),
7897 hello_frame_with_control_ops(
7898 "aft",
7899 PROTOCOL_VERSION,
7900 7,
7901 Some(vec![
7902 "future.synthetic".to_string(),
7903 "route.bind".to_string(),
7904 ]),
7905 ),
7906 )
7907 .unwrap();
7908 let registration = registry.get_module("aft").unwrap().unwrap();
7909 assert_eq!(
7910 registration.control_ops,
7911 vec![
7912 "route.bind".to_string(),
7913 "route.status".to_string(),
7914 "future.synthetic".to_string(),
7915 ]
7916 );
7917 let frame =
7918 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
7919 assert!(handler
7920 .guard_module_control_op(&frame, "aft", "future.synthetic")
7921 .unwrap()
7922 .is_none());
7923 }
7924
7925 #[tokio::test]
7926 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
7927 let registry = Arc::new(Registry::default());
7928 let forwarding = Arc::new(ForwardingTable::default());
7929 let handler =
7930 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
7931 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
7932 hello_via_sink(
7933 &handler,
7934 &module_ctx,
7935 &mut module_rx,
7936 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
7937 )
7938 .await;
7939
7940 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
7941 let responses = handler
7942 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
7943 .await
7944 .unwrap();
7945 assert_eq!(responses.len(), 1);
7946 assert_eq!(responses[0].header.ty, FrameType::Error);
7947 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
7948 assert!(module_rx.try_recv().is_err());
7949 }
7950
7951 #[tokio::test]
7952 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
7953 let registry = Arc::new(Registry::default());
7954 let forwarding = Arc::new(ForwardingTable::default());
7955 let handler =
7956 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
7957 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
7958 hello_via_sink(
7959 &handler,
7960 &module_ctx,
7961 &mut module_rx,
7962 hello_frame_with_control_ops(
7963 "aft",
7964 PROTOCOL_VERSION,
7965 7,
7966 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
7967 ),
7968 )
7969 .await;
7970
7971 let project_root = unique_project_root("demux");
7972 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
7973 let route_handler = handler.clone();
7974 let route_task = tokio::spawn(async move {
7975 route_handler
7976 .handle_control_frame(
7977 &route_client_ctx,
7978 route_open_frame(100, "aft", project_root),
7979 )
7980 .await
7981 .unwrap()
7982 });
7983 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7984 .await
7985 .unwrap()
7986 .unwrap();
7987 assert!(matches!(
7988 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
7989 ModuleControlRequest::RouteBind { .. }
7990 ));
7991
7992 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
7993 let health_handler = handler.clone();
7994 let health_task = tokio::spawn(async move {
7995 health_handler
7996 .handle_control_frame(
7997 &health_client_ctx,
7998 supervisor_health_probe_frame(101, "aft"),
7999 )
8000 .await
8001 .unwrap()
8002 });
8003 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8004 .await
8005 .unwrap()
8006 .unwrap();
8007 assert_eq!(
8008 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
8009 ModuleControlRequest::HealthCheck {}
8010 );
8011
8012 handler
8013 .handle_control_frame(
8014 &module_ctx,
8015 health_response(health_frame.header.corr, HealthStatus::Degraded),
8016 )
8017 .await
8018 .unwrap();
8019 let health_response = health_task.await.unwrap();
8020 assert_eq!(health_response.len(), 1);
8021 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
8022 ClientControlResponse::SupervisorHealthProbe {
8023 module_id,
8024 status,
8025 detail,
8026 metrics,
8027 } => {
8028 assert_eq!(module_id, "aft");
8029 assert_eq!(status, HealthStatus::Degraded);
8030 assert_eq!(detail.as_deref(), Some("warming"));
8031 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
8032 }
8033 other => panic!("unexpected health response: {other:?}"),
8034 }
8035
8036 handler
8037 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8038 .await
8039 .unwrap();
8040 let route_response = route_task.await.unwrap();
8041 assert!(route_response.is_empty());
8042 let published = route_client_rx.recv().await.unwrap();
8043 assert!(matches!(
8044 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8045 ClientControlResponse::RouteOpen { .. }
8046 ));
8047 }
8048
8049 async fn relay_route_open(
8054 handler: &ControlHandler,
8055 client_connection: ConnectionId,
8056 client_egress: &FrameSink,
8057 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
8058 corr: u64,
8059 module_id: &str,
8060 project_root_label: &str,
8061 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
8062 let ctx = RouteCtx {
8063 connection_id: client_connection,
8064 egress: client_egress.clone(),
8065 };
8066 let handler = handler.clone();
8067 let project_root = unique_project_root(project_root_label);
8068 let module_id = module_id.to_string();
8069 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
8070 let task = tokio::spawn(async move {
8071 let _guard = tracing::dispatcher::set_default(&dispatch);
8072 handler
8073 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
8074 .await
8075 .unwrap()
8076 });
8077 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
8078 .await
8079 .expect("module receives the relayed route.bind")
8080 .expect("module egress is open");
8081 (task, bind.frame)
8082 }
8083
8084 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
8085 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
8086 ModuleControlRequest::RouteBind {
8087 route_channel,
8088 epoch,
8089 ..
8090 } => (route_channel, epoch),
8091 other => panic!("expected a route.bind request, got {other:?}"),
8092 }
8093 }
8094
8095 fn published_route(frame: &Frame) -> (u16, u32) {
8096 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
8097 ClientControlResponse::RouteOpen {
8098 route_channel,
8099 route_epoch,
8100 } => (route_channel, route_epoch),
8101 other => panic!("expected a route.open response, got {other:?}"),
8102 }
8103 }
8104
8105 #[tokio::test]
8125 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
8126 let registry = Arc::new(Registry::default());
8127 let forwarding = Arc::new(ForwardingTable::default());
8128 let handler =
8129 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8130
8131 let module_connection = ConnectionId::new(30);
8132 let (module_ctx, mut module_rx) = route_ctx(module_connection);
8133 hello_via_sink(
8134 &handler,
8135 &module_ctx,
8136 &mut module_rx,
8137 hello_frame("aft", PROTOCOL_VERSION, 7),
8138 )
8139 .await;
8140
8141 let dying_client = ConnectionId::new(31);
8142 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
8143
8144 let (first_task, first_bind) = relay_route_open(
8147 &handler,
8148 dying_client,
8149 &dying_ctx.egress,
8150 &mut module_rx,
8151 100,
8152 "aft",
8153 "closing-first",
8154 )
8155 .await;
8156 handler
8157 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
8158 .await
8159 .unwrap();
8160 assert!(first_task.await.unwrap().is_empty());
8161 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
8162
8163 let (second_task, second_bind) = relay_route_open(
8165 &handler,
8166 dying_client,
8167 &dying_ctx.egress,
8168 &mut module_rx,
8169 101,
8170 "aft",
8171 "closing-second",
8172 )
8173 .await;
8174 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
8175
8176 assert!(forwarding
8179 .escalate_client_delivery_failure(
8180 dying_client,
8181 first_channel,
8182 first_epoch,
8183 CloseReason::new(
8184 "module_to_client_delivery_failed",
8185 "client egress refused a module frame",
8186 ),
8187 crate::forwarding::UndeliveredFrame {
8188 module_id: None,
8189 sink: &dying_ctx.egress,
8190 },
8191 )
8192 .unwrap());
8193 assert!(!dying_ctx.egress.is_closed());
8194
8195 let ack = handler
8197 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
8198 .await;
8199 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
8200 if module_loop_error.is_some() {
8201 handler.cleanup_connection(module_connection).unwrap();
8205 }
8206 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8210 .await
8211 .ok()
8212 .flatten();
8213
8214 assert!(
8216 registry
8217 .get_module_by_connection(module_connection)
8218 .unwrap()
8219 .is_some(),
8220 "one client's closing connection ended the shared module connection: \
8221 {module_loop_error:?}"
8222 );
8223 let cotenant = ConnectionId::new(32);
8225 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
8226 let (cotenant_task, cotenant_bind) = relay_route_open(
8227 &handler,
8228 cotenant,
8229 &cotenant_ctx.egress,
8230 &mut module_rx,
8231 102,
8232 "aft",
8233 "closing-cotenant",
8234 )
8235 .await;
8236 handler
8237 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
8238 .await
8239 .unwrap();
8240 assert!(cotenant_task.await.unwrap().is_empty());
8241 let (cotenant_channel, cotenant_epoch) =
8242 published_route(&cotenant_rx.recv().await.unwrap());
8243 assert!(matches!(
8244 forwarding
8245 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
8246 .unwrap(),
8247 DataRoute::Client(DataRouteState::Bound(_))
8248 ));
8249
8250 let goodbye = post_ack_module_frame
8253 .expect("module receives a GOODBYE for the abandoned route channel");
8254 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
8255 assert_eq!(goodbye.header.channel, abandoned_channel);
8256 assert_eq!(goodbye.header.epoch, abandoned_epoch);
8257
8258 assert!(dying_rx.try_recv().is_err());
8262 let second_response = second_task.await.unwrap();
8263 assert_eq!(second_response.len(), 1);
8264 assert_eq!(
8265 parse_error(&second_response[0])["code"],
8266 "target_unavailable"
8267 );
8268 }
8269
8270 async fn route_open_held_after_bind_send() -> (
8274 ControlHandler,
8275 Arc<tokio::sync::Barrier>,
8276 RouteCtx,
8277 mpsc::Receiver<crate::router::OutboundFrame>,
8278 ) {
8279 let hold = Arc::new(tokio::sync::Barrier::new(2));
8280 let handler = ControlHandler::with_forwarding(
8281 Arc::new(Registry::default()),
8282 Arc::new(ForwardingTable::default()),
8283 )
8284 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
8285 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
8286 hello_via_sink(
8287 &handler,
8288 &module_ctx,
8289 &mut module_rx,
8290 hello_frame("aft", PROTOCOL_VERSION, 7),
8291 )
8292 .await;
8293 (handler, hold, module_ctx, module_rx)
8294 }
8295
8296 #[tokio::test]
8303 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
8304 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
8305 let client = ConnectionId::new(41);
8306 let (client_ctx, mut client_rx) = route_ctx(client);
8307
8308 let (open_task, bind) = relay_route_open(
8309 &handler,
8310 client,
8311 &client_ctx.egress,
8312 &mut module_rx,
8313 200,
8314 "aft",
8315 "acked-before-mark",
8316 )
8317 .await;
8318 let (module_channel, module_epoch) = route_bind_channel(&bind);
8319 hold.wait().await;
8321
8322 assert!(handler
8324 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
8325 .await
8326 .unwrap()
8327 .is_empty());
8328 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
8329
8330 hold.wait().await;
8331 assert!(
8332 open_task.await.unwrap().is_empty(),
8333 "route.open answers accepted"
8334 );
8335
8336 match module_rx.try_recv() {
8339 Err(mpsc::error::TryRecvError::Empty) => {}
8340 Ok(outbound) => panic!(
8341 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
8342 outbound.frame.header.ty,
8343 outbound.frame.header.channel,
8344 outbound.frame.header.epoch
8345 ),
8346 Err(err) => panic!("module egress closed: {err}"),
8347 }
8348
8349 let router = Router::with_control_handler(Arc::new(handler.clone()));
8352 let request = Frame::build(
8353 FrameType::Request,
8354 Flags::new(true, Priority::Interactive, false),
8355 client_channel,
8356 client_epoch,
8357 201,
8358 b"first request".to_vec(),
8359 )
8360 .unwrap();
8361 router
8362 .route_for_connection(&client_ctx, request)
8363 .await
8364 .unwrap();
8365 let delivered = module_rx.try_recv().unwrap().frame;
8366 assert_eq!(delivered.header.ty, FrameType::Request);
8367 assert_eq!(
8368 (delivered.header.channel, delivered.header.epoch),
8369 (module_channel, module_epoch)
8370 );
8371 assert_eq!(delivered.body, b"first request");
8372 }
8373
8374 #[tokio::test]
8380 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
8381 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
8382 let client = ConnectionId::new(42);
8383 let (client_ctx, _client_rx) = route_ctx(client);
8384
8385 let (open_task, bind) = relay_route_open(
8386 &handler,
8387 client,
8388 &client_ctx.egress,
8389 &mut module_rx,
8390 300,
8391 "aft",
8392 "released-before-mark",
8393 )
8394 .await;
8395 let (module_channel, module_epoch) = route_bind_channel(&bind);
8396 hold.wait().await;
8397
8398 handler.cleanup_connection(client).unwrap();
8399 assert!(
8400 module_rx.try_recv().is_err(),
8401 "client teardown must not GOODBYE a bind route.open is still queueing"
8402 );
8403
8404 hold.wait().await;
8405 let response = open_task.await.unwrap();
8406 assert_eq!(response.len(), 1);
8407 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
8408
8409 let goodbye = module_rx
8410 .try_recv()
8411 .expect("module is told to drop the binding the daemon abandoned")
8412 .frame;
8413 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
8414 assert_eq!(
8415 (goodbye.header.channel, goodbye.header.epoch),
8416 (module_channel, module_epoch)
8417 );
8418 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
8419 }
8420
8421 #[tokio::test]
8426 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
8427 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
8428 let client = ConnectionId::new(43);
8429 let (client_ctx, client_rx) = route_ctx(client);
8430
8431 let (open_task, bind) = relay_route_open(
8432 &handler,
8433 client,
8434 &client_ctx.egress,
8435 &mut module_rx,
8436 400,
8437 "aft",
8438 "gone-client-before-mark",
8439 )
8440 .await;
8441 let (module_channel, module_epoch) = route_bind_channel(&bind);
8442 hold.wait().await;
8443
8444 drop(client_rx);
8445 assert!(client_ctx.egress.is_closed());
8446 handler
8447 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
8448 .await
8449 .unwrap();
8450
8451 hold.wait().await;
8452 let response = open_task.await.unwrap();
8453 assert_eq!(response.len(), 1);
8454 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
8455
8456 let goodbye = module_rx
8457 .try_recv()
8458 .expect("module is told to drop the binding it acked")
8459 .frame;
8460 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
8461 assert_eq!(
8462 (goodbye.header.channel, goodbye.header.epoch),
8463 (module_channel, module_epoch)
8464 );
8465 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
8466 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
8467 }
8468
8469 #[test]
8476 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
8477 let handler = ControlHandler::default();
8478 let module_connection = ConnectionId::new(30);
8479 let client_connection = ConnectionId::new(31);
8480
8481 handler
8482 .refuse_to_end_module_connection_for_a_client(
8483 module_connection,
8484 77,
8485 ForwardingError::ConnectionClosing {
8486 connection_id: client_connection,
8487 },
8488 )
8489 .expect("a closing client must never end the module connection");
8490
8491 assert!(matches!(
8492 handler.refuse_to_end_module_connection_for_a_client(
8493 module_connection,
8494 78,
8495 ForwardingError::ConnectionClosing {
8496 connection_id: module_connection,
8497 },
8498 ),
8499 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
8500 connection_id
8501 })) if connection_id == module_connection
8502 ));
8503 assert!(matches!(
8504 handler.refuse_to_end_module_connection_for_a_client(
8505 module_connection,
8506 79,
8507 ForwardingError::Poisoned,
8508 ),
8509 Err(RouterError::Forwarding(ForwardingError::Poisoned))
8510 ));
8511 assert!(matches!(
8512 handler.refuse_to_end_module_connection_for_a_client(
8513 module_connection,
8514 80,
8515 ForwardingError::StaleModuleEndpoint,
8516 ),
8517 Err(RouterError::Forwarding(
8518 ForwardingError::StaleModuleEndpoint
8519 ))
8520 ));
8521 }
8522
8523 #[tokio::test]
8536 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
8537 let registry = Arc::new(Registry::default());
8538 let forwarding = Arc::new(ForwardingTable::default());
8539 let supervisor = SupervisorHandle::new();
8540 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8541 let handler =
8542 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8543 .with_supervisor(supervisor);
8544
8545 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
8546 hello_via_sink(
8547 &handler,
8548 &target_ctx,
8549 &mut target_rx,
8550 hello_frame("target", PROTOCOL_VERSION, 1),
8551 )
8552 .await;
8553
8554 let wrong_nonce = handler
8558 .handle_control_frame(
8559 &route_ctx(ConnectionId::new(91)).0,
8560 route_open_frame_with_admission_facts(
8561 20,
8562 "target",
8563 unique_project_root("admission-facts"),
8564 Some(subc_control::ConsumerIdentity {
8565 module_id: "fed".to_string(),
8566 launch_nonce: "not-the-real-nonce".to_string(),
8567 }),
8568 None,
8569 ),
8570 )
8571 .await
8572 .unwrap();
8573 assert_eq!(
8574 parse_error(&wrong_nonce[0])["code"],
8575 "bad_consumer_identity",
8576 "a mismatched launch nonce must be refused, not stamped Reserved"
8577 );
8578
8579 let never_spawned = handler
8583 .handle_control_frame(
8584 &route_ctx(ConnectionId::new(92)).0,
8585 route_open_frame_with_admission_facts(
8586 21,
8587 "target",
8588 unique_project_root("admission-facts"),
8589 Some(subc_control::ConsumerIdentity {
8590 module_id: "never-spawned".to_string(),
8591 launch_nonce: "any-nonce".to_string(),
8592 }),
8593 None,
8594 ),
8595 )
8596 .await
8597 .unwrap();
8598 assert_eq!(
8599 parse_error(&never_spawned[0])["code"],
8600 "bad_consumer_identity",
8601 "an unspawned module_id must be refused rather than accepted for lack of a record"
8602 );
8603 }
8604
8605 #[tokio::test]
8628 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
8629 let registry = Arc::new(Registry::default());
8630 let forwarding = Arc::new(ForwardingTable::default());
8631 let supervisor = SupervisorHandle::new();
8632 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8633 let handler =
8634 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8635 .with_supervisor(supervisor);
8636
8637 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
8638 hello_via_sink(
8639 &handler,
8640 &target_ctx,
8641 &mut target_rx,
8642 hello_frame("target", PROTOCOL_VERSION, 1),
8643 )
8644 .await;
8645
8646 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
8647 let route_handler = handler.clone();
8648 let route_task = tokio::spawn(async move {
8649 route_handler
8650 .handle_control_frame(
8651 &client_ctx,
8652 route_open_frame_with_admission_facts(
8653 30,
8654 "target",
8655 unique_project_root("admission-facts"),
8656 Some(subc_control::ConsumerIdentity {
8657 module_id: "fed".to_string(),
8658 launch_nonce: "fed-nonce".to_string(),
8659 }),
8660 None,
8661 ),
8662 )
8663 .await
8664 .unwrap()
8665 });
8666
8667 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8674 .await
8675 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
8676 .expect("module control channel closed before route.bind");
8677 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8678 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
8679 panic!("expected route.bind")
8680 };
8681 assert_eq!(
8682 principal,
8683 Some(Principal::Reserved {
8684 module_id: "fed".to_string()
8685 }),
8686 "a correctly attested consumer must be stamped Reserved for its own id"
8687 );
8688
8689 handler
8690 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8691 .await
8692 .unwrap();
8693 assert!(route_task.await.unwrap().is_empty());
8694 assert!(
8695 matches!(
8696 serde_json::from_slice::<ClientControlResponse>(
8697 &client_rx.recv().await.unwrap().body
8698 )
8699 .unwrap(),
8700 ClientControlResponse::RouteOpen { .. }
8701 ),
8702 "the route must actually open, not merely avoid an error"
8703 );
8704 }
8705
8706 #[tokio::test(start_paused = true)]
8707 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
8708 let registry = Arc::new(Registry::default());
8709 let forwarding = Arc::new(ForwardingTable::default());
8710 let supervisor = SupervisorHandle::new();
8711 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8712 let handler =
8713 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8714 .with_supervisor(supervisor);
8715
8716 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
8717 hello_via_sink(
8718 &handler,
8719 &target_ctx,
8720 &mut target_rx,
8721 hello_frame("target", PROTOCOL_VERSION, 1),
8722 )
8723 .await;
8724
8725 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
8726 let direct_handler = handler.clone();
8727 let direct_open = tokio::spawn(async move {
8728 direct_handler
8729 .handle_control_frame(
8730 &direct_ctx,
8731 route_open_frame(2, "target", unique_project_root("route-census-direct")),
8732 )
8733 .await
8734 .unwrap()
8735 });
8736 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8737 .await
8738 .expect("no direct route.bind within 5s")
8739 .expect("target control channel closed before direct route.bind");
8740 handler
8741 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
8742 .await
8743 .unwrap();
8744 assert!(direct_open.await.unwrap().is_empty());
8745 let _ = direct_rx.recv().await.unwrap();
8746
8747 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
8748 let reserved_handler = handler.clone();
8749 let reserved_open = tokio::spawn(async move {
8750 reserved_handler
8751 .handle_control_frame(
8752 &reserved_ctx,
8753 route_open_frame_with_admission_facts(
8754 3,
8755 "target",
8756 unique_project_root("admission-facts"),
8757 Some(ConsumerIdentity {
8758 module_id: "fed".to_string(),
8759 launch_nonce: "fed-nonce".to_string(),
8760 }),
8761 None,
8762 ),
8763 )
8764 .await
8765 .unwrap()
8766 });
8767 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8768 .await
8769 .expect("no reserved route.bind within 5s")
8770 .expect("target control channel closed before reserved route.bind");
8771 handler
8772 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
8773 .await
8774 .unwrap();
8775 assert!(reserved_open.await.unwrap().is_empty());
8776 let _ = reserved_rx.recv().await.unwrap();
8777
8778 forwarding
8779 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
8780 .unwrap();
8781 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
8782 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
8783 module_id: Some("target".to_string()),
8784 })
8785 .unwrap();
8786 let census_frame =
8787 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
8788 let response = handler
8789 .handle_control_frame(&census_ctx, census_frame)
8790 .await
8791 .unwrap()
8792 .pop()
8793 .unwrap();
8794 let actual: Value = serde_json::from_slice(&response.body).unwrap();
8795 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
8796 assert!(matches!(
8797 decoded,
8798 ClientControlResponse::SupervisorRoutes { .. }
8799 ));
8800 let routes = actual["modules"][0]["routes"].as_array().unwrap();
8801 assert_eq!(routes.len(), 2);
8802 assert!(routes.iter().all(|route| route["draining"] == true));
8803 assert!(
8806 routes.iter().all(|route| route["drain_reason"] == "reload"),
8807 "draining routes must name the drain's reason: {routes:?}"
8808 );
8809 assert!(routes.iter().any(|route| {
8810 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
8811 }));
8812 assert!(routes.iter().any(|route| {
8813 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
8814 }));
8815
8816 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8817 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
8818 if std::env::var_os("UPDATE_GOLDEN").is_some() {
8819 std::fs::write(
8820 &golden_path,
8821 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
8822 )
8823 .unwrap();
8824 }
8825 let expected: Value =
8826 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
8827 assert_eq!(actual, expected);
8828 }
8829
8830 async fn query_live_roots(
8831 handler: &ControlHandler,
8832 module_ctx: &RouteCtx,
8833 ) -> ModuleControlResponseToModule {
8834 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
8835 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
8836 let response = handler
8837 .handle_control_frame(module_ctx, frame)
8838 .await
8839 .unwrap()
8840 .pop()
8841 .unwrap();
8842 serde_json::from_slice(&response.body).unwrap()
8843 }
8844
8845 #[tokio::test(start_paused = true)]
8846 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
8847 let registry = Arc::new(Registry::default());
8848 let forwarding = Arc::new(ForwardingTable::default());
8849 let handler = ControlHandler::with_forwarding(registry, forwarding);
8850 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
8851 hello_via_sink(
8852 &handler,
8853 &target_ctx,
8854 &mut target_rx,
8855 hello_frame("target", PROTOCOL_VERSION, 1),
8856 )
8857 .await;
8858 let root = unique_project_root("live-roots-known");
8859 let path = ProjectRootId::from_path_allowing_missing(root.path())
8860 .unwrap()
8861 .as_path()
8862 .to_path_buf();
8863 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
8864 let open_handler = handler.clone();
8865 let opened = tokio::spawn(async move {
8866 open_handler
8867 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
8868 .await
8869 .unwrap()
8870 });
8871 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8872 .await
8873 .unwrap()
8874 .unwrap();
8875 handler
8876 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
8877 .await
8878 .unwrap();
8879 assert!(opened.await.unwrap().is_empty());
8880 let _ = client_rx.recv().await.unwrap();
8881
8882 let root = unique_project_root("live-roots-pending");
8883 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
8884 .unwrap()
8885 .as_path()
8886 .to_path_buf();
8887 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
8888 let open_handler = handler.clone();
8889 let pending = tokio::spawn(async move {
8890 open_handler
8891 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
8892 .await
8893 .unwrap()
8894 });
8895 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8896 .await
8897 .unwrap()
8898 .unwrap();
8899 let actual = query_live_roots(&handler, &target_ctx).await;
8900 let ModuleControlResponseToModule::LiveRoots {
8901 roots,
8902 unknown_root_bindings,
8903 total_bindings,
8904 } = actual
8905 else {
8906 panic!("expected live roots")
8907 };
8908 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
8909 assert_eq!(unknown_root_bindings, 0);
8910 assert_eq!(
8911 roots.len(),
8912 2,
8913 "root-known arm must retain each canonical root"
8914 );
8915 assert_eq!(
8916 total_bindings,
8917 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
8918 );
8919 let counts = roots
8920 .iter()
8921 .map(|root| (root.project_root.clone(), root.bound, root.pending))
8922 .collect::<Vec<_>>();
8923 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
8924 expected.sort_by(|a, b| a.0.cmp(&b.0));
8925 assert_eq!(
8926 counts, expected,
8927 "roots must sort by path and count pending separately"
8928 );
8929 handler
8930 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
8931 .await
8932 .unwrap();
8933 assert!(pending.await.unwrap().is_empty());
8934 }
8935
8936 #[tokio::test(start_paused = true)]
8937 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
8938 let forwarding = Arc::new(ForwardingTable::default());
8939 let handler =
8940 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
8941 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
8942 hello_via_sink(
8943 &handler,
8944 &target_ctx,
8945 &mut target_rx,
8946 hello_frame("target", PROTOCOL_VERSION, 1),
8947 )
8948 .await;
8949 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
8950 let pending = forwarding
8951 .begin_route_bind_relay_for_test(
8952 client_ctx.connection_id,
8953 client_ctx.egress.clone(),
8954 2,
8955 "target",
8956 )
8957 .unwrap();
8958 forwarding
8959 .complete_pending_relay(
8960 target_ctx.connection_id,
8961 pending.corr,
8962 RouteBindRelayOutcome::Accepted,
8963 )
8964 .unwrap();
8965 let actual = query_live_roots(&handler, &target_ctx).await;
8966 let ModuleControlResponseToModule::LiveRoots {
8967 roots,
8968 unknown_root_bindings,
8969 total_bindings,
8970 } = actual
8971 else {
8972 panic!("expected live roots")
8973 };
8974 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
8975 assert_eq!(
8976 unknown_root_bindings, 1,
8977 "unknown-root arm must not read as no bindings"
8978 );
8979 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
8980 assert_eq!(
8981 total_bindings,
8982 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
8983 );
8984 }
8985
8986 #[tokio::test(start_paused = true)]
8994 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
8995 let forwarding = Arc::new(ForwardingTable::default());
8996 let handler =
8997 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
8998 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
8999 let replies = handler
9000 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
9001 .await
9002 .unwrap();
9003 let queued_by_hello = module_rx.len();
9004
9005 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
9006 let open_handler = handler.clone();
9007 let open = tokio::spawn(async move {
9008 open_handler
9009 .handle_control_frame(
9010 &client_ctx,
9011 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
9012 )
9013 .await
9014 .unwrap()
9015 });
9016 let mut spins = 0;
9018 while module_rx.len() == queued_by_hello {
9019 spins += 1;
9020 assert!(spins < 10_000, "route.open never queued a route.bind");
9021 tokio::task::yield_now().await;
9022 }
9023
9024 for reply in replies {
9026 module_ctx.egress.send(reply).await.unwrap();
9027 }
9028
9029 let first = module_rx.recv().await.unwrap().frame;
9030 assert_eq!(
9031 first.header.ty,
9032 FrameType::HelloAck,
9033 "the first frame a registering module reads must be its HELLO_ACK"
9034 );
9035 assert_eq!(first.header.corr, 7);
9036 let second = module_rx.recv().await.unwrap().frame;
9037 assert_eq!(second.header.ty, FrameType::Request);
9038 assert!(
9039 matches!(
9040 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
9041 ModuleControlRequest::RouteBind { .. }
9042 ),
9043 "the route.bind follows the ack"
9044 );
9045 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
9046
9047 handler
9048 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
9049 .await
9050 .unwrap();
9051 assert!(open.await.unwrap().is_empty());
9052 let _ = client_rx.recv().await.unwrap();
9053 }
9054
9055 #[tokio::test(start_paused = true)]
9056 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
9057 let handler = ControlHandler::with_forwarding(
9058 Arc::new(Registry::default()),
9059 Arc::new(ForwardingTable::default()),
9060 );
9061 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
9062 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
9063 hello_via_sink(
9064 &handler,
9065 &first_ctx,
9066 &mut first_rx,
9067 hello_frame("first", PROTOCOL_VERSION, 1),
9068 )
9069 .await;
9070 hello_via_sink(
9071 &handler,
9072 &second_ctx,
9073 &mut second_rx,
9074 hello_frame("second", PROTOCOL_VERSION, 2),
9075 )
9076 .await;
9077 let root = unique_project_root("second-only");
9078 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
9079 let cloned = handler.clone();
9080 let open = tokio::spawn(async move {
9081 cloned
9082 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
9083 .await
9084 .unwrap()
9085 });
9086 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
9087 .await
9088 .unwrap()
9089 .unwrap();
9090 let first = query_live_roots(&handler, &first_ctx).await;
9091 let second = query_live_roots(&handler, &second_ctx).await;
9092 assert!(
9093 matches!(
9094 first,
9095 ModuleControlResponseToModule::LiveRoots {
9096 total_bindings: 0,
9097 ..
9098 }
9099 ),
9100 "cross-module scope must not expose another module's roots"
9101 );
9102 assert!(
9103 matches!(
9104 second,
9105 ModuleControlResponseToModule::LiveRoots {
9106 total_bindings: 1,
9107 ..
9108 }
9109 ),
9110 "second module must see its pending route"
9111 );
9112 handler
9113 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
9114 .await
9115 .unwrap();
9116 assert!(open.await.unwrap().is_empty());
9117 }
9118
9119 #[tokio::test(start_paused = true)]
9120 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
9121 let handler = ControlHandler::with_forwarding(
9122 Arc::new(Registry::default()),
9123 Arc::new(ForwardingTable::default()),
9124 );
9125 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
9126 hello_via_sink(
9127 &handler,
9128 &target_ctx,
9129 &mut target_rx,
9130 hello_frame("target", PROTOCOL_VERSION, 1),
9131 )
9132 .await;
9133 let actual = query_live_roots(&handler, &target_ctx).await;
9134 let ModuleControlResponseToModule::LiveRoots {
9135 roots,
9136 unknown_root_bindings,
9137 total_bindings,
9138 } = actual
9139 else {
9140 panic!("expected live roots")
9141 };
9142 assert!(roots.is_empty());
9143 assert_eq!(unknown_root_bindings, 0);
9144 assert_eq!(total_bindings, 0);
9145 assert_eq!(
9146 total_bindings,
9147 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
9148 );
9149 }
9150
9151 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
9158 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9159 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
9160 let text = std::fs::read_to_string(&path)
9161 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
9162 let vectors: Vec<(String, Value)> = text
9163 .lines()
9164 .filter(|line| !line.trim().is_empty())
9165 .map(|line| {
9166 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
9167 let id = entry["corpus_id"]
9168 .as_str()
9169 .expect("every vector carries a corpus_id")
9170 .to_string();
9171 (id, entry["package"].clone())
9172 })
9173 .collect();
9174 assert_eq!(
9178 vectors.len(),
9179 3,
9180 "vendored fed corpus changed size; re-sync from subc-federation"
9181 );
9182
9183 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
9198 let richest = vectors
9199 .iter()
9200 .filter_map(|(_, package)| package.as_object())
9201 .map(|object| {
9202 object
9203 .keys()
9204 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
9205 .count()
9206 })
9207 .max()
9208 .unwrap_or(0);
9209 assert!(
9210 richest >= 2,
9211 "vendored corpus no longer carries a package with unmodelled fields, \
9212 so the relay test can no longer distinguish a verbatim relay from a lossy one"
9213 );
9214
9215 vectors
9216 }
9217
9218 #[tokio::test]
9226 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
9227 for (corpus_id, package) in fed_admission_facts_vectors() {
9228 let registry = Arc::new(Registry::default());
9229 let forwarding = Arc::new(ForwardingTable::default());
9230 let supervisor = SupervisorHandle::new();
9231 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9232 let handler =
9233 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9234 .with_supervisor(supervisor)
9235 .with_admission_facts_config(
9236 Some("fed".to_string()),
9237 Some(vec!["target".to_string()]),
9238 );
9239
9240 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
9241 hello_via_sink(
9242 &handler,
9243 &target_ctx,
9244 &mut target_rx,
9245 hello_frame("target", PROTOCOL_VERSION, 1),
9246 )
9247 .await;
9248
9249 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
9250 let route_handler = handler.clone();
9251 let expected = package.clone();
9252 let route_task = tokio::spawn(async move {
9253 route_handler
9254 .handle_control_frame(
9255 &client_ctx,
9256 route_open_frame_with_admission_facts(
9257 20,
9258 "target",
9259 unique_project_root("admission-facts"),
9260 Some(subc_control::ConsumerIdentity {
9261 module_id: "fed".to_string(),
9262 launch_nonce: "fed-nonce".to_string(),
9263 }),
9264 Some(package),
9265 ),
9266 )
9267 .await
9268 .unwrap()
9269 });
9270
9271 let bind_frame = target_rx.recv().await.unwrap();
9272 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9273 let ModuleControlRequest::RouteBind {
9274 admission_facts, ..
9275 } = bind
9276 else {
9277 panic!("{corpus_id}: expected route.bind")
9278 };
9279 assert_eq!(
9280 admission_facts,
9281 Some(expected),
9282 "{corpus_id}: relay must not add, drop or reshape any field"
9283 );
9284
9285 handler
9286 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
9287 .await
9288 .unwrap();
9289 route_task.await.unwrap();
9290 }
9291 }
9292
9293 #[tokio::test]
9294 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
9295 let registry = Arc::new(Registry::default());
9296 let forwarding = Arc::new(ForwardingTable::default());
9297 let supervisor = SupervisorHandle::new();
9298 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9299 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
9300 let handler =
9301 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9302 .with_supervisor(supervisor)
9303 .with_admission_facts_config(
9304 Some("fed".to_string()),
9305 Some(vec!["target".to_string()]),
9306 );
9307
9308 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
9309 hello_via_sink(
9310 &handler,
9311 &target_ctx,
9312 &mut target_rx,
9313 hello_frame("target", PROTOCOL_VERSION, 1),
9314 )
9315 .await;
9316 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
9317 hello_via_sink(
9318 &handler,
9319 &other_ctx,
9320 &mut other_rx,
9321 hello_frame("other", PROTOCOL_VERSION, 2),
9322 )
9323 .await;
9324
9325 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
9326 let expected_facts = facts.clone();
9327 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
9328 let route_handler = handler.clone();
9329 let route_task = tokio::spawn(async move {
9330 route_handler
9331 .handle_control_frame(
9332 &client_ctx,
9333 route_open_frame_with_admission_facts(
9334 10,
9335 "target",
9336 unique_project_root("admission-facts"),
9337 Some(subc_control::ConsumerIdentity {
9338 module_id: "fed".to_string(),
9339 launch_nonce: "fed-nonce".to_string(),
9340 }),
9341 Some(facts.clone()),
9342 ),
9343 )
9344 .await
9345 .unwrap()
9346 });
9347 let bind_frame = target_rx.recv().await.unwrap();
9348 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9349 let ModuleControlRequest::RouteBind {
9350 admission_facts, ..
9351 } = bind
9352 else {
9353 panic!("expected route.bind")
9354 };
9355 assert_eq!(admission_facts, Some(expected_facts));
9356 handler
9357 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
9358 .await
9359 .unwrap();
9360 assert!(route_task.await.unwrap().is_empty());
9361 assert!(matches!(
9362 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
9363 .unwrap(),
9364 ClientControlResponse::RouteOpen { .. }
9365 ));
9366
9367 let direct = handler
9368 .handle_control_frame(
9369 &route_ctx(ConnectionId::new(73)).0,
9370 route_open_frame_with_admission_facts(
9371 11,
9372 "target",
9373 unique_project_root("admission-facts"),
9374 None,
9375 Some(json!({"x": 1})),
9376 ),
9377 )
9378 .await
9379 .unwrap();
9380 assert_eq!(
9381 parse_error(&direct[0])["code"],
9382 "admission_facts_not_permitted"
9383 );
9384
9385 let different_reserved = handler
9386 .handle_control_frame(
9387 &route_ctx(ConnectionId::new(77)).0,
9388 route_open_frame_with_admission_facts(
9389 15,
9390 "target",
9391 unique_project_root("admission-facts"),
9392 Some(subc_control::ConsumerIdentity {
9393 module_id: "other".to_string(),
9394 launch_nonce: "other-nonce".to_string(),
9395 }),
9396 Some(json!({"x": 1})),
9397 ),
9398 )
9399 .await
9400 .unwrap();
9401 assert_eq!(
9402 parse_error(&different_reserved[0])["code"],
9403 "admission_facts_not_permitted"
9404 );
9405
9406 let other_target = handler
9407 .handle_control_frame(
9408 &route_ctx(ConnectionId::new(74)).0,
9409 route_open_frame_with_admission_facts(
9410 12,
9411 "other",
9412 unique_project_root("admission-facts"),
9413 Some(subc_control::ConsumerIdentity {
9414 module_id: "fed".to_string(),
9415 launch_nonce: "fed-nonce".to_string(),
9416 }),
9417 Some(json!({"x": 1})),
9418 ),
9419 )
9420 .await
9421 .unwrap();
9422 assert_eq!(
9423 parse_error(&other_target[0])["code"],
9424 "admission_facts_target_not_allowed"
9425 );
9426
9427 let nonexistent = handler
9428 .handle_control_frame(
9429 &route_ctx(ConnectionId::new(75)).0,
9430 route_open_frame_with_admission_facts(
9431 13,
9432 "missing",
9433 unique_project_root("admission-facts"),
9434 None,
9435 Some(json!({"x": 1})),
9436 ),
9437 )
9438 .await
9439 .unwrap();
9440 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
9441
9442 let described = handler
9443 .handle_control_frame(
9444 &route_ctx(ConnectionId::new(76)).0,
9445 Frame::build(
9446 FrameType::Request,
9447 control_flags(),
9448 0,
9449 0,
9450 14,
9451 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
9452 )
9453 .unwrap(),
9454 )
9455 .await
9456 .unwrap();
9457 let ClientControlResponse::ServerDescribe { capabilities, .. } =
9458 serde_json::from_slice(&described[0].body).unwrap()
9459 else {
9460 panic!("expected server.describe response")
9461 };
9462 assert!(capabilities
9463 .iter()
9464 .any(|cap| cap == "admission_facts_relay_v1"));
9465 }
9466
9467 #[tokio::test]
9468 async fn admission_facts_without_configured_carrier_are_rejected() {
9469 let registry = Arc::new(Registry::default());
9470 let forwarding = Arc::new(ForwardingTable::default());
9471 let handler = ControlHandler::with_forwarding(registry, forwarding);
9472 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
9473 hello_via_sink(
9474 &handler,
9475 &target_ctx,
9476 &mut target_rx,
9477 hello_frame("target", PROTOCOL_VERSION, 1),
9478 )
9479 .await;
9480
9481 let responses = handler
9482 .handle_control_frame(
9483 &route_ctx(ConnectionId::new(79)).0,
9484 route_open_frame_with_admission_facts(
9485 16,
9486 "target",
9487 unique_project_root("admission-facts"),
9488 None,
9489 Some(json!({"x": 1})),
9490 ),
9491 )
9492 .await
9493 .unwrap();
9494 assert_eq!(
9495 parse_error(&responses[0])["code"],
9496 "admission_facts_not_permitted"
9497 );
9498 }
9499
9500 #[tokio::test]
9501 async fn route_open_relays_consumer_capabilities_verbatim() {
9502 let registry = Arc::new(Registry::default());
9503 let forwarding = Arc::new(ForwardingTable::default());
9504 let handler =
9505 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9506 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
9507 hello_via_sink(
9508 &handler,
9509 &module_ctx,
9510 &mut module_rx,
9511 hello_frame("aft", PROTOCOL_VERSION, 7),
9512 )
9513 .await;
9514
9515 let expected = vec!["elicitation".to_string(), "roots".to_string()];
9516 let expected_for_request = expected.clone();
9517 let project_root = unique_project_root("consumer-capabilities-present");
9518 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
9519 let route_handler = handler.clone();
9520 let route_task = tokio::spawn(async move {
9521 route_handler
9522 .handle_control_frame(
9523 &client_ctx,
9524 route_open_frame_with_consumer_capabilities(
9525 401,
9526 "aft",
9527 project_root,
9528 Some(expected_for_request),
9529 ),
9530 )
9531 .await
9532 .unwrap()
9533 });
9534 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9535 .await
9536 .unwrap()
9537 .unwrap();
9538 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9539 let ModuleControlRequest::RouteBind {
9540 consumer_capabilities,
9541 ..
9542 } = bind
9543 else {
9544 panic!("expected route.bind request, got {bind:?}");
9545 };
9546 assert_eq!(consumer_capabilities, Some(expected.clone()));
9547
9548 handler
9549 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9550 .await
9551 .unwrap();
9552 let route_response = route_task.await.unwrap();
9553 assert!(route_response.is_empty());
9554 let published = client_rx.recv().await.unwrap();
9555 assert!(matches!(
9556 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9557 ClientControlResponse::RouteOpen { .. }
9558 ));
9559 }
9560
9561 #[tokio::test]
9562 async fn route_open_without_consumer_capabilities_relays_none() {
9563 let registry = Arc::new(Registry::default());
9564 let forwarding = Arc::new(ForwardingTable::default());
9565 let handler =
9566 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9567 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
9568 hello_via_sink(
9569 &handler,
9570 &module_ctx,
9571 &mut module_rx,
9572 hello_frame("aft", PROTOCOL_VERSION, 7),
9573 )
9574 .await;
9575
9576 let project_root = unique_project_root("consumer-capabilities-absent");
9577 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
9578 let route_handler = handler.clone();
9579 let route_task = tokio::spawn(async move {
9580 route_handler
9581 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
9582 .await
9583 .unwrap()
9584 });
9585 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9586 .await
9587 .unwrap()
9588 .unwrap();
9589 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9590 let ModuleControlRequest::RouteBind {
9591 consumer_capabilities,
9592 ..
9593 } = bind
9594 else {
9595 panic!("expected route.bind request, got {bind:?}");
9596 };
9597 assert_eq!(consumer_capabilities, None);
9598
9599 handler
9600 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9601 .await
9602 .unwrap();
9603 let route_response = route_task.await.unwrap();
9604 assert!(route_response.is_empty());
9605 let published = client_rx.recv().await.unwrap();
9606 assert!(matches!(
9607 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9608 ClientControlResponse::RouteOpen { .. }
9609 ));
9610 }
9611
9612 async fn bind_role_versions_for(
9616 sent: Option<BTreeMap<String, String>>,
9617 connection: u64,
9618 ) -> Option<BTreeMap<String, String>> {
9619 let registry = Arc::new(Registry::default());
9620 let forwarding = Arc::new(ForwardingTable::default());
9621 let handler =
9622 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9623 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
9624 hello_via_sink(
9625 &handler,
9626 &module_ctx,
9627 &mut module_rx,
9628 hello_frame("aft", PROTOCOL_VERSION, 7),
9629 )
9630 .await;
9631 let project_root = unique_project_root("role-versions");
9632 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
9633 let route_handler = handler.clone();
9634 let route_task = tokio::spawn(async move {
9635 route_handler
9636 .handle_control_frame(
9637 &client_ctx,
9638 route_open_frame_with_role_versions(403, "aft", project_root, sent),
9639 )
9640 .await
9641 .unwrap()
9642 });
9643 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9644 .await
9645 .expect("a well-formed route.open reaches the module as a bind")
9646 .unwrap();
9647 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9648 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
9649 panic!("expected route.bind request, got {bind:?}");
9650 };
9651 handler
9652 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9653 .await
9654 .unwrap();
9655 assert!(route_task.await.unwrap().is_empty());
9656 let published = client_rx.recv().await.unwrap();
9657 assert!(matches!(
9658 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9659 ClientControlResponse::RouteOpen { .. }
9660 ));
9661 role_versions
9662 }
9663
9664 #[tokio::test]
9665 async fn route_open_relays_role_versions_verbatim() {
9666 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
9667 assert_eq!(
9668 bind_role_versions_for(Some(sent.clone()), 141).await,
9669 Some(sent)
9670 );
9671 }
9672
9673 #[tokio::test]
9676 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
9677 assert_eq!(bind_role_versions_for(None, 143).await, None);
9678 assert_eq!(
9679 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
9680 None
9681 );
9682 }
9683
9684 #[tokio::test]
9685 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
9686 let registry = Arc::new(Registry::default());
9687 let forwarding = Arc::new(ForwardingTable::default());
9688 let handler =
9689 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9690 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
9691 hello_via_sink(
9692 &handler,
9693 &module_ctx,
9694 &mut module_rx,
9695 hello_frame("aft", PROTOCOL_VERSION, 7),
9696 )
9697 .await;
9698
9699 let nine: BTreeMap<String, String> = (0..9)
9700 .map(|index| (format!("role-{index}"), "v1".to_string()))
9701 .collect();
9702 for (label, malformed) in [
9703 (
9704 "invalid role name",
9705 role_versions(&[("Tool_Provider", "v1")]),
9706 ),
9707 ("invalid version", role_versions(&[("tool-provider", "v0")])),
9708 ("nine entries", nine),
9709 ] {
9710 let responses = tokio::time::timeout(
9713 Duration::from_secs(1),
9714 handler.handle_control_frame(
9715 &route_ctx(ConnectionId::new(148)).0,
9716 route_open_frame_with_role_versions(
9717 404,
9718 "aft",
9719 unique_project_root("role-versions-malformed"),
9720 Some(malformed),
9721 ),
9722 ),
9723 )
9724 .await
9725 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
9726 .unwrap();
9727 assert_eq!(responses.len(), 1, "{label}");
9728 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
9729 let error = parse_error(&responses[0]);
9730 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
9731 assert_eq!(
9732 error["detail"]["field"], "role_versions",
9733 "{label}: {error}"
9734 );
9735 assert!(
9736 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
9737 "{label}: a malformed declaration is terminal"
9738 );
9739 assert!(
9740 module_rx.try_recv().is_err(),
9741 "{label}: the module must never see a bind"
9742 );
9743 }
9744 }
9745
9746 #[tokio::test]
9750 async fn route_role_versions_capability_is_advertised() {
9751 let handler = ControlHandler::new(Arc::new(Registry::default()));
9752 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
9753 let ack = hello_via_sink(
9754 &handler,
9755 &ctx,
9756 &mut rx,
9757 hello_frame("m", PROTOCOL_VERSION, 1),
9758 )
9759 .await;
9760 let ack = parse_ack(&ack);
9761 assert!(
9762 ack.subc_capabilities
9763 .iter()
9764 .any(|c| c == "route-role-versions/v1"),
9765 "{:?}",
9766 ack.subc_capabilities
9767 );
9768
9769 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
9770 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
9771 let reply = handler
9772 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
9773 .await
9774 .unwrap()
9775 .pop()
9776 .unwrap();
9777 let ClientControlResponse::ServerDescribe { capabilities, .. } =
9778 serde_json::from_slice(&reply.body).unwrap()
9779 else {
9780 panic!("not a server.describe reply");
9781 };
9782 assert!(
9783 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
9784 "{capabilities:?}"
9785 );
9786 }
9787
9788 #[tokio::test]
9789 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
9790 let registry = Arc::new(Registry::default());
9791 let forwarding = Arc::new(ForwardingTable::default());
9792 let handler =
9793 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9794 .with_health_probe_timeout(Duration::from_secs(5));
9795 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
9796 hello_via_sink(
9797 &handler,
9798 &module_ctx,
9799 &mut module_rx,
9800 non_routable_hello_frame_with_control_ops(
9801 "mcp",
9802 300,
9803 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9804 ),
9805 )
9806 .await;
9807 assert!(registry
9808 .get_module("mcp")
9809 .unwrap()
9810 .unwrap()
9811 .manifest
9812 .provides
9813 .is_empty());
9814
9815 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
9816 let route_response = handler
9817 .handle_control_frame(
9818 &route_client_ctx,
9819 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
9820 )
9821 .await
9822 .unwrap();
9823 assert_eq!(route_response[0].header.ty, FrameType::Error);
9824 assert_eq!(
9825 parse_error(&route_response[0])["code"],
9826 "target_unavailable"
9827 );
9828 assert!(parse_error(&route_response[0])["message"]
9829 .as_str()
9830 .unwrap()
9831 .contains("does not provide the requested target"));
9832 assert!(module_rx.try_recv().is_err());
9833
9834 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
9835 let health_handler = handler.clone();
9836 let health_task = tokio::spawn(async move {
9837 health_handler
9838 .handle_control_frame(
9839 &health_client_ctx,
9840 supervisor_health_probe_frame(302, "mcp"),
9841 )
9842 .await
9843 .unwrap()
9844 });
9845 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9846 .await
9847 .unwrap()
9848 .unwrap();
9849 assert_eq!(
9850 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
9851 ModuleControlRequest::HealthCheck {}
9852 );
9853 handler
9854 .handle_control_frame(
9855 &module_ctx,
9856 health_response(health_frame.header.corr, HealthStatus::Ok),
9857 )
9858 .await
9859 .unwrap();
9860 let health_response = health_task.await.unwrap();
9861 assert_eq!(health_response[0].header.ty, FrameType::Response);
9862 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
9863 ClientControlResponse::SupervisorHealthProbe {
9864 module_id, status, ..
9865 } => {
9866 assert_eq!(module_id, "mcp");
9867 assert_eq!(status, HealthStatus::Ok);
9868 }
9869 other => panic!("unexpected health response: {other:?}"),
9870 }
9871
9872 forwarding
9877 .cleanup_connection(module_ctx.connection_id)
9878 .unwrap();
9879 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
9880 let cleanup_response = tokio::time::timeout(
9881 Duration::from_millis(200),
9882 handler.handle_control_frame(
9883 &cleanup_probe_ctx,
9884 supervisor_health_probe_frame(303, "mcp"),
9885 ),
9886 )
9887 .await
9888 .expect("probe should fail immediately when the control lane is gone")
9889 .unwrap();
9890 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
9891 assert_eq!(
9892 parse_error(&cleanup_response[0])["code"],
9893 "target_unavailable"
9894 );
9895 assert!(parse_error(&cleanup_response[0])["message"]
9896 .as_str()
9897 .unwrap()
9898 .contains("no module connection"));
9899
9900 handler
9901 .cleanup_connection(module_ctx.connection_id)
9902 .unwrap();
9903 }
9904
9905 #[tokio::test]
9906 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
9907 let registry = Arc::new(Registry::default());
9908 let supervisor_handle = SupervisorHandle::new();
9909 let supervisor =
9910 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
9911 .with_handle(supervisor_handle.clone())
9912 .with_connection_file_path(
9913 std::env::temp_dir()
9914 .join(format!("subc-route-open-warming-{}", std::process::id())),
9915 );
9916 let module = supervisor
9917 .supervise_configured(
9918 ModuleSpec {
9919 module_id: "warming".to_string(),
9920 program: fake_aft_stub_path(),
9921 args: Vec::new(),
9922 env: Vec::new(),
9923 reserved: false,
9924 reserved_prefixes: Vec::new(),
9925 protocol: ModuleProtocol::Subc,
9926 overlap: Default::default(),
9927 },
9928 true,
9929 )
9930 .unwrap();
9931 assert_eq!(module.state().unwrap(), ModuleState::Running);
9932
9933 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9934 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
9935 let response = handler
9936 .handle_control_frame(
9937 &ctx,
9938 route_open_frame(304, "warming", unique_project_root("warming")),
9939 )
9940 .await
9941 .unwrap();
9942 module.stop().await.unwrap();
9943
9944 assert_eq!(response[0].header.ty, FrameType::Error);
9945 let error = parse_error(&response[0]);
9946 assert_eq!(error["code"], "module_warming");
9947 assert!(error["message"]
9948 .as_str()
9949 .unwrap()
9950 .contains("state=running, enabled=true, live=false"));
9951 }
9952
9953 #[test]
9954 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
9955 let handler = ControlHandler::new(Arc::new(Registry::default()));
9956 let capture = EventCapture::default();
9957 let _subscriber =
9958 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9959 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
9960 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
9961 let pending = (0..limit).collect::<Vec<_>>();
9962 let response = handler
9963 .route_open_capacity_refusal(
9964 &ctx,
9965 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
9966 "busy",
9967 pending.len(),
9968 limit,
9969 )
9970 .unwrap();
9971 assert_eq!(parse_error(&response)["code"], "target_unavailable");
9972 let event = capture
9973 .events()
9974 .into_iter()
9975 .find(|event| {
9976 event.target == "control"
9977 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
9978 })
9979 .expect("connection admission refusal event");
9980 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
9981 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
9982 }
9983
9984 #[test]
9985 fn route_open_target_cap_logs_admission_reason_and_capacity() {
9986 let handler = ControlHandler::new(Arc::new(Registry::default()));
9987 let capture = EventCapture::default();
9988 let _subscriber =
9989 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9990 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
9991 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
9992 let guards = (0..limit)
9993 .map(|_| {
9994 handler
9995 .route_bind_concurrency
9996 .try_admit("busy", limit)
9997 .unwrap()
9998 })
9999 .collect::<Vec<_>>();
10000 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
10001 Err(in_flight) => in_flight,
10002 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
10003 };
10004 let response = handler
10005 .route_open_target_capacity_refusal(
10006 &ctx,
10007 &route_open_frame(397, "busy", unique_project_root("target-cap")),
10008 "busy",
10009 in_flight,
10010 )
10011 .unwrap();
10012 assert_eq!(parse_error(&response)["code"], "target_unavailable");
10013 let event = capture
10014 .events()
10015 .into_iter()
10016 .find(|event| {
10017 event.target == "control"
10018 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
10019 })
10020 .expect("target admission refusal event");
10021 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
10022 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
10023 drop(guards);
10024 }
10025
10026 #[tokio::test]
10031 async fn route_open_refusal_names_the_check_that_refused() {
10032 let handler = ControlHandler::new(Arc::new(Registry::default()));
10033 let capture = EventCapture::default();
10034 let _subscriber =
10035 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10036 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
10037 let response = handler
10038 .handle_control_frame(
10039 &ctx,
10040 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
10041 )
10042 .await
10043 .unwrap();
10044
10045 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
10046 let event = capture
10047 .events()
10048 .into_iter()
10049 .find(|event| {
10050 event.target == "control"
10051 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
10052 })
10053 .expect("route.open refusal event");
10054 assert_eq!(
10055 event.fields.get("reason"),
10056 Some(&"\"not_registered\"".to_string())
10057 );
10058 }
10059
10060 #[tokio::test]
10061 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
10062 let registry = Arc::new(Registry::default());
10063 let supervisor_handle = SupervisorHandle::new();
10064 let supervisor =
10065 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
10066 .with_handle(supervisor_handle.clone())
10067 .with_connection_file_path(std::env::temp_dir().join(format!(
10068 "subc-route-open-refusal-info-{}",
10069 std::process::id()
10070 )));
10071 let module = supervisor
10072 .supervise_configured(
10073 ModuleSpec {
10074 module_id: "warming".to_string(),
10075 program: fake_aft_stub_path(),
10076 args: Vec::new(),
10077 env: Vec::new(),
10078 reserved: false,
10079 reserved_prefixes: Vec::new(),
10080 protocol: ModuleProtocol::Subc,
10081 overlap: Default::default(),
10082 },
10083 true,
10084 )
10085 .unwrap();
10086 assert_eq!(module.state().unwrap(), ModuleState::Running);
10087
10088 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10089 assert!(handler
10090 .counters()
10091 .snapshot()
10092 .get("route_open_refused_by_code")
10093 .is_none());
10094 let capture = EventCapture::default();
10095 let _subscriber =
10096 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10097 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
10098 let response = handler
10099 .handle_control_frame(
10100 &ctx,
10101 route_open_frame(394, "warming", unique_project_root("refusal-info")),
10102 )
10103 .await
10104 .unwrap();
10105 module.stop().await.unwrap();
10106
10107 assert_eq!(parse_error(&response[0])["code"], "module_warming");
10108 let event = capture
10109 .events()
10110 .into_iter()
10111 .find(|event| {
10112 event.target == "control"
10113 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
10114 })
10115 .expect("route.open refusal event");
10116 assert_eq!(
10117 event.fields.get("module_id"),
10118 Some(&"\"warming\"".to_string())
10119 );
10120 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
10121 assert_eq!(
10122 event.fields.get("reason"),
10123 Some(&"\"supervised_not_registered\"".to_string())
10124 );
10125 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
10126 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
10127 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
10128 assert_eq!(
10129 handler.counters().snapshot()["route_open_refused_by_code"],
10130 json!({ "module_warming": 1 })
10131 );
10132 }
10133
10134 const OUTAGE_START: &str = "route.open refusing module: not serving";
10135 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
10136
10137 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
10138 capture
10139 .events()
10140 .into_iter()
10141 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
10142 .collect()
10143 }
10144
10145 fn supervise_stub(
10146 registry: &Arc<Registry>,
10147 module_id: &str,
10148 enabled: bool,
10149 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
10150 let supervisor_handle = SupervisorHandle::new();
10151 let supervisor =
10152 Supervisor::new(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
10153 .with_handle(supervisor_handle.clone())
10154 .with_connection_file_path(std::env::temp_dir().join(format!(
10155 "subc-route-outage-{module_id}-{}",
10156 std::process::id()
10157 )));
10158 let module = supervisor
10159 .supervise_configured(
10160 ModuleSpec {
10161 module_id: module_id.to_string(),
10162 program: fake_aft_stub_path(),
10163 args: Vec::new(),
10164 env: Vec::new(),
10165 reserved: false,
10166 reserved_prefixes: Vec::new(),
10167 protocol: ModuleProtocol::Subc,
10168 overlap: Default::default(),
10169 },
10170 enabled,
10171 )
10172 .unwrap();
10173 (supervisor_handle, module)
10174 }
10175
10176 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
10177 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
10178 module_id: module_id.to_string(),
10179 drain_timeout_ms: Some(50),
10180 })
10181 .unwrap();
10182 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
10183 }
10184
10185 #[test]
10189 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
10190 let registry = Arc::new(Registry::default());
10191 let forwarding = Arc::new(ForwardingTable::default());
10192 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10193 let second = ControlHandler::with_forwarding(registry, forwarding);
10194 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
10195 }
10196
10197 #[tokio::test(flavor = "current_thread")]
10201 async fn route_open_unknown_module_refusals_add_no_outage_state() {
10202 let handler = ControlHandler::new(Arc::new(Registry::default()));
10203 let capture = EventCapture::default();
10204 let _subscriber =
10205 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10206 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
10207 for corr in 0..8 {
10208 let response = handler
10209 .handle_control_frame(
10210 &ctx,
10211 route_open_frame(
10212 380 + corr,
10213 &format!("nobody-{corr}"),
10214 unique_project_root("outage-unknown"),
10215 ),
10216 )
10217 .await
10218 .unwrap();
10219 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
10220 }
10221
10222 assert_eq!(handler.route_outages.tracked_module_count(), 0);
10223 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
10224 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
10225 }
10226
10227 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
10232 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
10233 let registry = Arc::new(Registry::default());
10234 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
10235 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10236 let capture = EventCapture::default();
10237 let _subscriber =
10238 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10239 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
10240 handler.route_outages.record_accepted("outage-restart");
10243
10244 let response = handler
10245 .handle_control_frame(
10246 &ctx,
10247 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
10248 )
10249 .await
10250 .unwrap();
10251 assert_eq!(response[0].header.ty, FrameType::Error);
10252 let starts = outage_lines(&capture, OUTAGE_START);
10253 assert_eq!(starts.len(), 1, "{starts:?}");
10254 assert_eq!(starts[0].level, tracing::Level::WARN);
10255 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
10256 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
10257 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
10258 handler.route_outages.record_accepted("outage-restart");
10259 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
10260
10261 let restart = handler
10262 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
10263 .await
10264 .unwrap();
10265 assert_eq!(
10266 restart[0].header.ty,
10267 FrameType::Response,
10268 "{:?}",
10269 parse_error(&restart[0])
10270 );
10271 handler
10272 .handle_control_frame(
10273 &ctx,
10274 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
10275 )
10276 .await
10277 .unwrap();
10278 module.stop().await.unwrap();
10279
10280 let starts = outage_lines(&capture, OUTAGE_START);
10281 assert_eq!(starts.len(), 2, "{starts:?}");
10282 assert_eq!(starts[1].level, tracing::Level::INFO);
10283 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
10284 }
10285
10286 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
10290 async fn failed_operator_restart_leaves_no_operator_mark() {
10291 let registry = Arc::new(Registry::default());
10292 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
10293 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10294 let capture = EventCapture::default();
10295 let _subscriber =
10296 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10297 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
10298 handler.route_outages.record_accepted("outage-disabled");
10299
10300 let restart = handler
10301 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
10302 .await
10303 .unwrap();
10304 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
10305 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
10306
10307 handler
10308 .handle_control_frame(
10309 &ctx,
10310 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
10311 )
10312 .await
10313 .unwrap();
10314 let starts = outage_lines(&capture, OUTAGE_START);
10315 assert_eq!(starts.len(), 1, "{starts:?}");
10316 assert_eq!(starts[0].level, tracing::Level::WARN);
10317 }
10318
10319 #[tokio::test(flavor = "current_thread")]
10320 async fn route_open_unknown_module_escapes_target_module_id() {
10321 let handler = ControlHandler::new(Arc::new(Registry::default()));
10322 let capture = EventCapture::default();
10323 let _subscriber =
10324 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10325 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
10326 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
10327 let response = handler
10328 .handle_control_frame(
10329 &ctx,
10330 route_open_frame(
10331 395,
10332 hostile_module_id,
10333 unique_project_root("hostile-target-module-id"),
10334 ),
10335 )
10336 .await
10337 .unwrap();
10338
10339 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
10340 let event = capture
10341 .events()
10342 .into_iter()
10343 .find(|event| {
10344 event.target == "control"
10345 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
10346 })
10347 .expect("route.open unknown-module refusal event");
10348 let logged = event.fields.get("module_id").expect("module_id field");
10349 assert!(!logged.bytes().any(|byte| byte < 0x20));
10350 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
10351 }
10352
10353 #[tokio::test(flavor = "current_thread")]
10354 async fn route_open_module_rejection_uses_daemon_counter_key() {
10355 let registry = Arc::new(Registry::default());
10356 let forwarding = Arc::new(ForwardingTable::default());
10357 let handler =
10358 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10359 let module_connection = ConnectionId::new(95);
10360 let (module_ctx, mut module_rx) = route_ctx(module_connection);
10361 hello_via_sink(
10362 &handler,
10363 &module_ctx,
10364 &mut module_rx,
10365 hello_frame("aft", PROTOCOL_VERSION, 395),
10366 )
10367 .await;
10368
10369 let client_connection = ConnectionId::new(96);
10370 let (client_ctx, _client_rx) = route_ctx(client_connection);
10371 let capture = EventCapture::default();
10372 let _subscriber =
10373 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10374 let (route_task, bind) = relay_route_open(
10375 &handler,
10376 client_connection,
10377 &client_ctx.egress,
10378 &mut module_rx,
10379 396,
10380 "aft",
10381 "hostile-module-code",
10382 )
10383 .await;
10384 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
10385 let rejection = Frame::build(
10386 FrameType::Error,
10387 control_flags(),
10388 0,
10389 0,
10390 bind.header.corr,
10391 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
10392 )
10393 .unwrap();
10394 handler
10395 .handle_control_frame(&module_ctx, rejection)
10396 .await
10397 .unwrap();
10398
10399 let response = route_task.await.unwrap();
10400 assert_eq!(parse_error(&response[0])["code"], hostile_code);
10401 let counters = handler.counters().snapshot();
10402 assert_eq!(
10403 counters["route_open_refused_by_code"],
10404 json!({ "module_rejected": 1 })
10405 );
10406 assert!(counters["route_open_refused_by_code"]
10407 .get(hostile_code)
10408 .is_none());
10409
10410 let event = capture
10411 .events()
10412 .into_iter()
10413 .find(|event| {
10414 event.target == "control"
10415 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
10416 })
10417 .expect("route.open module-rejection refusal event");
10418 let logged = event.fields.get("module_code").expect("module_code field");
10419 assert!(!logged.bytes().any(|byte| byte < 0x20));
10420 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
10421 }
10422
10423 #[tokio::test]
10424 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
10425 let registry = Arc::new(Registry::default());
10426 let supervisor_handle = SupervisorHandle::new();
10427 let missing_program = std::env::temp_dir().join(format!(
10428 "subc-route-open-missing-program-{}",
10429 std::process::id()
10430 ));
10431 let supervisor =
10432 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
10433 .with_handle(supervisor_handle.clone());
10434 let module = supervisor
10435 .supervise_configured(
10436 ModuleSpec {
10437 module_id: "failed".to_string(),
10438 program: missing_program,
10439 args: Vec::new(),
10440 env: Vec::new(),
10441 reserved: false,
10442 reserved_prefixes: Vec::new(),
10443 protocol: ModuleProtocol::Subc,
10444 overlap: Default::default(),
10445 },
10446 true,
10447 )
10448 .unwrap();
10449 assert_eq!(module.state().unwrap(), ModuleState::Failed);
10450
10451 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10452 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
10453 let response = handler
10454 .handle_control_frame(
10455 &ctx,
10456 route_open_frame(305, "failed", unique_project_root("failed")),
10457 )
10458 .await
10459 .unwrap();
10460
10461 assert_eq!(response[0].header.ty, FrameType::Error);
10462 let error = parse_error(&response[0]);
10463 assert_eq!(error["code"], "target_unavailable");
10464 assert!(error["message"]
10465 .as_str()
10466 .unwrap()
10467 .contains("state=failed, enabled=true, live=false"));
10468 }
10469
10470 #[tokio::test]
10471 async fn route_open_role_mismatch_remains_target_unavailable() {
10472 let registry = Arc::new(Registry::default());
10473 let handler = ControlHandler::new(Arc::clone(®istry));
10474 handler
10475 .handle_control(
10476 ConnectionId::new(41),
10477 non_routable_hello_frame_with_control_ops("health-only", 306, None),
10478 )
10479 .unwrap();
10480
10481 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
10482 let response = handler
10483 .handle_control_frame(
10484 &ctx,
10485 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
10486 )
10487 .await
10488 .unwrap();
10489
10490 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10491 assert!(parse_error(&response[0])["message"]
10492 .as_str()
10493 .unwrap()
10494 .contains("does not provide the requested target"));
10495 }
10496
10497 #[tokio::test]
10498 async fn route_open_inactive_registration_remains_target_unavailable() {
10499 let registry = Arc::new(Registry::default());
10500 let handler = ControlHandler::new(Arc::clone(®istry));
10501 handler
10502 .handle_control(
10503 ConnectionId::new(43),
10504 hello_frame("inactive", PROTOCOL_VERSION, 308),
10505 )
10506 .unwrap();
10507 assert!(registry
10508 .set_module_state_for_test("inactive", ChannelState::Closed)
10509 .unwrap());
10510
10511 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
10512 let response = handler
10513 .handle_control_frame(
10514 &ctx,
10515 route_open_frame(309, "inactive", unique_project_root("inactive")),
10516 )
10517 .await
10518 .unwrap();
10519
10520 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10521 assert!(parse_error(&response[0])["message"]
10522 .as_str()
10523 .unwrap()
10524 .contains("is not active"));
10525 }
10526
10527 #[tokio::test]
10528 async fn late_health_reply_is_recorded_through_the_module_response_path() {
10529 let registry = Arc::new(Registry::default());
10530 let forwarding = Arc::new(ForwardingTable::default());
10531 let supervisor_handle = SupervisorHandle::new();
10532 let supervisor = Supervisor::new(Arc::clone(®istry), crate::RestartPolicy::default())
10533 .with_forwarding(Arc::clone(&forwarding))
10534 .with_handle(supervisor_handle.clone());
10535 let module = supervisor
10536 .supervise_configured(
10537 crate::ModuleSpec {
10538 module_id: "late-health-response".to_string(),
10539 program: PathBuf::from("disabled-module"),
10540 args: Vec::new(),
10541 env: Vec::new(),
10542 reserved: false,
10543 reserved_prefixes: Vec::new(),
10544 protocol: ModuleProtocol::Subc,
10545 overlap: Default::default(),
10546 },
10547 false,
10548 )
10549 .unwrap();
10550 let handler =
10551 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10552 .with_supervisor(supervisor_handle);
10553 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
10554 handler
10555 .handle_control_frame(
10556 &module_ctx,
10557 hello_frame_with_control_ops(
10558 "late-health-response",
10559 PROTOCOL_VERSION,
10560 7,
10561 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10562 ),
10563 )
10564 .await
10565 .unwrap();
10566 let probe_started_at = Instant::now() - Duration::from_millis(80);
10567 let pending = forwarding
10568 .begin_health_probe_rpc_for(
10569 "late-health-response",
10570 MODULE_CONTROL_OP_HEALTH_CHECK,
10571 probe_started_at,
10572 Instant::now() - Duration::from_millis(1),
10573 )
10574 .unwrap();
10575 assert!(forwarding
10576 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
10577 .unwrap());
10578
10579 let responses = handler
10580 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
10581 .await
10582 .unwrap();
10583
10584 assert!(responses.is_empty());
10585 let health = module.status().unwrap().health;
10586 assert_eq!(health.late_answer_count, 1);
10587 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
10588 }
10589
10590 #[tokio::test]
10591 async fn health_probe_timeout_and_module_death_are_typed() {
10592 let registry = Arc::new(Registry::default());
10593 let forwarding = Arc::new(ForwardingTable::default());
10594 let handler =
10595 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10596 .with_health_probe_timeout(Duration::from_millis(50));
10597 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
10598 hello_via_sink(
10599 &handler,
10600 &module_ctx,
10601 &mut module_rx,
10602 hello_frame_with_control_ops(
10603 "aft",
10604 PROTOCOL_VERSION,
10605 7,
10606 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10607 ),
10608 )
10609 .await;
10610
10611 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10612 let responses = handler
10613 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
10614 .await
10615 .unwrap();
10616 assert_eq!(responses[0].header.ty, FrameType::Error);
10617 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
10618 let _ = module_rx.try_recv();
10619
10620 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
10621 let health_handler = handler.clone();
10622 let death_task = tokio::spawn(async move {
10623 health_handler
10624 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
10625 .await
10626 .unwrap()
10627 });
10628 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10629 .await
10630 .unwrap()
10631 .unwrap();
10632 handler
10633 .cleanup_connection(module_ctx.connection_id)
10634 .unwrap();
10635 let responses = death_task.await.unwrap();
10636 assert_eq!(responses[0].header.ty, FrameType::Error);
10637 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
10638 }
10639
10640 #[test]
10641 fn hello_requires_exact_protocol_version() {
10642 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
10643 let registry = Arc::new(Registry::default());
10644 let handler = ControlHandler::new(Arc::clone(®istry));
10645 let responses = handler
10646 .handle_control(
10647 ConnectionId::new(connection),
10648 hello_frame("aft", offered, 9),
10649 )
10650 .unwrap();
10651
10652 assert_eq!(responses.len(), 1);
10653 assert_eq!(responses[0].header.ty, FrameType::Error);
10654 let error = parse_error(&responses[0]);
10655 assert_eq!(error["code"], "version_unsupported");
10656 assert!(registry.get_module("aft").unwrap().is_none());
10657 assert_eq!(registry.active_registration_count().unwrap(), 0);
10658 }
10659 }
10660
10661 #[test]
10662 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
10663 let registry = Arc::new(Registry::default());
10664 let forwarding = Arc::new(ForwardingTable::default());
10665 let handler =
10666 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10667 let module_connection = ConnectionId::new(301);
10668 let registration = registry
10669 .register_with_control_ops(
10670 manifest("aft-push", PROTOCOL_VERSION),
10671 PROTOCOL_VERSION,
10672 module_connection,
10673 module_baseline_control_ops(),
10674 )
10675 .unwrap();
10676 let (module_tx, _module_rx) = mpsc::channel(8);
10677 let endpoint = forwarding
10678 .register_module_connection(
10679 module_connection,
10680 "aft-push".to_string(),
10681 PROTOCOL_VERSION,
10682 manifest_concurrency(®istration.manifest),
10683 FrameSink::new(module_tx),
10684 )
10685 .unwrap();
10686
10687 let unknown = Frame::build(
10689 FrameType::Push,
10690 control_flags(),
10691 0,
10692 0,
10693 5,
10694 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
10695 )
10696 .unwrap();
10697 let out = handler.handle_status_update(endpoint, unknown).unwrap();
10698 assert!(
10699 out.is_empty(),
10700 "unknown push op must be ignored, got {out:?}"
10701 );
10702
10703 let malformed = Frame::build(
10705 FrameType::Push,
10706 control_flags(),
10707 0,
10708 0,
10709 6,
10710 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
10711 )
10712 .unwrap();
10713 let out = handler.handle_status_update(endpoint, malformed).unwrap();
10714 assert_eq!(out.len(), 1);
10715 assert_eq!(out[0].header.ty, FrameType::Error);
10716 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
10717 }
10718
10719 #[test]
10720 fn hello_rejected_when_connection_already_owns_client_routes() {
10721 let registry = Arc::new(Registry::default());
10722 let forwarding = Arc::new(ForwardingTable::default());
10723 let handler =
10724 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10725 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
10727 let client_connection = ConnectionId::new(202);
10728
10729 let responses = handler
10732 .handle_control(
10733 client_connection,
10734 hello_frame("aft-second", PROTOCOL_VERSION, 9),
10735 )
10736 .unwrap();
10737 assert_eq!(responses[0].header.ty, FrameType::Error);
10738 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
10739 assert!(registry.get_module("aft-second").unwrap().is_none());
10740 }
10741
10742 #[tokio::test]
10743 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
10744 let registry = Arc::new(Registry::default());
10745 let forwarding = Arc::new(ForwardingTable::default());
10746 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
10747 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
10748 hello_via_sink(
10749 &handler,
10750 &module_ctx,
10751 &mut module_rx,
10752 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
10753 )
10754 .await;
10755 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
10756 let pending = forwarding
10757 .begin_route_bind_relay_for_test(
10758 client_ctx.connection_id,
10759 client_ctx.egress.clone(),
10760 2,
10761 "alpha",
10762 )
10763 .unwrap();
10764 forwarding
10765 .complete_pending_relay(
10766 module_ctx.connection_id,
10767 pending.corr,
10768 RouteBindRelayOutcome::Accepted,
10769 )
10770 .unwrap();
10771 client_rx.try_recv().unwrap();
10772 for module_id in ["beta", "alpha"] {
10773 let replies = handler
10774 .handle_control_frame(
10775 &module_ctx,
10776 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
10777 )
10778 .await
10779 .unwrap();
10780 assert_eq!(replies.len(), 1, "second HELLO must be refused");
10781 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
10782 }
10783 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
10784 assert!(registry.get_module("beta").unwrap().is_none());
10785 assert!(matches!(
10786 forwarding
10787 .lookup_data_route(
10788 client_ctx.connection_id,
10789 pending.client_channel,
10790 pending.client_epoch,
10791 )
10792 .unwrap(),
10793 DataRoute::Client(DataRouteState::Bound(_))
10794 ));
10795 assert!(handler
10796 .hello_launch_nonces
10797 .lock()
10798 .unwrap()
10799 .presented(module_ctx.connection_id, Some("alpha-nonce")));
10800 assert!(module_rx.try_recv().is_err());
10801 }
10802
10803 #[test]
10804 fn reserved_module_hello_requires_matching_launch_nonce() {
10805 let registry = Arc::new(Registry::default());
10806 let supervisor = SupervisorHandle::new();
10807 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
10810 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
10811
10812 let no_nonce = handler
10814 .handle_control(
10815 ConnectionId::new(1),
10816 hello_frame("vault", PROTOCOL_VERSION, 1),
10817 )
10818 .unwrap();
10819 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
10820 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
10821 assert!(registry.get_module("vault").unwrap().is_none());
10822
10823 let wrong = handler
10825 .handle_control(
10826 ConnectionId::new(2),
10827 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
10828 )
10829 .unwrap();
10830 assert_eq!(wrong[0].header.ty, FrameType::Error);
10831 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
10832 assert!(registry.get_module("vault").unwrap().is_none());
10833
10834 let ok = handler
10836 .handle_control(
10837 ConnectionId::new(3),
10838 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
10839 )
10840 .unwrap();
10841 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
10842 assert!(registry.get_module("vault").unwrap().is_some());
10843 }
10844
10845 #[test]
10846 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
10847 let registry = Arc::new(Registry::default());
10848 let supervisor = SupervisorHandle::new();
10849 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
10850 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
10851 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
10852
10853 let squat = handler
10854 .handle_control(
10855 ConnectionId::new(1),
10856 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
10857 )
10858 .unwrap();
10859 assert_eq!(squat[0].header.ty, FrameType::Error);
10860 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
10861 assert!(parse_error(&squat[0])["message"]
10862 .as_str()
10863 .unwrap()
10864 .contains("fed:"));
10865
10866 let accepted_peer = handler
10867 .handle_control(
10868 ConnectionId::new(2),
10869 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
10870 )
10871 .unwrap();
10872 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
10873
10874 let accepted_short = handler
10875 .handle_control(
10876 ConnectionId::new(3),
10877 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
10878 )
10879 .unwrap();
10880 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
10881
10882 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
10883 let response = handler
10884 .handle_control(
10885 ConnectionId::new(conn),
10886 hello_frame(module_id, PROTOCOL_VERSION, conn),
10887 )
10888 .unwrap();
10889 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
10890 }
10891 }
10892
10893 #[test]
10894 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
10895 let registry = Arc::new(Registry::default());
10896 let supervisor = SupervisorHandle::new();
10897 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
10898 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
10899 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
10900 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
10901
10902 let owner_nonce = handler
10903 .handle_control(
10904 ConnectionId::new(1),
10905 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
10906 )
10907 .unwrap();
10908 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
10909 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
10910 assert!(registry.get_module("fed:special").unwrap().is_none());
10911
10912 let exact_nonce = handler
10913 .handle_control(
10914 ConnectionId::new(2),
10915 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
10916 )
10917 .unwrap();
10918 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
10919 assert!(registry.get_module("fed:special").unwrap().is_some());
10920 }
10921
10922 #[test]
10923 fn non_reserved_module_ignores_launch_nonce() {
10924 let registry = Arc::new(Registry::default());
10925 let handler = ControlHandler::new(Arc::clone(®istry));
10928 let no_nonce = handler
10929 .handle_control(
10930 ConnectionId::new(1),
10931 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
10932 )
10933 .unwrap();
10934 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
10935 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
10936
10937 let echoed_nonce = handler
10938 .handle_control(
10939 ConnectionId::new(2),
10940 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
10941 )
10942 .unwrap();
10943 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
10944 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
10945 }
10946
10947 #[test]
10948 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
10949 let handler = ControlHandler::default();
10950 let conn = ConnectionId::new(1);
10951 let malformed = Frame::build(
10952 FrameType::Hello,
10953 control_flags(),
10954 0,
10955 0,
10956 3,
10957 b"{not json".to_vec(),
10958 )
10959 .unwrap();
10960
10961 let error = handler.handle_control(conn, malformed).unwrap();
10962 assert_eq!(error[0].header.ty, FrameType::Error);
10963 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
10964
10965 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
10966 let pong = handler.handle_control(conn, ping).unwrap();
10967 assert_eq!(pong[0].header.ty, FrameType::Pong);
10968 assert_eq!(pong[0].header.corr, 4);
10969 }
10970
10971 #[test]
10972 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
10973 let registry = Arc::new(Registry::default());
10974 let handler = ControlHandler::new(Arc::clone(®istry));
10975
10976 handler
10977 .handle_control(
10978 ConnectionId::new(1),
10979 hello_frame("aft", PROTOCOL_VERSION, 1),
10980 )
10981 .unwrap();
10982 let duplicate = handler
10983 .handle_control(
10984 ConnectionId::new(2),
10985 hello_frame("aft", PROTOCOL_VERSION, 2),
10986 )
10987 .unwrap();
10988
10989 assert_eq!(duplicate[0].header.ty, FrameType::Error);
10990 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
10991 let registration = registry.get_module("aft").unwrap().unwrap();
10992 assert_eq!(registration.connection_id, ConnectionId::new(1));
10993 }
10994
10995 #[test]
10996 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
10997 let registry = Arc::new(Registry::default());
10998 let forwarding = Arc::new(ForwardingTable::default());
10999 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
11000 let handler =
11001 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11002 .with_process_liveness(process_liveness);
11003 let (ctx, route_channel, route_epoch) =
11004 bind_liveness_route(®istry, &forwarding, "aft-dead");
11005 let responses = handler
11006 .handle_route_poll(
11007 &ctx,
11008 route_poll_frame(41, PollKind::Liveness, route_channel),
11009 route_channel,
11010 route_epoch,
11011 PollKind::Liveness,
11012 )
11013 .unwrap();
11014
11015 assert_eq!(responses.len(), 1);
11016 assert_eq!(responses[0].header.ty, FrameType::Response);
11017 assert_route_poll_liveness(&responses[0], false);
11018 }
11019
11020 #[test]
11021 fn liveness_poll_without_process_source_uses_bound_route() {
11022 let registry = Arc::new(Registry::default());
11023 let forwarding = Arc::new(ForwardingTable::default());
11024 let handler =
11025 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11026 let (ctx, route_channel, route_epoch) =
11027 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
11028 let responses = handler
11029 .handle_route_poll(
11030 &ctx,
11031 route_poll_frame(42, PollKind::Liveness, route_channel),
11032 route_channel,
11033 route_epoch,
11034 PollKind::Liveness,
11035 )
11036 .unwrap();
11037
11038 assert_route_poll_liveness(&responses[0], true);
11039 }
11040
11041 #[test]
11042 fn liveness_poll_untracked_process_source_uses_bound_route() {
11043 let registry = Arc::new(Registry::default());
11044 let forwarding = Arc::new(ForwardingTable::default());
11045 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
11046 let handler =
11047 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11048 .with_process_liveness(process_liveness);
11049 let (ctx, route_channel, route_epoch) =
11050 bind_liveness_route(®istry, &forwarding, "aft-untracked");
11051 let responses = handler
11052 .handle_route_poll(
11053 &ctx,
11054 route_poll_frame(43, PollKind::Liveness, route_channel),
11055 route_channel,
11056 route_epoch,
11057 PollKind::Liveness,
11058 )
11059 .unwrap();
11060
11061 assert_route_poll_liveness(&responses[0], true);
11062 }
11063
11064 #[tokio::test]
11065 async fn unknown_op_returns_unknown_control_op() {
11066 let handler = ControlHandler::default();
11067 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
11068 let request = Frame::build(
11069 FrameType::Request,
11070 control_flags(),
11071 0,
11072 0,
11073 55,
11074 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
11075 )
11076 .unwrap();
11077
11078 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
11079
11080 assert_eq!(response.len(), 1);
11081 assert_eq!(response[0].header.ty, FrameType::Error);
11082 assert_eq!(response[0].header.corr, 55);
11083 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
11084 }
11085
11086 #[tokio::test]
11087 async fn supervisor_provenance_rejects_unknown_exact_module() {
11088 let handler = ControlHandler::default();
11089 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
11090 let request = Frame::build(
11091 FrameType::Request,
11092 control_flags(),
11093 0,
11094 0,
11095 57,
11096 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
11097 )
11098 .unwrap();
11099
11100 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
11101
11102 assert_eq!(response.len(), 1);
11103 assert_eq!(response[0].header.ty, FrameType::Error);
11104 assert_eq!(response[0].header.corr, 57);
11105 let error = parse_error(&response[0]);
11106 assert_eq!(error["code"], "unknown_module");
11107 assert_eq!(error["message"], "module_id 'missing' is not supervised");
11108 }
11109
11110 #[test]
11111 fn provenance_probe_override_keeps_handler_tests_deterministic() {
11112 let expected = subc_control::RunningImageAgreement::Unavailable {
11113 reason: subc_control::RunningImageUnavailableReason::HashFailed,
11114 };
11115 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
11116 assert_eq!(handler.provenance_probe_override, Some(expected));
11117 }
11118
11119 #[test]
11120 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
11121 let verdict = reload_verdict(
11122 std::path::Path::new("/bin/new"),
11123 Some(std::path::Path::new("/bin/old")),
11124 subc_control::RunningImageAgreement::Unavailable {
11125 reason: subc_control::RunningImageUnavailableReason::HashFailed,
11126 },
11127 );
11128 assert!(matches!(
11129 verdict.path,
11130 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
11131 if configured == std::path::Path::new("/bin/new")
11132 && spawned_from == std::path::Path::new("/bin/old")
11133 ));
11134 }
11135
11136 #[test]
11137 fn reload_verdict_detects_replaced_image_at_same_path() {
11138 let image = subc_control::RunningImageAgreement::Mismatch {
11139 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
11140 digest: "old".into(),
11141 },
11142 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
11143 digest: "new".into(),
11144 },
11145 };
11146 let verdict = reload_verdict(
11147 std::path::Path::new("/bin/same"),
11148 Some(std::path::Path::new("/bin/same")),
11149 image.clone(),
11150 );
11151 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
11152 assert_eq!(verdict.image, image);
11153 }
11154
11155 #[test]
11156 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
11157 let image = subc_control::RunningImageAgreement::Unavailable {
11158 reason: subc_control::RunningImageUnavailableReason::NotRunning,
11159 };
11160 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
11161 assert_eq!(
11162 verdict.path,
11163 subc_control::ReloadPathAgreement::Unavailable {
11164 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
11165 }
11166 );
11167 assert_eq!(verdict.image, image);
11168
11169 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
11170 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
11171 };
11172 let verdict = reload_verdict(
11173 std::path::Path::new("/bin/same"),
11174 Some(std::path::Path::new("/bin/same")),
11175 unconfirmed.clone(),
11176 );
11177 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
11178 assert_eq!(verdict.image, unconfirmed);
11179 }
11180
11181 #[test]
11182 fn reload_verdict_preserves_each_image_unavailability_reason() {
11183 use subc_control::RunningImageUnavailableReason as Reason;
11184
11185 for reason in [
11186 Reason::NotRunning,
11187 Reason::UnsupportedPlatform,
11188 Reason::RunningExecutableUnreadable,
11189 Reason::SpawnedPathUnreadable,
11190 Reason::HashFailed,
11191 Reason::ProcessIdentityUnconfirmed,
11192 Reason::Unknown("future_probe_reason".to_string()),
11193 ] {
11194 let image = subc_control::RunningImageAgreement::Unavailable {
11195 reason: reason.clone(),
11196 };
11197 let verdict = reload_verdict(
11198 std::path::Path::new("/bin/same"),
11199 Some(std::path::Path::new("/bin/same")),
11200 image.clone(),
11201 );
11202 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
11203 assert_eq!(verdict.image, image, "{reason:?}");
11204 }
11205 }
11206
11207 #[tokio::test]
11208 async fn malformed_control_bodies_return_invalid_control_body() {
11209 let handler = ControlHandler::default();
11210 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
11211
11212 for (corr, body) in [
11213 (56, br#"{"route_channel":1}"#.as_slice()),
11214 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
11215 (
11216 58,
11217 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
11218 ),
11219 ] {
11220 let request = Frame::build(
11221 FrameType::Request,
11222 control_flags(),
11223 0,
11224 0,
11225 corr,
11226 body.to_vec(),
11227 )
11228 .unwrap();
11229 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
11230
11231 assert_eq!(response.len(), 1);
11232 assert_eq!(response[0].header.ty, FrameType::Error);
11233 assert_eq!(response[0].header.corr, corr);
11234 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
11235 }
11236 }
11237
11238 #[tokio::test]
11239 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
11240 let registry = Arc::new(Registry::default());
11241 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
11242 let router = Router::with_control_handler(Arc::clone(&control));
11243 let connection = router.begin_connection();
11244 let (ctx, mut rx) = route_ctx(connection.id());
11245
11246 router
11247 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
11248 .await
11249 .unwrap();
11250 let response = rx.recv().await.unwrap();
11251 let ack = parse_ack(&response);
11252 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
11253 let channel = 1;
11254
11255 let goodbye =
11256 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
11257 router.route_for_connection(&ctx, goodbye).await.unwrap();
11258 assert!(rx.try_recv().is_err());
11259 assert!(registry.get_module("aft").unwrap().is_none());
11260
11261 router
11262 .route_for_connection(&ctx, channel_request(channel, 13))
11263 .await
11264 .unwrap();
11265 let error_frame = rx.recv().await.unwrap();
11266 assert_eq!(error_frame.header.ty, FrameType::Error);
11267 assert_eq!(error_frame.header.channel, channel);
11268 }
11269
11270 #[tokio::test]
11271 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
11272 let registry = Arc::new(Registry::default());
11273 let handler = ControlHandler::new(registry).with_capability_config(
11274 [("prov".to_string(), true), ("cons".to_string(), true)],
11275 BTreeMap::new(),
11276 );
11277 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
11278 register_capability_manifest(
11279 &handler,
11280 &provider_ctx,
11281 &mut provider_rx,
11282 capability_manifest("prov", &["thing/v1"], &[]),
11283 1,
11284 )
11285 .await;
11286 let mut consumer = capability_manifest("cons", &[], &[]);
11287 consumer.capabilities.as_mut().unwrap().requires.push(
11288 subc_protocol::manifest::CapabilityRequirement {
11289 capability: "thing/v1".to_string(),
11290 need: subc_protocol::manifest::CapabilityNeed::Required,
11291 },
11292 );
11293 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
11294 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
11295 assert_eq!(
11296 handler.capability_evaluator.verdict("cons", "thing/v1"),
11297 Some(CapabilityVerdict::Provided)
11298 );
11299 let (mut client_rx, _) = open_route_for_capability_test(
11300 &handler,
11301 &provider_ctx,
11302 &mut provider_rx,
11303 703,
11304 3,
11305 "prov",
11306 None,
11307 )
11308 .await;
11309 let goodbye =
11310 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
11311 handler
11312 .handle_control_frame(&provider_ctx, goodbye)
11313 .await
11314 .unwrap();
11315 assert_eq!(
11316 handler.capability_evaluator.verdict("cons", "thing/v1"),
11317 Some(CapabilityVerdict::NeverProvided)
11318 );
11319 let closed = client_rx
11320 .try_recv()
11321 .expect("GOODBYE pushes route.closed before route GOODBYE");
11322 assert!(
11323 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
11324 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
11325 );
11326 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
11327 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
11328 }
11329
11330 #[tokio::test]
11331 async fn dropping_router_connection_releases_registration() {
11332 let registry = Arc::new(Registry::default());
11333 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
11334 let router = Router::with_control_handler(control);
11335 let connection = router.begin_connection();
11336 let (ctx, mut rx) = route_ctx(connection.id());
11337
11338 router
11339 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
11340 .await
11341 .unwrap();
11342 let response = rx.recv().await.unwrap();
11343 let ack = parse_ack(&response);
11344 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
11345 assert!(registry.get_module("aft").unwrap().is_some());
11346
11347 drop(connection);
11348
11349 assert!(registry.get_module("aft").unwrap().is_none());
11350 assert_eq!(registry.active_registration_count().unwrap(), 0);
11351 }
11352
11353 fn capability_manifest(
11354 module_id: &str,
11355 provides: &[&str],
11356 must_never_reach: &[&str],
11357 ) -> ModuleManifest {
11358 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
11359 manifest.capabilities = Some(CapabilityDeclarations {
11360 provides: provides
11361 .iter()
11362 .map(|capability| (*capability).to_string())
11363 .collect(),
11364 requires: Vec::new(),
11365 must_never_reach: must_never_reach
11366 .iter()
11367 .map(|capability| (*capability).to_string())
11368 .collect(),
11369 });
11370 manifest
11371 }
11372
11373 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
11374 Frame::build(
11375 FrameType::Hello,
11376 control_flags(),
11377 0,
11378 0,
11379 corr,
11380 serde_json::to_vec(&ModuleHelloBody {
11381 protocol_ver: manifest.protocol_ver,
11382 manifest,
11383 control_ops: None,
11384 launch_nonce: None,
11385 })
11386 .expect("capability test HELLO serializes"),
11387 )
11388 .expect("capability test HELLO frame builds")
11389 }
11390
11391 fn catalog_update_with_capabilities_frame(
11392 corr: u64,
11393 capabilities: CapabilityDeclarations,
11394 ) -> Frame {
11395 Frame::build(
11396 FrameType::Request,
11397 control_flags(),
11398 0,
11399 0,
11400 corr,
11401 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
11402 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
11403 capabilities: Some(capabilities),
11404 ready: None,
11405 })
11406 .expect("capability catalog.update serializes"),
11407 )
11408 .expect("capability catalog.update frame builds")
11409 }
11410
11411 async fn register_capability_manifest(
11412 handler: &ControlHandler,
11413 ctx: &RouteCtx,
11414 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
11415 manifest: ModuleManifest,
11416 corr: u64,
11417 ) {
11418 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
11419 }
11420
11421 async fn open_route_for_capability_test(
11422 handler: &ControlHandler,
11423 target_ctx: &RouteCtx,
11424 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
11425 client_connection_id: u64,
11426 corr: u64,
11427 target_module_id: &str,
11428 consumer_identity: Option<ConsumerIdentity>,
11429 ) -> (
11430 mpsc::Receiver<crate::router::OutboundFrame>,
11431 ModuleControlRequest,
11432 ) {
11433 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
11434 let route_handler = handler.clone();
11435 let target_module_id = target_module_id.to_string();
11436 let route_task = tokio::spawn(async move {
11437 route_handler
11438 .handle_control_frame(
11439 &client_ctx,
11440 route_open_frame_with_admission_facts(
11441 corr,
11442 &target_module_id,
11443 unique_project_root("admission-facts"),
11444 consumer_identity,
11445 None,
11446 ),
11447 )
11448 .await
11449 .expect("capability test route.open succeeds")
11450 });
11451 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
11452 .await
11453 .expect("capability test route.open must reach route.bind")
11454 .expect("target control receiver stays open");
11455 let bind_request: ModuleControlRequest =
11456 serde_json::from_slice(&bind.body).expect("route.bind decodes");
11457 handler
11458 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
11459 .await
11460 .expect("capability test route.bind ACK succeeds");
11461 assert!(route_task.await.expect("route.open task joins").is_empty());
11462 let opened = client_rx
11463 .recv()
11464 .await
11465 .expect("successful route.open publishes a response");
11466 assert!(matches!(
11467 serde_json::from_slice::<ClientControlResponse>(&opened.body),
11468 Ok(ClientControlResponse::RouteOpen { .. })
11469 ));
11470 (client_rx, bind_request)
11471 }
11472
11473 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
11474 assert_eq!(frame.header.ty, FrameType::Push);
11475 assert_eq!(frame.header.channel, 0);
11476 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
11477 .expect("route.closed control push decodes");
11478 let ClientControlPush::RouteClosed { channels, .. } = &push else {
11479 panic!("expected route.closed");
11480 };
11481 assert_eq!(channels.len(), 1, "exactly one violating route closed");
11482 let channels = channels.clone();
11483 assert_eq!(
11484 push,
11485 ClientControlPush::RouteClosed {
11486 module_id: target_module_id.to_string(),
11487 channels,
11488 reason: RouteCloseReason::CapabilityDenied,
11489 drained: false,
11490 abandoned: 0,
11491 excluded_subscriptions: 0,
11492 terminal: Some(false),
11493 }
11494 );
11495 }
11496
11497 #[tokio::test]
11498 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
11499 let registry = Arc::new(Registry::default());
11500 let forwarding = Arc::new(ForwardingTable::default());
11501 let supervisor = SupervisorHandle::new();
11502 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11503 let handler =
11504 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11505 .with_supervisor(supervisor);
11506 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
11507 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
11508 register_capability_manifest(
11509 &handler,
11510 &target_ctx,
11511 &mut target_rx,
11512 capability_manifest("target", &["credentials-provider/v1"], &[]),
11513 1,
11514 )
11515 .await;
11516 register_capability_manifest(
11517 &handler,
11518 &opener_ctx,
11519 &mut opener_rx,
11520 capability_manifest("opener", &[], &["credentials-provider/v1"]),
11521 2,
11522 )
11523 .await;
11524
11525 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
11526 let replies = handler
11527 .handle_control_frame(
11528 &client_ctx,
11529 route_open_frame_with_admission_facts(
11530 3,
11531 "target",
11532 unique_project_root("admission-facts"),
11533 Some(ConsumerIdentity {
11534 module_id: "opener".to_string(),
11535 launch_nonce: "opener-nonce".to_string(),
11536 }),
11537 None,
11538 ),
11539 )
11540 .await
11541 .expect("denied route.open returns a typed frame");
11542 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
11543 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
11544 assert!(
11545 target_rx.try_recv().is_err(),
11546 "forbidden route.open must not relay route.bind"
11547 );
11548 }
11549
11550 #[tokio::test]
11551 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
11552 let registry = Arc::new(Registry::default());
11553 let forwarding = Arc::new(ForwardingTable::default());
11554 let supervisor = SupervisorHandle::new();
11555 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11556 let handler =
11557 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11558 .with_supervisor(supervisor);
11559 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
11560 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
11561 register_capability_manifest(
11562 &handler,
11563 &target_ctx,
11564 &mut target_rx,
11565 capability_manifest("target", &["credentials-provider/v1"], &[]),
11566 1,
11567 )
11568 .await;
11569 register_capability_manifest(
11570 &handler,
11571 &old_opener_ctx,
11572 &mut old_opener_rx,
11573 capability_manifest("opener", &[], &[]),
11574 2,
11575 )
11576 .await;
11577 let (mut client_rx, _) = open_route_for_capability_test(
11578 &handler,
11579 &target_ctx,
11580 &mut target_rx,
11581 712,
11582 3,
11583 "target",
11584 Some(ConsumerIdentity {
11585 module_id: "opener".to_string(),
11586 launch_nonce: "opener-nonce".to_string(),
11587 }),
11588 )
11589 .await;
11590 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
11591
11592 handler
11593 .cleanup_connection(old_opener_ctx.connection_id)
11594 .expect("old opener registration cleans up");
11595 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
11596 register_capability_manifest(
11597 &handler,
11598 &new_opener_ctx,
11599 &mut new_opener_rx,
11600 capability_manifest("opener", &[], &["credentials-provider/v1"]),
11601 4,
11602 )
11603 .await;
11604
11605 assert_capability_denied_push(
11606 client_rx
11607 .try_recv()
11608 .expect("HELLO deny addition must emit route.closed")
11609 .frame,
11610 "target",
11611 );
11612 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
11613 assert!(matches!(
11614 target_rx.try_recv(),
11615 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
11616 ));
11617 }
11618
11619 #[tokio::test]
11620 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
11621 let registry = Arc::new(Registry::default());
11622 let forwarding = Arc::new(ForwardingTable::default());
11623 let supervisor = SupervisorHandle::new();
11624 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11625 let handler =
11626 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11627 .with_supervisor(supervisor);
11628 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
11629 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
11630 register_capability_manifest(
11631 &handler,
11632 &target_ctx,
11633 &mut target_rx,
11634 capability_manifest("target", &[], &[]),
11635 1,
11636 )
11637 .await;
11638 register_capability_manifest(
11639 &handler,
11640 &opener_ctx,
11641 &mut opener_rx,
11642 capability_manifest("opener", &[], &["credentials-provider/v1"]),
11643 2,
11644 )
11645 .await;
11646 let (mut client_rx, _) = open_route_for_capability_test(
11647 &handler,
11648 &target_ctx,
11649 &mut target_rx,
11650 722,
11651 3,
11652 "target",
11653 Some(ConsumerIdentity {
11654 module_id: "opener".to_string(),
11655 launch_nonce: "opener-nonce".to_string(),
11656 }),
11657 )
11658 .await;
11659 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
11660
11661 let replies = handler
11662 .handle_control_frame(
11663 &target_ctx,
11664 catalog_update_with_capabilities_frame(
11665 4,
11666 CapabilityDeclarations {
11667 provides: vec!["credentials-provider/v1".to_string()],
11668 requires: Vec::new(),
11669 must_never_reach: Vec::new(),
11670 },
11671 ),
11672 )
11673 .await
11674 .expect("claim catalog.update succeeds");
11675 assert!(matches!(
11676 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
11677 Ok(ModuleControlResponseToModule::CatalogUpdate {})
11678 ));
11679 assert_capability_denied_push(
11680 client_rx
11681 .try_recv()
11682 .expect("claim addition must emit route.closed")
11683 .frame,
11684 "target",
11685 );
11686 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
11687 assert!(matches!(
11688 target_rx.try_recv(),
11689 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
11690 ));
11691 }
11692
11693 #[tokio::test]
11694 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
11695 let registry = Arc::new(Registry::default());
11696 let forwarding = Arc::new(ForwardingTable::default());
11697 let supervisor = SupervisorHandle::new();
11698 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11699 let handler =
11700 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11701 .with_supervisor(supervisor);
11702 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
11703 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
11704 register_capability_manifest(
11705 &handler,
11706 &target_ctx,
11707 &mut target_rx,
11708 capability_manifest("target", &["credentials-provider/v1"], &[]),
11709 1,
11710 )
11711 .await;
11712 register_capability_manifest(
11713 &handler,
11714 &opener_ctx,
11715 &mut opener_rx,
11716 capability_manifest("opener", &[], &[]),
11717 2,
11718 )
11719 .await;
11720 let (mut client_rx, _) = open_route_for_capability_test(
11721 &handler,
11722 &target_ctx,
11723 &mut target_rx,
11724 732,
11725 3,
11726 "target",
11727 Some(ConsumerIdentity {
11728 module_id: "opener".to_string(),
11729 launch_nonce: "opener-nonce".to_string(),
11730 }),
11731 )
11732 .await;
11733 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
11734
11735 handler
11736 .handle_control_frame(
11737 &target_ctx,
11738 catalog_update_with_capabilities_frame(
11739 4,
11740 CapabilityDeclarations {
11741 provides: Vec::new(),
11742 requires: Vec::new(),
11743 must_never_reach: Vec::new(),
11744 },
11745 ),
11746 )
11747 .await
11748 .expect("claim removal catalog.update succeeds");
11749 assert_eq!(
11750 forwarding.active_binding_count().unwrap(),
11751 1,
11752 "removing an attested target claim must leave the route census unchanged"
11753 );
11754 assert!(
11755 client_rx.try_recv().is_err(),
11756 "claim removal must not emit route.closed capability_denied"
11757 );
11758 assert!(
11759 target_rx.try_recv().is_err(),
11760 "claim removal must not send the target a route GOODBYE"
11761 );
11762 }
11763
11764 #[tokio::test]
11767 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
11768 let registry = Arc::new(Registry::default());
11769 let forwarding = Arc::new(ForwardingTable::default());
11770 let supervisor = SupervisorHandle::new();
11771 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11772 let handler =
11773 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11774 .with_supervisor(supervisor);
11775 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
11776 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
11777 register_capability_manifest(
11778 &handler,
11779 &target_ctx,
11780 &mut target_rx,
11781 capability_manifest("target", &["credentials-provider/v1"], &[]),
11782 1,
11783 )
11784 .await;
11785 register_capability_manifest(
11786 &handler,
11787 &opener_ctx,
11788 &mut opener_rx,
11789 capability_manifest("opener", &[], &["credentials-provider/v1"]),
11790 2,
11791 )
11792 .await;
11793
11794 let (_client_rx, bind) = open_route_for_capability_test(
11795 &handler,
11796 &target_ctx,
11797 &mut target_rx,
11798 742,
11799 3,
11800 "target",
11801 None,
11802 )
11803 .await;
11804 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
11805 panic!("direct scope-honesty route must bind");
11806 };
11807 assert_eq!(principal, Some(Principal::Direct));
11808 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
11809 }
11810
11811 #[tokio::test]
11814 async fn must_never_reach_self_route_is_capability_forbidden() {
11815 let registry = Arc::new(Registry::default());
11816 let forwarding = Arc::new(ForwardingTable::default());
11817 let supervisor = SupervisorHandle::new();
11818 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
11819 let handler =
11820 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11821 .with_supervisor(supervisor);
11822 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
11823 register_capability_manifest(
11824 &handler,
11825 &self_ctx,
11826 &mut self_rx,
11827 capability_manifest(
11828 "self-provider",
11829 &["credentials-provider/v1"],
11830 &["credentials-provider/v1"],
11831 ),
11832 1,
11833 )
11834 .await;
11835
11836 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
11837 let replies = handler
11838 .handle_control_frame(
11839 &client_ctx,
11840 route_open_frame_with_admission_facts(
11841 2,
11842 "self-provider",
11843 unique_project_root("admission-facts"),
11844 Some(ConsumerIdentity {
11845 module_id: "self-provider".to_string(),
11846 launch_nonce: "self-nonce".to_string(),
11847 }),
11848 None,
11849 ),
11850 )
11851 .await
11852 .expect("self-route refusal returns a typed frame");
11853 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
11854 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
11855 assert!(
11856 self_rx.try_recv().is_err(),
11857 "self denial must not relay route.bind"
11858 );
11859 }
11860
11861 #[test]
11862 fn unsupported_channel_zero_frame_returns_error() {
11863 let handler = ControlHandler::default();
11864 let request = Frame::build(
11865 FrameType::Request,
11866 control_flags(),
11867 0,
11868 0,
11869 21,
11870 b"opaque".to_vec(),
11871 )
11872 .unwrap();
11873
11874 let response = handler
11875 .handle_control(ConnectionId::new(1), request)
11876 .unwrap();
11877
11878 assert_eq!(response[0].header.ty, FrameType::Error);
11879 assert_eq!(
11880 parse_error(&response[0])["code"],
11881 "unsupported_control_frame"
11882 );
11883 }
11884
11885 mod swap {
11890 use super::*;
11891
11892 const INCUMBENT: ConnectionId = ConnectionId::new(30);
11893 const CANDIDATE: ConnectionId = ConnectionId::new(40);
11894
11895 struct Swap {
11896 registry: Arc<Registry>,
11897 forwarding: Arc<ForwardingTable>,
11898 handler: ControlHandler,
11899 incumbent_ctx: RouteCtx,
11900 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
11901 candidate_ctx: RouteCtx,
11902 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
11903 }
11904
11905 async fn swap_with_incumbent() -> Swap {
11906 let registry = Arc::new(Registry::default());
11907 let forwarding = Arc::new(ForwardingTable::default());
11908 let handler =
11909 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11910 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
11911 hello_via_sink(
11912 &handler,
11913 &incumbent_ctx,
11914 &mut incumbent_rx,
11915 hello_frame("aft", PROTOCOL_VERSION, 7),
11916 )
11917 .await;
11918 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
11919 Swap {
11920 registry,
11921 forwarding,
11922 handler,
11923 incumbent_ctx,
11924 incumbent_rx,
11925 candidate_ctx,
11926 candidate_rx,
11927 }
11928 }
11929
11930 fn register_candidate(swap: &Swap, ready: Option<bool>) {
11931 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
11932 candidate_manifest.ready = ready;
11933 let registration = swap
11934 .registry
11935 .register_candidate_with_control_ops(
11936 candidate_manifest,
11937 PROTOCOL_VERSION,
11938 CANDIDATE,
11939 module_baseline_control_ops(),
11940 )
11941 .unwrap();
11942 swap.forwarding
11943 .register_candidate_module_connection(
11944 CANDIDATE,
11945 "aft".to_string(),
11946 PROTOCOL_VERSION,
11947 manifest_concurrency(®istration.manifest),
11948 swap.candidate_ctx.egress.clone(),
11949 )
11950 .unwrap();
11951 }
11952
11953 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
11954 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
11955 swap.registry.promote_candidate("aft").unwrap().unwrap();
11956 cutover.incumbent.unwrap()
11957 }
11958
11959 fn keyed_total(counters: &Value, key: &str) -> u64 {
11960 counters[key]
11961 .as_object()
11962 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
11963 .unwrap_or(0)
11964 }
11965
11966 #[tokio::test]
11972 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
11973 let mut swap = swap_with_incumbent().await;
11974 let handler = swap.handler.clone();
11975
11976 let cotenant = ConnectionId::new(31);
11978 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
11979 let (cotenant_task, cotenant_bind) = relay_route_open(
11980 &handler,
11981 cotenant,
11982 &cotenant_ctx.egress,
11983 &mut swap.incumbent_rx,
11984 100,
11985 "aft",
11986 "swap-cotenant",
11987 )
11988 .await;
11989 handler
11990 .handle_control_frame(
11991 &swap.incumbent_ctx,
11992 route_bind_ack(cotenant_bind.header.corr),
11993 )
11994 .await
11995 .unwrap();
11996 assert!(cotenant_task.await.unwrap().is_empty());
11997 let (cotenant_channel, cotenant_epoch) =
11998 published_route(&cotenant_rx.recv().await.unwrap());
11999
12000 let caller = ConnectionId::new(32);
12002 let (caller_ctx, mut caller_rx) = route_ctx(caller);
12003 let (caller_task, caller_bind) = relay_route_open(
12004 &handler,
12005 caller,
12006 &caller_ctx.egress,
12007 &mut swap.incumbent_rx,
12008 101,
12009 "aft",
12010 "swap-caller",
12011 )
12012 .await;
12013 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
12014
12015 register_candidate(&swap, None);
12016 cutover(&swap);
12017
12018 let ack = handler
12020 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
12021 .await;
12022 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
12023 if module_loop_error.is_some() {
12024 handler.cleanup_connection(INCUMBENT).unwrap();
12027 }
12028
12029 assert!(
12031 cotenant_rx.try_recv().is_err(),
12032 "the co-tenant route on the incumbent was torn down by one late ack: \
12033 {module_loop_error:?}"
12034 );
12035 assert!(matches!(
12036 swap.forwarding
12037 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
12038 .unwrap(),
12039 DataRoute::Client(DataRouteState::Bound(_))
12040 ));
12041 assert_eq!(module_loop_error, None);
12042 assert!(swap
12043 .registry
12044 .get_module_by_connection(INCUMBENT)
12045 .unwrap()
12046 .is_some());
12047
12048 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
12050 .await
12051 .expect("the incumbent is told to drop the abandoned binding")
12052 .unwrap()
12053 .frame;
12054 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
12055 assert_eq!(goodbye.header.channel, abandoned_channel);
12056 assert_eq!(goodbye.header.epoch, abandoned_epoch);
12057 assert!(swap.incumbent_rx.try_recv().is_err());
12058
12059 let response = caller_task.await.unwrap();
12061 assert_eq!(response.len(), 1);
12062 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
12063 assert!(caller_rx.try_recv().is_err());
12064
12065 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
12069 let counters = handler.counters().snapshot();
12070 assert_eq!(
12071 keyed_total(&counters, "route_open_accepted_by_principal"),
12072 1
12073 );
12074 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
12075 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
12076 }
12077
12078 #[tokio::test]
12082 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
12083 let mut swap = swap_with_incumbent().await;
12084 register_candidate(&swap, None);
12085 let incumbent = cutover(&swap);
12086 swap.forwarding
12087 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
12088 .unwrap()
12089 .expect("the incumbent is still registered");
12090
12091 let client = ConnectionId::new(33);
12092 let (client_ctx, mut client_rx) = route_ctx(client);
12093 let route_handler = swap.handler.clone();
12094 let open_ctx = RouteCtx {
12095 connection_id: client,
12096 egress: client_ctx.egress.clone(),
12097 };
12098 let mut route_task = tokio::spawn(async move {
12099 route_handler
12100 .handle_control_frame(
12101 &open_ctx,
12102 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
12103 )
12104 .await
12105 .unwrap()
12106 });
12107 let bind = tokio::select! {
12108 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
12109 response = &mut route_task => {
12110 let response = response.unwrap();
12111 panic!(
12112 "post-cutover route.open was refused instead of relayed to the candidate: {}",
12113 parse_error(&response[0])["code"]
12114 );
12115 }
12116 };
12117 swap.handler
12118 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
12119 .await
12120 .unwrap();
12121 assert!(route_task.await.unwrap().is_empty());
12122 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
12123 match swap
12124 .forwarding
12125 .lookup_data_route(client, channel, epoch)
12126 .unwrap()
12127 {
12128 DataRoute::Client(DataRouteState::Bound(route)) => {
12129 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
12130 }
12131 other => panic!("expected a bound route on the candidate, got {other:?}"),
12132 }
12133 assert!(swap.incumbent_rx.try_recv().is_err());
12134 }
12135
12136 #[tokio::test]
12141 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
12142 let swap = swap_with_incumbent().await;
12143 register_candidate(&swap, Some(false));
12144 let update = Frame::build(
12145 FrameType::Request,
12146 control_flags(),
12147 0,
12148 0,
12149 55,
12150 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
12151 provides: manifest("aft", PROTOCOL_VERSION).provides,
12152 capabilities: None,
12153 ready: Some(true),
12154 })
12155 .unwrap(),
12156 )
12157 .unwrap();
12158
12159 let replies = swap
12160 .handler
12161 .handle_control_frame(&swap.candidate_ctx, update)
12162 .await
12163 .unwrap();
12164
12165 assert_eq!(replies.len(), 1);
12166 assert_eq!(
12167 replies[0].header.ty,
12168 FrameType::Response,
12169 "candidate catalog.update was refused: {:?}",
12170 serde_json::from_slice::<Value>(&replies[0].body).ok()
12171 );
12172 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
12173 assert_eq!(
12174 swap.registry
12175 .get_module("aft")
12176 .unwrap()
12177 .unwrap()
12178 .connection_id,
12179 INCUMBENT
12180 );
12181 }
12182 }
12183
12184 mod swap_admission {
12188 use super::*;
12189
12190 const INCUMBENT_NONCE: &str = "incumbent-nonce";
12191 const CANDIDATE_NONCE: &str = "candidate-nonce";
12192
12193 fn handler_with_incumbent(
12194 module_id: &str,
12195 reserved: bool,
12196 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
12197 let registry = Arc::new(Registry::default());
12198 let supervisor = SupervisorHandle::new();
12199 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
12200 if reserved {
12201 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
12202 }
12203 let handler =
12204 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
12205 let incumbent = handler
12206 .handle_control(
12207 ConnectionId::new(1),
12208 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
12209 )
12210 .unwrap();
12211 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
12212 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
12213 (registry, supervisor, handler)
12214 }
12215
12216 #[test]
12222 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
12223 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
12224
12225 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
12226 let replies = handler
12227 .handle_control(
12228 ConnectionId::new(connection),
12229 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
12230 )
12231 .unwrap();
12232 assert_eq!(replies[0].header.ty, FrameType::Error);
12233 assert_eq!(
12234 parse_error(&replies[0])["code"],
12235 "swap_token_invalid",
12236 "nonce {nonce:?}"
12237 );
12238 }
12239 assert!(registry.get_candidate("aft").unwrap().is_none());
12240 assert_eq!(
12241 registry.get_module("aft").unwrap().unwrap().connection_id,
12242 ConnectionId::new(1)
12243 );
12244
12245 let admitted = handler
12248 .handle_control(
12249 ConnectionId::new(4),
12250 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
12251 )
12252 .unwrap();
12253 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
12254 assert_eq!(
12255 registry
12256 .get_candidate("aft")
12257 .unwrap()
12258 .unwrap()
12259 .connection_id,
12260 ConnectionId::new(4)
12261 );
12262 assert_eq!(
12263 registry.get_module("aft").unwrap().unwrap().connection_id,
12264 ConnectionId::new(1),
12265 "the candidate must not take the active slot"
12266 );
12267 let replayed = handler
12268 .handle_control(
12269 ConnectionId::new(5),
12270 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
12271 )
12272 .unwrap();
12273 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
12274
12275 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
12279 let squatter = handler
12280 .handle_control(
12281 ConnectionId::new(6),
12282 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
12283 )
12284 .unwrap();
12285 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
12286 assert!(
12287 registry.get_module("aft").unwrap().is_none(),
12288 "a squatter took the active slot of an id being swapped"
12289 );
12290 }
12291
12292 #[test]
12297 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
12298 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
12299
12300 let replies = handler
12301 .handle_control(
12302 ConnectionId::new(2),
12303 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
12304 )
12305 .unwrap();
12306
12307 assert_eq!(
12308 replies[0].header.ty,
12309 FrameType::HelloAck,
12310 "reserved candidate refused: {:?}",
12311 serde_json::from_slice::<Value>(&replies[0].body).ok()
12312 );
12313 assert_eq!(
12314 registry
12315 .get_candidate("vault")
12316 .unwrap()
12317 .unwrap()
12318 .connection_id,
12319 ConnectionId::new(2)
12320 );
12321 }
12322
12323 #[test]
12326 fn without_an_open_swap_the_ordinary_gates_decide() {
12327 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
12328 supervisor.close_swap("vault");
12329
12330 let candidate = handler
12331 .handle_control(
12332 ConnectionId::new(2),
12333 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
12334 )
12335 .unwrap();
12336 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
12337 let duplicate = handler
12338 .handle_control(
12339 ConnectionId::new(3),
12340 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
12341 )
12342 .unwrap();
12343 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
12344 assert!(registry.get_candidate("vault").unwrap().is_none());
12345 }
12346 }
12347
12348 mod scopes {
12352 use subc_protocol::scope::{
12353 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
12354 ScopeStatus,
12355 };
12356
12357 use super::*;
12358
12359 const OWNER: &str = "prefrontal-core";
12360
12361 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
12362 ScopeRecord {
12363 scope_ref: scope_ref.to_string(),
12364 scope_epoch,
12365 kind: ScopeKind::Head,
12366 parent: None,
12367 child_owners: Vec::new(),
12368 carriers: Vec::new(),
12369 attributes: Default::default(),
12370 }
12371 }
12372
12373 async fn call(
12374 handler: &ControlHandler,
12375 ctx: &RouteCtx,
12376 request: &ModuleControlRequestFromModule,
12377 ) -> Frame {
12378 let body = serde_json::to_vec(request).unwrap();
12379 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
12380 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
12381 assert_eq!(replies.len(), 1, "{replies:?}");
12382 replies.pop().unwrap()
12383 }
12384
12385 async fn sync(
12386 handler: &ControlHandler,
12387 ctx: &RouteCtx,
12388 generation: u64,
12389 scopes: Vec<ScopeRecord>,
12390 ) -> Result<ModuleControlResponseToModule, String> {
12391 let reply = call(
12392 handler,
12393 ctx,
12394 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
12395 )
12396 .await;
12397 match reply.header.ty {
12398 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
12399 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
12400 }
12401 }
12402
12403 async fn describe(
12404 handler: &ControlHandler,
12405 ctx: &RouteCtx,
12406 owner: &str,
12407 scope_ref: &str,
12408 ) -> ModuleControlResponseToModule {
12409 let reply = call(
12410 handler,
12411 ctx,
12412 &ModuleControlRequestFromModule::ScopeDescribe {
12413 owner: Principal::Reserved {
12414 module_id: owner.to_string(),
12415 },
12416 scope_ref: scope_ref.to_string(),
12417 },
12418 )
12419 .await;
12420 assert_eq!(
12421 reply.header.ty,
12422 FrameType::Response,
12423 "{:?}",
12424 parse_error(&reply)
12425 );
12426 serde_json::from_slice(&reply.body).unwrap()
12427 }
12428
12429 async fn module(
12431 handler: &ControlHandler,
12432 connection: u64,
12433 module_id: &str,
12434 nonce: Option<&str>,
12435 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
12436 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
12437 hello_via_sink(
12438 handler,
12439 &ctx,
12440 &mut rx,
12441 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
12442 )
12443 .await;
12444 (ctx, rx)
12445 }
12446
12447 #[tokio::test]
12450 async fn a_client_connection_cannot_sync_or_describe() {
12451 let handler = ControlHandler::new(Arc::new(Registry::default()));
12452 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
12453 for request in [
12454 ModuleControlRequestFromModule::ScopeSync {
12455 generation: 1,
12456 scopes: vec![head("s", 1)],
12457 },
12458 ModuleControlRequestFromModule::ScopeDescribe {
12459 owner: Principal::Direct,
12460 scope_ref: "s".to_string(),
12461 },
12462 ] {
12463 let reply = call(&handler, &ctx, &request).await;
12464 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
12465 }
12466 assert!(
12467 !handler
12468 .scopes
12469 .read()
12470 .unwrap()
12471 .describe(
12472 &Principal::Reserved {
12473 module_id: OWNER.to_string()
12474 },
12475 "s"
12476 )
12477 .owner_synced
12478 );
12479 }
12480
12481 #[tokio::test]
12484 async fn a_module_without_a_supervised_launch_cannot_sync() {
12485 let handler = ControlHandler::new(Arc::new(Registry::default()));
12486 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
12487 assert_eq!(
12488 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
12489 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
12490 );
12491 }
12492
12493 #[tokio::test]
12494 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
12495 let supervisor = SupervisorHandle::new();
12496 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
12497 let handler = ControlHandler::new(Arc::new(Registry::default()))
12498 .with_supervisor(supervisor.clone());
12499 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
12500 sync(&handler, &incumbent, 1, vec![head("s", 1)])
12501 .await
12502 .expect("the current launch syncs");
12503
12504 supervisor.open_swap(OWNER, "n2".to_string());
12507 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
12508 assert_eq!(
12509 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
12510 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
12511 );
12512 sync(&handler, &incumbent, 2, vec![head("s", 1)])
12513 .await
12514 .expect("the serving owner syncs during the swap");
12515
12516 supervisor.close_swap(OWNER);
12519 assert_eq!(
12520 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
12521 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
12522 );
12523 sync(&handler, &incumbent, 3, vec![head("s", 1)])
12524 .await
12525 .expect("the serving owner syncs after the rollback");
12526 handler.cleanup_connection(candidate.connection_id).unwrap();
12527
12528 supervisor.open_swap(OWNER, "n3".to_string());
12533 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
12534 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
12535 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
12536 .await
12537 .expect("the promoted launch takes authority");
12538 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
12539 panic!("unexpected reply {reply:?}");
12540 };
12541 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
12542 assert_eq!(
12543 sync(&handler, &incumbent, 4, Vec::new()).await,
12544 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
12545 );
12546 }
12547
12548 #[tokio::test]
12551 async fn closing_the_authority_connection_frees_sync_authority() {
12552 let supervisor = SupervisorHandle::new();
12553 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
12554 let handler = ControlHandler::new(Arc::new(Registry::default()))
12555 .with_supervisor(supervisor.clone());
12556 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
12557 sync(&handler, &first, 10, vec![head("s", 1)])
12558 .await
12559 .unwrap();
12560 handler.cleanup_connection(first.connection_id).unwrap();
12561
12562 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
12563 sync(&handler, &second, 1, vec![head("s", 1)])
12564 .await
12565 .expect("the next connection takes the released authority");
12566 }
12567
12568 #[tokio::test]
12569 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
12570 let supervisor = SupervisorHandle::new();
12571 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
12572 let handler =
12573 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
12574 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
12575 sync(&handler, &first, 10, vec![head("s", 1)])
12576 .await
12577 .unwrap();
12578 handler
12579 .handle_control_frame(
12580 &first,
12581 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
12582 )
12583 .await
12584 .unwrap();
12585 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
12586 sync(&handler, &second, 1, vec![head("s", 1)])
12587 .await
12588 .expect("GOODBYE releases authority even if the old socket remains open");
12589 }
12590
12591 #[tokio::test]
12592 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
12593 let registry = Arc::new(Registry::default());
12594 let supervisor_handle = SupervisorHandle::new();
12595 let supervisor = Supervisor::new(Arc::clone(®istry), RestartPolicy::default())
12596 .with_handle(supervisor_handle.clone())
12597 .with_daemon_incarnation("incarnation-7".to_string());
12598 supervisor
12601 .supervise_configured(
12602 ModuleSpec {
12603 module_id: OWNER.to_string(),
12604 program: PathBuf::from("/nonexistent/prefrontal-core"),
12605 args: Vec::new(),
12606 env: Vec::new(),
12607 reserved: false,
12608 reserved_prefixes: Vec::new(),
12609 protocol: ModuleProtocol::Subc,
12610 overlap: Default::default(),
12611 },
12612 false,
12613 )
12614 .unwrap();
12615 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
12616 let handler =
12617 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12618 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
12619
12620 let ModuleControlResponseToModule::ScopeDescribe {
12622 status,
12623 daemon_incarnation,
12624 owner_synced,
12625 owner_configured,
12626 scope,
12627 ..
12628 } = describe(&handler, &reader, OWNER, "s").await
12629 else {
12630 panic!("not a describe reply");
12631 };
12632 assert_eq!(status, ScopeStatus::NotLive);
12633 assert_eq!(daemon_incarnation, "incarnation-7");
12634 assert!(!owner_synced);
12635 assert!(owner_configured);
12636 assert!(scope.is_none());
12637
12638 let ModuleControlResponseToModule::ScopeDescribe {
12641 status,
12642 owner_configured,
12643 ..
12644 } = describe(&handler, &reader, "ghost", "s").await
12645 else {
12646 panic!("not a describe reply");
12647 };
12648 assert_eq!(status, ScopeStatus::NotLive);
12649 assert!(!owner_configured);
12650
12651 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
12653 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
12654 let ModuleControlResponseToModule::ScopeDescribe {
12655 status,
12656 scope_epoch,
12657 owner_synced,
12658 scope,
12659 ..
12660 } = describe(&handler, &reader, OWNER, "s").await
12661 else {
12662 panic!("not a describe reply");
12663 };
12664 assert_eq!(status, ScopeStatus::Live);
12665 assert_eq!(scope_epoch, Some(4));
12666 assert!(owner_synced);
12667 let stamp = scope.expect("a live scope carries its stamp");
12668 assert!(
12669 stamp.owner_authorized,
12670 "prefrontal-core is the default authority"
12671 );
12672 assert_eq!(stamp.kind, ScopeKind::Head);
12673 }
12674
12675 #[tokio::test]
12676 async fn scope_authority_owners_decides_owner_authorized() {
12677 let supervisor = SupervisorHandle::new();
12678 supervisor.set_spawn_nonce("broca", "b1".to_string());
12679 let handler = ControlHandler::new(Arc::new(Registry::default()))
12680 .with_supervisor(supervisor)
12681 .with_scope_authority_owners(vec!["broca".to_string()]);
12682 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
12683 let mut gated = head("s", 1);
12684 gated.attributes.agent_id = Some("agent".to_string());
12685 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
12686 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
12687 describe(&handler, &broca, "broca", "s").await
12688 else {
12689 panic!("not a describe reply");
12690 };
12691 assert!(scope.unwrap().owner_authorized);
12692 }
12693
12694 #[tokio::test]
12698 async fn scope_ops_and_the_scopes_capability_are_advertised() {
12699 let handler = ControlHandler::new(Arc::new(Registry::default()));
12700 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
12701 let ack = hello_via_sink(
12702 &handler,
12703 &ctx,
12704 &mut rx,
12705 hello_frame("m", PROTOCOL_VERSION, 1),
12706 )
12707 .await;
12708 let ack = parse_ack(&ack);
12709 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
12710 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
12711 }
12712 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
12713
12714 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
12715 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12716 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12717 let reply = handler
12718 .handle_control_frame(&client, frame)
12719 .await
12720 .unwrap()
12721 .pop()
12722 .unwrap();
12723 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12724 serde_json::from_slice(&reply.body).unwrap()
12725 else {
12726 panic!("not a server.describe reply");
12727 };
12728 assert!(
12729 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
12730 "{capabilities:?}"
12731 );
12732 }
12733
12734 const PLEXUS: &str = "plexus";
12737 const OTHER: &str = "other";
12738 const AFT: &str = "aft";
12739 const BROCA: &str = "broca";
12740 const MAGIC: &str = "magic-context";
12741
12742 fn nonce(module_id: &str) -> String {
12743 format!("nonce-{module_id}")
12744 }
12745
12746 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
12747 let (tx, rx) = mpsc::channel(64);
12748 (
12749 RouteCtx {
12750 connection_id: ConnectionId::new(connection),
12751 egress: FrameSink::new(tx),
12752 },
12753 rx,
12754 )
12755 }
12756
12757 struct Rig {
12761 handler: ControlHandler,
12762 forwarding: Arc<ForwardingTable>,
12763 owner: RouteCtx,
12764 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12765 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
12766 generation: u64,
12767 next_connection: u64,
12768 _supervisor: Supervisor,
12769 }
12770
12771 async fn rig() -> Rig {
12772 rig_with_flow_support(true).await
12773 }
12774
12775 async fn rig_with_flow_support(flow_support: bool) -> Rig {
12776 let registry = Arc::new(Registry::default());
12777 let forwarding = Arc::new(ForwardingTable::default());
12778 let supervisor_handle = SupervisorHandle::new();
12779 let supervisor = Supervisor::new(Arc::clone(®istry), RestartPolicy::default())
12780 .with_handle(supervisor_handle.clone());
12781 supervisor
12782 .supervise_configured(
12783 ModuleSpec {
12784 module_id: OWNER.to_string(),
12785 program: PathBuf::from("/nonexistent/prefrontal-core"),
12786 args: Vec::new(),
12787 env: Vec::new(),
12788 reserved: false,
12789 reserved_prefixes: Vec::new(),
12790 protocol: ModuleProtocol::Subc,
12791 overlap: Default::default(),
12792 },
12793 false,
12794 )
12795 .unwrap();
12796 for module_id in [OWNER, AFT, BROCA, MAGIC] {
12797 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
12798 }
12799 let handler =
12800 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12801 .with_supervisor(supervisor_handle);
12802 let (owner, mut owner_rx) = wide_ctx(1);
12803 hello_via_sink(
12804 &handler,
12805 &owner,
12806 &mut owner_rx,
12807 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
12808 )
12809 .await;
12810 let mut modules = BTreeMap::new();
12811 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
12812 let (ctx, mut rx) = wide_ctx(connection);
12813 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
12814 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
12815 body["manifest"]["provenance"] =
12819 serde_json::json!({"wire_crate_version": "0.29.0"});
12820 if flow_support {
12821 body["manifest"]["capabilities"] =
12822 serde_json::json!({"provides": ["flow-scopes/v1"]});
12823 }
12824 let hello = Frame::build(
12825 FrameType::Hello,
12826 control_flags(),
12827 0,
12828 0,
12829 connection,
12830 serde_json::to_vec(&body).unwrap(),
12831 )
12832 .unwrap();
12833 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
12834 modules.insert(module_id.to_string(), (ctx, rx));
12835 }
12836 Rig {
12837 handler,
12838 forwarding,
12839 owner,
12840 _owner_rx: owner_rx,
12841 modules,
12842 generation: 0,
12843 next_connection: 100,
12844 _supervisor: supervisor,
12845 }
12846 }
12847
12848 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
12849 ScopeCarrier {
12850 principal: Principal::Reserved {
12851 module_id: module_id.to_string(),
12852 },
12853 targets: targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()),
12854 }
12855 }
12856
12857 fn session(scope_epoch: u64) -> ScopeRecord {
12860 let mut record = head("s", scope_epoch);
12861 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
12862 record.attributes.agent_id = Some("agent-1".to_string());
12863 record.attributes.delegates = true;
12864 record
12865 }
12866
12867 impl Rig {
12868 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
12869 self.generation += 1;
12870 sync(&self.handler, &self.owner, self.generation, scopes)
12871 .await
12872 .expect("the owner's sync is accepted");
12873 }
12874
12875 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
12876 ScopeSelector {
12877 owner: Principal::Reserved {
12878 module_id: OWNER.to_string(),
12879 },
12880 scope_ref: scope_ref.to_string(),
12881 scope_epoch,
12882 }
12883 }
12884
12885 fn open_frame(
12886 &mut self,
12887 opener: Option<&str>,
12888 target: &str,
12889 scope: Option<ScopeSelector>,
12890 ) -> (
12891 RouteCtx,
12892 mpsc::Receiver<crate::router::OutboundFrame>,
12893 Frame,
12894 ) {
12895 self.next_connection += 1;
12896 let (ctx, rx) = wide_ctx(self.next_connection);
12897 let root = unique_project_root("scoped-open");
12898 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
12899 target: RouteTarget::ToolProvider {
12900 module_id: target.to_string(),
12901 },
12902 identity: BindIdentity::new(
12903 root.path().to_path_buf(),
12904 "unit".to_string(),
12905 "session".to_string(),
12906 ),
12907 consumer_identity: opener.map(|module_id| ConsumerIdentity {
12908 module_id: module_id.to_string(),
12909 launch_nonce: nonce(module_id),
12910 }),
12911 consumer_capabilities: None,
12912 role_versions: None,
12913 admission_facts: None,
12914 scope,
12915 })
12916 .unwrap();
12917 let frame = Frame::build(
12918 FrameType::Request,
12919 control_flags(),
12920 0,
12921 0,
12922 self.next_connection,
12923 body,
12924 )
12925 .unwrap();
12926 (ctx, rx, frame)
12927 }
12928
12929 async fn refused(
12931 &mut self,
12932 opener: Option<&str>,
12933 target: &str,
12934 scope: Option<ScopeSelector>,
12935 ) -> String {
12936 self.refusal_body(opener, target, scope).await["code"]
12937 .as_str()
12938 .unwrap()
12939 .to_string()
12940 }
12941
12942 async fn refusal_body(
12943 &mut self,
12944 opener: Option<&str>,
12945 target: &str,
12946 scope: Option<ScopeSelector>,
12947 ) -> Value {
12948 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
12949 let replies = tokio::time::timeout(
12950 Duration::from_secs(2),
12951 self.handler.handle_control_frame(&ctx, frame),
12952 )
12953 .await
12954 .expect("the open must be refused before waiting for a bind ack")
12955 .unwrap();
12956 assert_eq!(replies.len(), 1, "{replies:?}");
12957 assert_eq!(replies[0].header.ty, FrameType::Error);
12958 let (_, module_rx) = self.modules.get_mut(target).unwrap();
12959 assert!(
12960 module_rx.try_recv().is_err(),
12961 "a refused open relays nothing"
12962 );
12963 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
12964 parse_error(&replies[0])
12965 }
12966
12967 async fn relayed(
12969 &mut self,
12970 opener: Option<&str>,
12971 target: &str,
12972 scope: Option<ScopeSelector>,
12973 ) -> Relayed {
12974 let (ctx, rx, frame) = self.open_frame(opener, target, scope);
12975 let handler = self.handler.clone();
12976 let task_ctx = ctx.clone();
12977 let task = tokio::spawn(async move {
12978 handler
12979 .handle_control_frame(&task_ctx, frame)
12980 .await
12981 .unwrap()
12982 });
12983 let (_, module_rx) = self.modules.get_mut(target).unwrap();
12984 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
12985 .await
12986 .expect("the target receives the relayed route.bind")
12987 .unwrap()
12988 .frame;
12989 Relayed {
12990 target: target.to_string(),
12991 client: ctx,
12992 client_rx: rx,
12993 task,
12994 bind,
12995 }
12996 }
12997
12998 async fn ack(&self, relayed: &Relayed) {
12999 let (module, _) = &self.modules[&relayed.target];
13000 self.handler
13001 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
13002 .await
13003 .unwrap();
13004 }
13005
13006 async fn bound(
13008 &mut self,
13009 opener: Option<&str>,
13010 target: &str,
13011 scope: Option<ScopeSelector>,
13012 ) -> Bound {
13013 let relayed = self.relayed(opener, target, scope).await;
13014 self.ack(&relayed).await;
13015 let Relayed {
13016 target,
13017 client,
13018 mut client_rx,
13019 task,
13020 bind,
13021 } = relayed;
13022 assert!(
13023 task.await.unwrap().is_empty(),
13024 "the open is answered by commit"
13025 );
13026 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
13027 Bound {
13028 target,
13029 client,
13030 client_rx,
13031 channel,
13032 epoch,
13033 bind,
13034 }
13035 }
13036
13037 fn live(&self, route: &Bound) -> bool {
13038 matches!(
13039 self.forwarding
13040 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
13041 .unwrap(),
13042 DataRoute::Client(DataRouteState::Bound(_))
13043 )
13044 }
13045 }
13046
13047 struct Relayed {
13048 target: String,
13049 client: RouteCtx,
13050 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13051 task: tokio::task::JoinHandle<Vec<Frame>>,
13052 bind: Frame,
13053 }
13054
13055 struct Bound {
13056 target: String,
13057 client: RouteCtx,
13058 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13059 channel: u16,
13060 epoch: u32,
13061 bind: Frame,
13062 }
13063
13064 impl Bound {
13065 fn closed_reason(&mut self) -> RouteCloseReason {
13068 let mut reason = None;
13069 let mut goodbye = false;
13070 while let Ok(outbound) = self.client_rx.try_recv() {
13071 let frame = outbound.frame;
13072 match frame.header.ty {
13073 FrameType::Goodbye => {
13074 assert_eq!(
13075 (frame.header.channel, frame.header.epoch),
13076 (self.channel, self.epoch)
13077 );
13078 goodbye = true;
13079 }
13080 FrameType::Push => {
13081 let ClientControlPush::RouteClosed {
13082 reason: r,
13083 module_id,
13084 ..
13085 } = serde_json::from_slice(&frame.body).unwrap()
13086 else {
13087 panic!("unexpected push");
13088 };
13089 assert_eq!(module_id, self.target);
13090 reason = Some(r);
13091 }
13092 other => panic!("unexpected frame {other:?}"),
13093 }
13094 }
13095 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
13096 reason.expect("the client is told why the route closed")
13097 }
13098
13099 fn untouched(&mut self) -> bool {
13100 self.client_rx.try_recv().is_err()
13101 }
13102
13103 fn stamp(&self) -> Option<ScopeStamp> {
13104 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
13105 ModuleControlRequest::RouteBind { scope, .. } => scope,
13106 other => panic!("expected a route.bind, got {other:?}"),
13107 }
13108 }
13109 }
13110
13111 #[tokio::test]
13112 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
13113 ) {
13114 let mut rig = rig().await;
13115 let mut record = session(1);
13116 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
13117 record.child_owners = vec![Principal::Reserved {
13118 module_id: MAGIC.to_string(),
13119 }];
13120 rig.sync(vec![record]).await;
13121 let scope = || Some(rig_selector("s", Some(1)));
13122
13123 rig.bound(Some(OWNER), PLEXUS, scope()).await;
13126 rig.bound(Some(AFT), OTHER, scope()).await;
13127 rig.bound(Some(BROCA), PLEXUS, scope()).await;
13128
13129 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
13133 assert_eq!(
13134 rig.refused(opener, target, scope()).await,
13135 error_codes::SCOPE_NOT_CARRIER,
13136 "{opener:?} -> {target}"
13137 );
13138 }
13139 }
13140
13141 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
13142 ScopeSelector {
13143 owner: Principal::Reserved {
13144 module_id: OWNER.to_string(),
13145 },
13146 scope_ref: scope_ref.to_string(),
13147 scope_epoch,
13148 }
13149 }
13150
13151 #[tokio::test]
13152 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
13153 let mut rig = rig().await;
13154 rig.sync(vec![session(1)]).await;
13155 for opener in [OWNER, AFT] {
13156 assert_eq!(
13157 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
13158 .await,
13159 error_codes::SCOPE_EPOCH_REQUIRED,
13160 "{opener}"
13161 );
13162 }
13163 }
13164
13165 #[tokio::test]
13166 async fn admission_separates_not_synced_not_live_and_ended() {
13167 let mut rig = rig().await;
13168 let code = rig
13170 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13171 .await;
13172 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
13173 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
13174
13175 let ghost = ScopeSelector {
13177 owner: Principal::Reserved {
13178 module_id: "ghost".to_string(),
13179 },
13180 scope_ref: "s".to_string(),
13181 scope_epoch: Some(1),
13182 };
13183 assert_eq!(
13184 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
13185 error_codes::SCOPE_NOT_LIVE
13186 );
13187
13188 rig.sync(vec![session(2)]).await;
13189 assert_eq!(
13190 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
13191 .await,
13192 error_codes::SCOPE_NOT_LIVE
13193 );
13194 for epoch in [1, 3] {
13195 assert_eq!(
13196 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
13197 .await,
13198 error_codes::SCOPE_ENDED,
13199 "epoch {epoch}"
13200 );
13201 }
13202 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
13204 .await;
13205 }
13206
13207 #[tokio::test]
13208 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
13209 let mut rig = rig().await;
13210 rig.sync(vec![session(1)]).await;
13211 let route = rig
13212 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13213 .await;
13214 let stamp = route.stamp().expect("a scoped bind carries the stamp");
13215 assert_eq!(stamp.scope_ref, "s");
13216 assert_eq!(stamp.scope_epoch, 1);
13217 assert_eq!(stamp.kind, ScopeKind::Head);
13218 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
13219 assert!(stamp.attributes.delegates);
13220 assert!(stamp.owner_authorized);
13221 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
13222 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
13223
13224 let (broca, mut broca_rx) = wide_ctx(50);
13227 hello_via_sink(
13228 &rig.handler,
13229 &broca,
13230 &mut broca_rx,
13231 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
13232 )
13233 .await;
13234 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
13235 .await
13236 .unwrap();
13237 let own = ScopeSelector {
13238 owner: Principal::Reserved {
13239 module_id: BROCA.to_string(),
13240 },
13241 scope_ref: "b".to_string(),
13242 scope_epoch: Some(1),
13243 };
13244 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
13245 assert!(!route.stamp().unwrap().owner_authorized);
13246 }
13247
13248 #[tokio::test]
13249 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
13250 let mut rig = rig().await;
13251 let mut record = head("s", 1);
13252 record.carriers = vec![carrier(AFT, None)];
13253 let flow_id = "Flow:run-7/step_2!~";
13254 record.attributes.flow_id = Some(flow_id.to_string());
13255 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
13256 .await
13257 .unwrap();
13258 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
13259 panic!("not a sync reply");
13260 };
13261 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
13262 let route = rig
13263 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13264 .await;
13265 assert!(rig.live(&route), "the stamped bind committed");
13266 let stamp = route.stamp().expect("a flow scope carries a stamp");
13267 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
13268 assert_eq!(stamp.attributes.agent_id, None);
13269 assert!(!stamp.attributes.delegates);
13270 assert!(stamp.owner_authorized);
13271 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
13272 assert_eq!(unscoped.stamp(), None);
13273 }
13274
13275 #[tokio::test]
13276 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
13277 let mut rig = rig_with_flow_support(false).await;
13278 let mut record = session(1);
13279 record.attributes.flow_id = Some("flow:7".to_string());
13280 rig.sync(vec![record]).await;
13281 for opener in [OWNER, AFT] {
13282 let body = rig
13283 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
13284 .await;
13285 assert_eq!(body["code"], "target_flow_unsupported");
13286 let message = body["message"].as_str().unwrap();
13287 for required in [PLEXUS, "flow-scopes/v1"] {
13288 assert!(message.contains(required), "{message}");
13289 }
13290 }
13291 }
13292
13293 #[tokio::test]
13294 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
13295 let mut rig = rig_with_flow_support(true).await;
13296 let mut record = session(1);
13297 record.attributes.flow_id = Some("flow:7".to_string());
13298 rig.sync(vec![record]).await;
13299 for opener in [OWNER, AFT] {
13300 let route = rig
13301 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
13302 .await;
13303 assert!(rig.live(&route));
13304 assert_eq!(
13305 route.stamp().unwrap().attributes.flow_id.as_deref(),
13306 Some("flow:7")
13307 );
13308 }
13309 }
13310
13311 #[tokio::test]
13312 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
13313 use std::future::Future;
13314
13315 let mut rig = rig_with_flow_support(true).await;
13316 let mut record = session(1);
13317 record.attributes.flow_id = Some("flow:7".to_string());
13318 rig.sync(vec![record]).await;
13319 let (client, mut client_rx, frame) =
13320 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
13321 for _ in 0..64 {
13324 client.egress.try_send(route_bind_ack(1)).unwrap();
13325 }
13326 let handler = rig.handler.clone();
13327 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
13328 std::future::poll_fn(|cx| {
13329 assert!(open.as_mut().poll(cx).is_pending());
13330 std::task::Poll::Ready(())
13331 })
13332 .await;
13333 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
13334
13335 let old_connection = rig.modules[PLEXUS].0.connection_id;
13336 rig.handler.cleanup_connection(old_connection).unwrap();
13337 let (replacement, mut replacement_rx) = wide_ctx(200);
13338 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
13339 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
13340 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
13341 let hello = Frame::build(
13342 FrameType::Hello,
13343 control_flags(),
13344 0,
13345 0,
13346 200,
13347 serde_json::to_vec(&body).unwrap(),
13348 )
13349 .unwrap();
13350 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
13351
13352 client_rx.try_recv().unwrap();
13353 let replies = tokio::time::timeout(Duration::from_secs(2), open)
13354 .await
13355 .expect("the replacement is refused without waiting for a bind ack")
13356 .unwrap();
13357 assert_eq!(replies.len(), 1);
13358 let body = parse_error(&replies[0]);
13359 assert_eq!(body["code"], "target_flow_unsupported");
13360 for required in [PLEXUS, "flow-scopes/v1"] {
13361 assert!(body["message"].as_str().unwrap().contains(required));
13362 }
13363 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
13364 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
13365 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
13366 assert!(rig
13367 .handler
13368 .registry
13369 .get_module_by_connection(replacement.connection_id)
13370 .unwrap()
13371 .is_some());
13372 }
13373
13374 #[tokio::test]
13375 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
13376 {
13377 let mut rig = rig_with_flow_support(false).await;
13378 rig.sync(vec![session(1)]).await;
13379 let route = rig
13380 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13381 .await;
13382 assert!(rig.live(&route));
13383 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
13384 let mut record = session(1);
13385 record.attributes.flow_id = Some("flow:7".to_string());
13386 rig.sync(vec![record]).await;
13387 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
13388 assert!(rig.live(&unscoped));
13389 assert_eq!(unscoped.stamp(), None);
13390 }
13391
13392 #[tokio::test]
13393 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
13394 let mut rig = rig().await;
13395 let mut record = session(1);
13396 record.attributes.flow_id = Some("flow:7".to_string());
13397 rig.sync(vec![record.clone()]).await;
13398 let before = rig.forwarding.published_scope_tag(OWNER, "s");
13399 let mut owner_route = rig
13400 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
13401 .await;
13402 let mut carrier_route = rig
13403 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
13404 .await;
13405 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
13406 rig.sync(vec![record.clone()]).await;
13407 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
13408 assert!(rig.live(&owner_route) && owner_route.untouched());
13409 assert!(rig.live(&carrier_route) && carrier_route.untouched());
13410
13411 record.attributes.flow_id = Some("flow:8".to_string());
13412 rig.sync(vec![record]).await;
13413 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
13414 let before = before.unwrap();
13415 assert_eq!(after.scope_epoch, before.scope_epoch);
13416 assert!(after.version > before.version);
13417 for route in [&mut owner_route, &mut carrier_route] {
13418 assert!(!rig.live(route));
13419 assert_eq!(
13420 route.closed_reason(),
13421 RouteCloseReason::ScopeDelegationChanged
13422 );
13423 }
13424 assert!(rig.live(&unscoped) && unscoped.untouched());
13425 for target in [PLEXUS, OTHER] {
13428 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
13429 let goodbye = module_rx
13430 .try_recv()
13431 .expect("the provider sees the drain")
13432 .frame;
13433 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
13434 assert!(module_rx.try_recv().is_err());
13435 }
13436 let rebound = rig
13437 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13438 .await;
13439 assert_eq!(
13440 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
13441 Some("flow:8")
13442 );
13443 }
13444
13445 #[tokio::test]
13450 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
13451 let mut rig = rig().await;
13452 rig.sync(vec![session(1)]).await;
13453 let mut cotenant = rig
13454 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
13455 .await;
13456
13457 let mut changed = session(1);
13458 changed.child_owners.push(Principal::Reserved {
13459 module_id: MAGIC.to_string(),
13460 });
13461 let mut ended = None;
13462 for (code, next) in [
13463 (error_codes::SCOPE_CHANGED, vec![changed]),
13464 (error_codes::SCOPE_ENDED, Vec::new()),
13465 ] {
13466 let relayed = rig
13467 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13468 .await;
13469 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
13470 ended = Some(next.is_empty());
13471 rig.sync(next).await;
13472 rig.ack(&relayed).await;
13473 let replies = relayed.task.await.unwrap();
13474 assert_eq!(replies.len(), 1, "{replies:?}");
13475 assert_eq!(parse_error(&replies[0])["code"], code);
13476 assert_eq!(
13477 subc_protocol::error_codes::is_retryable_route_open(code),
13478 code == error_codes::SCOPE_CHANGED
13479 );
13480 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
13481 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
13483 let mut goodbyes = Vec::new();
13486 while let Ok(outbound) = plexus_rx.try_recv() {
13487 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
13488 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
13489 }
13490 assert!(
13491 goodbyes.contains(&(bind_channel, bind_epoch)),
13492 "{goodbyes:?}"
13493 );
13494 assert!(rig
13495 .handler
13496 .registry
13497 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
13498 .unwrap()
13499 .is_some());
13500 }
13501 assert_eq!(ended, Some(true));
13502 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
13505 }
13506
13507 #[tokio::test]
13510 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
13511 struct Case {
13512 name: &'static str,
13513 change: fn(&mut ScopeRecord),
13514 closed: [Option<RouteCloseReason>; 4],
13517 }
13518 use RouteCloseReason::*;
13519 let cases = [
13520 Case {
13521 name: "a carrier entry removed",
13522 change: |r| {
13523 r.carriers.retain(|c| {
13524 c.principal
13525 != Principal::Reserved {
13526 module_id: AFT.to_string(),
13527 }
13528 })
13529 },
13530 closed: [None, Some(ScopeCarrierRemoved), None, None],
13531 },
13532 Case {
13533 name: "a target removed from a carrier",
13534 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
13535 closed: [None, None, None, Some(ScopeCarrierRemoved)],
13536 },
13537 Case {
13538 name: "a bare carrier narrowed to targets",
13539 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
13540 closed: [None, Some(ScopeCarrierRemoved), None, None],
13541 },
13542 Case {
13543 name: "delegates turned off",
13544 change: |r| r.attributes.delegates = false,
13545 closed: [Some(ScopeDelegationChanged); 4],
13546 },
13547 Case {
13548 name: "agent_id changed",
13549 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
13550 closed: [Some(ScopeDelegationChanged); 4],
13551 },
13552 Case {
13553 name: "a carrier added, child owners changed, the record re-sent",
13554 change: |r| {
13555 r.carriers.push(carrier(MAGIC, None));
13556 r.child_owners.push(Principal::Reserved {
13557 module_id: MAGIC.to_string(),
13558 });
13559 },
13560 closed: [None; 4],
13561 },
13562 Case {
13563 name: "a target added",
13564 change: |r| {
13565 r.carriers[1]
13566 .targets
13567 .as_mut()
13568 .unwrap()
13569 .push("third".to_string())
13570 },
13571 closed: [None; 4],
13572 },
13573 Case {
13574 name: "delegates turned on",
13575 change: |r| r.attributes.delegates = true,
13576 closed: [None; 4],
13577 },
13578 ];
13579 for case in cases {
13580 let mut rig = rig().await;
13581 rig.sync(vec![session(1)]).await;
13582 let scope = || Some(rig_selector("s", Some(1)));
13583 let mut routes = [
13584 rig.bound(Some(OWNER), PLEXUS, scope()).await,
13585 rig.bound(Some(AFT), PLEXUS, scope()).await,
13586 rig.bound(Some(BROCA), PLEXUS, scope()).await,
13587 rig.bound(Some(BROCA), OTHER, scope()).await,
13588 ];
13589 let mut record = session(1);
13590 (case.change)(&mut record);
13591 rig.sync(vec![record]).await;
13592 for (index, expected) in case.closed.iter().enumerate() {
13593 let route = &mut routes[index];
13594 match expected {
13595 Some(reason) => {
13596 assert!(!rig.live(route), "{}: route {index} still live", case.name);
13597 assert_eq!(
13598 route.closed_reason(),
13599 *reason,
13600 "{}: route {index}",
13601 case.name
13602 );
13603 }
13604 None => {
13605 assert!(rig.live(route), "{}: route {index} closed", case.name);
13606 assert!(
13607 route.untouched(),
13608 "{}: route {index} was told something",
13609 case.name
13610 );
13611 }
13612 }
13613 }
13614 }
13615 }
13616
13617 #[tokio::test]
13618 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
13619 for next in [Vec::new(), vec![session(2)]] {
13621 let mut rig = rig().await;
13622 rig.sync(vec![session(1)]).await;
13623 let mut route = rig
13624 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13625 .await;
13626 rig.sync(next).await;
13627 assert!(!rig.live(&route));
13628 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
13629 }
13630
13631 let mut rig = rig().await;
13634 let mut child = session(1);
13635 child.scope_ref = "child".to_string();
13636 child.kind = ScopeKind::Worker;
13637 child.parent = Some(ScopeParent {
13638 owner: Principal::Reserved {
13639 module_id: OWNER.to_string(),
13640 },
13641 scope_ref: "s".to_string(),
13642 scope_epoch: 1,
13643 });
13644 rig.sync(vec![session(1), child.clone()]).await;
13645 let mut child_route = rig
13646 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
13647 .await;
13648 assert_eq!(
13649 child_route.stamp().unwrap().parent_state,
13650 Some(ParentState::Linked)
13651 );
13652 rig.sync(vec![child]).await;
13653 assert!(!rig.live(&child_route));
13654 assert_eq!(
13655 child_route.closed_reason(),
13656 RouteCloseReason::ScopeParentEnded
13657 );
13658 }
13659
13660 #[tokio::test]
13661 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
13662 ) {
13663 let mut rig = rig().await;
13664 rig.sync(vec![session(1)]).await;
13665 let mut route = rig
13666 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13667 .await;
13668 let before = rig.forwarding.published_scope_tag(OWNER, "s");
13669 rig.sync(vec![session(1)]).await;
13670 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
13671 assert!(rig.live(&route) && route.untouched());
13672
13673 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
13677 .forwarding
13678 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
13679 .unwrap()
13680 else {
13681 panic!("the route is bound");
13682 };
13683 binding.flow.acquire_tagged(9, false).await.unwrap();
13684 let mut widened = session(1);
13685 widened.carriers.push(carrier(MAGIC, None));
13686 rig.sync(vec![widened]).await;
13687 assert!(rig.live(&route) && route.untouched());
13688 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
13689 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
13690 assert_eq!(binding.flow.in_flight(), 1);
13693 binding
13694 .flow
13695 .acquire_tagged(10, false)
13696 .await
13697 .expect("the flow is still open");
13698 }
13699
13700 #[tokio::test]
13703 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
13704 let mut rig = rig().await;
13705 rig.sync(vec![session(1)]).await;
13706 let mut on_incumbent = rig
13707 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13708 .await;
13709
13710 let (candidate, _candidate_rx) = wide_ctx(9);
13713 let registration = rig
13714 .handler
13715 .registry
13716 .register_candidate_with_control_ops(
13717 manifest(PLEXUS, PROTOCOL_VERSION),
13718 PROTOCOL_VERSION,
13719 candidate.connection_id,
13720 module_baseline_control_ops(),
13721 )
13722 .unwrap();
13723 rig.forwarding
13724 .register_candidate_module_connection(
13725 candidate.connection_id,
13726 PLEXUS.to_string(),
13727 PROTOCOL_VERSION,
13728 manifest_concurrency(®istration.manifest),
13729 candidate.egress.clone(),
13730 )
13731 .unwrap();
13732 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
13733 rig.handler
13734 .registry
13735 .promote_candidate(PLEXUS)
13736 .unwrap()
13737 .unwrap();
13738 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
13739
13740 rig.sync(Vec::new()).await;
13741 assert!(!rig.live(&on_incumbent));
13742 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
13743 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
13744 let goodbye = incumbent_rx
13745 .try_recv()
13746 .expect("the superseded endpoint is told")
13747 .frame;
13748 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
13749 }
13750 }
13751}
13752
13753#[cfg(test)]
13754mod concurrency_default_exposure_tests {
13755 use super::*;
13756
13757 fn hello_body(role_json: &str) -> Vec<u8> {
13758 format!(
13759 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
13760 )
13761 .into_bytes()
13762 }
13763
13764 fn manifest_from(body: &[u8]) -> ModuleManifest {
13765 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
13766 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
13767 .expect("manifest parses")
13768 }
13769
13770 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
13771
13772 #[test]
13773 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
13774 let body = hello_body(&format!(
13775 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
13776 ));
13777 let manifest = manifest_from(&body);
13778 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
13781 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
13782 }
13783
13784 #[test]
13785 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
13786 let body = hello_body(&format!(
13787 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
13788 ));
13789 let manifest = manifest_from(&body);
13790 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
13791 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
13792 }
13793
13794 #[test]
13795 fn non_management_roles_are_never_reported() {
13796 let body = hello_body(
13797 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
13798 );
13799 let manifest = manifest_from(&body);
13800 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
13801 }
13802}