1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 path::{Path, PathBuf},
5 sync::{Arc, Mutex, RwLock},
6 time::{Duration, Instant as StdInstant},
7};
8
9use serde::{Deserialize, Serialize};
10use subc_control::{
11 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
12 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
13 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
14 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
15 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
16 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
17 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
18};
19use subc_protocol::{
20 error_codes,
21 manifest::{
22 validate_hello_capability_grammar, validate_hello_self_signal_declarations,
23 CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
24 ProviderRole,
25 },
26 scope::{
27 ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
28 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_DESCRIBE_OP, SCOPE_SYNC_OP,
29 },
30 session::{
31 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
32 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
33 MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE, ROLE_VERSIONS_FIELD,
34 },
35 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
36 Priority, RouteTarget, PROTOCOL_VERSION,
37};
38use tokio::time::{timeout_at, Instant};
39use tracing::{debug, info, warn};
40
41use crate::{
42 capability_requirements::{
43 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
44 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
45 RuntimeModule,
46 },
47 daemon_config::RestartRequiredSection,
48 forwarding::{
49 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
50 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
51 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
52 },
53 observability::{
54 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
55 },
56 provenance::{
57 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
58 },
59 registry::{ChannelState, ConnectionId, Registry, RegistryError},
60 router::{RouteCtx, RouterError},
61 scopes::{BoundScope, HelloLaunchNonces, ScopeTable},
62 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
63 stderr_tail::{CaptureState, TailEntry},
64 supervise::{
65 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
66 SupervisorHandle, SwapHelloAdmission,
67 },
68 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
69};
70
71pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
77
78const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
79const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
80const CAP_PING_PONG: &str = "ping_pong_v1";
81const CAP_SESSION_ATTACH: &str = "session_attach_v1";
82const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
83
84const SUBC_CONTROL_OPS: &[&str] = &[
85 ops::SERVER_DESCRIBE,
86 ops::CATALOG_LIST,
87 ops::ROUTE_OPEN,
88 ops::ROUTE_POLL,
89 ops::ROUTE_CLOSING,
90 ops::ROUTE_CLOSED,
91 ops::SUPERVISOR_LIST,
92 ops::SUPERVISOR_RESTART,
93 ops::SUPERVISOR_SWAP,
94 ops::SUPERVISOR_RELOAD,
95 ops::SUPERVISOR_RESCAN,
96 ops::SUPERVISOR_RELEASE_RESERVED,
97 ops::SUPERVISOR_SET_ENABLED,
98 ops::SUPERVISOR_HEALTH_PROBE,
99 ops::SUPERVISOR_HEALTH,
100 ops::SUPERVISOR_STDERR_TAIL,
101 ops::SUPERVISOR_TERMINALS,
102 ops::SUPERVISOR_ROUTES,
103 ops::SUPERVISOR_PROVENANCE,
104 ops::SUPERVISOR_SPAWN_SNAPSHOT,
105 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
106];
107
108const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
109 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
110 "supervisor.live_roots",
111 SCOPE_SYNC_OP,
112 SCOPE_DESCRIBE_OP,
113];
114
115const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
119
120const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
121
122pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
130
131pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
146
147pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
165
166const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
167const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
168
169fn reload_verdict(
170 configured: &Path,
171 spawned_from: Option<&Path>,
172 image: subc_control::RunningImageAgreement,
173) -> PendingReloadVerdict {
174 let path = match spawned_from {
175 Some(spawned_from) if configured == spawned_from => ReloadPathAgreement::Match,
176 Some(spawned_from) => ReloadPathAgreement::Mismatch {
177 configured: configured.to_path_buf(),
178 spawned_from: spawned_from.to_path_buf(),
179 },
180 None => ReloadPathAgreement::Unavailable {
181 reason: if matches!(
182 image,
183 subc_control::RunningImageAgreement::Unavailable {
184 reason: subc_control::RunningImageUnavailableReason::NotRunning
185 }
186 ) {
187 ReloadPathUnavailableReason::NotRunning
188 } else {
189 ReloadPathUnavailableReason::SpawnedPathUnavailable
190 },
191 },
192 };
193 PendingReloadVerdict { path, image }
194}
195
196#[derive(Clone)]
197struct DaemonProvenanceFacts {
198 build: DaemonBuildProvenance,
199 pid: Option<u32>,
200 started_at_ms: Option<u64>,
201 start_clock: Option<crate::clock::StartClock>,
202 executable_path: Option<PathBuf>,
203 executable_identity: Option<SpawnedFileIdentity>,
204 process_start_time: Option<u64>,
205 probe: ExecutableIdentityProbe,
206}
207
208impl Default for DaemonProvenanceFacts {
209 fn default() -> Self {
210 Self {
211 build: DaemonBuildProvenance {
212 build_git_sha: None,
213 build_lock_digest: None,
214 },
215 pid: None,
216 started_at_ms: None,
217 start_clock: None,
218 executable_path: None,
219 executable_identity: None,
220 process_start_time: None,
221 probe: ExecutableIdentityProbe::default(),
222 }
223 }
224}
225
226#[derive(Debug, Clone)]
227struct SupervisorRescanContext {
228 supervisor: Supervisor,
229 config_path: PathBuf,
230 configured_port: Option<u16>,
231 storage_config: Option<crate::daemon_config::StorageConfig>,
232 admission_facts_carrier_module_id: Option<String>,
233 admission_facts_targets: Option<Vec<String>>,
234 scope_authority_owners: Vec<String>,
235}
236
237const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
258 "reloading",
259 "supervisor_not_live",
260 "registration_not_active",
261 "no_forwarding_connection",
262 "relay_send_failed",
263];
264
265#[derive(Clone)]
267pub struct ControlHandler {
268 registry: Arc<Registry>,
269 forwarding: Arc<ForwardingTable>,
270 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
271 supervisor: SupervisorHandle,
272 subc_capabilities: Arc<[String]>,
273 route_bind_relay_timeout: Duration,
277 route_bind_relay_timeouts: BTreeMap<String, Duration>,
281 route_bind_breakers: RouteBindBreakers,
284 route_bind_concurrency: RouteBindConcurrency,
287 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
292 route_bind_breaker_threshold: u32,
294 route_bind_breaker_cooldown: Duration,
296 health_probe_timeout: Duration,
297 storage_config: Option<crate::daemon_config::StorageConfig>,
300 machine_id: Option<crate::machine_id::MachineId>,
304 admission_facts_carrier_module_id: Option<String>,
305 admission_facts_targets: Option<Vec<String>>,
306 scopes: Arc<RwLock<ScopeTable>>,
310 scope_authority_owners: Vec<String>,
313 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
316 rescan: Option<SupervisorRescanContext>,
317 connected_clients: ConnectedClients,
318 counters: DaemonCounters,
319 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
320 daemon_provenance: DaemonProvenanceFacts,
321 #[cfg(test)]
322 control_dispatch_delay: Option<Duration>,
323 #[cfg(test)]
324 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
325}
326
327impl fmt::Debug for ControlHandler {
328 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
329 f.debug_struct("ControlHandler")
330 .field("registry", &self.registry)
331 .field("forwarding", &self.forwarding)
332 .field("process_liveness", &self.process_liveness.is_some())
333 .field("supervisor", &self.supervisor)
334 .field("subc_capabilities", &self.subc_capabilities)
335 .finish()
336 }
337}
338
339struct RouteOpenRequest {
340 target: RouteTarget,
341 identity: BindIdentity,
342 consumer_identity: Option<ConsumerIdentity>,
343 consumer_capabilities: Option<Vec<String>>,
344 role_versions: Option<BTreeMap<String, String>>,
345 admission_facts: Option<serde_json::Value>,
346 scope: Option<ScopeSelector>,
347}
348
349struct RouteBindReservationGuard {
350 forwarding: Arc<ForwardingTable>,
351 endpoint: ModuleEndpointId,
352 relay_corr: u64,
353 armed: bool,
354}
355
356struct ModuleControlRpcGuard {
357 forwarding: Arc<ForwardingTable>,
358 endpoint: ModuleEndpointId,
359 corr: u64,
360 armed: bool,
361}
362
363impl ModuleControlRpcGuard {
364 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
365 Self {
366 forwarding,
367 endpoint,
368 corr,
369 armed: true,
370 }
371 }
372
373 fn disarm(&mut self) {
374 self.armed = false;
375 }
376}
377
378impl Drop for ModuleControlRpcGuard {
379 fn drop(&mut self) {
380 if self.armed {
381 let _ = self
382 .forwarding
383 .cancel_module_control_rpc(self.endpoint, self.corr);
384 }
385 }
386}
387
388impl RouteBindReservationGuard {
389 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
390 Self {
391 forwarding,
392 endpoint,
393 relay_corr,
394 armed: true,
395 }
396 }
397
398 fn release_and_disarm(&mut self) {
399 if !self.armed {
400 return;
401 }
402 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
403 self.endpoint,
404 self.relay_corr,
405 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
406 ) {
407 send_goodbye_target_best_effort(
408 &self.forwarding.counters(),
409 &target,
410 "canceled route.bind",
411 );
412 }
413 self.armed = false;
414 }
415
416 fn disarm(&mut self) {
417 self.armed = false;
418 }
419}
420
421impl Drop for RouteBindReservationGuard {
422 fn drop(&mut self) {
423 self.release_and_disarm();
424 }
425}
426
427#[derive(Debug, Clone, Default)]
457pub(crate) struct RouteBindBreakers {
458 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
459}
460
461#[derive(Debug, Clone, Default)]
462pub(crate) struct RouteBindConcurrency {
463 modules: Arc<Mutex<HashMap<String, usize>>>,
464}
465
466struct RouteBindConcurrencyGuard {
467 concurrency: RouteBindConcurrency,
468 module_id: String,
469}
470
471impl RouteBindConcurrency {
472 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
475 let mut modules = self
476 .modules
477 .lock()
478 .expect("route.bind concurrency mutex poisoned");
479 let in_flight = modules.entry(module_id.to_string()).or_default();
480 if *in_flight >= limit {
481 return Err(*in_flight);
482 }
483 *in_flight += 1;
484 Ok(RouteBindConcurrencyGuard {
485 concurrency: self.clone(),
486 module_id: module_id.to_string(),
487 })
488 }
489}
490
491impl Drop for RouteBindConcurrencyGuard {
492 fn drop(&mut self) {
493 let mut modules = self
494 .concurrency
495 .modules
496 .lock()
497 .expect("route.bind concurrency mutex poisoned");
498 let remove = {
499 let in_flight = modules
500 .get_mut(&self.module_id)
501 .expect("admitted route.bind has a concurrency entry");
502 *in_flight -= 1;
503 *in_flight == 0
504 };
505 if remove {
506 modules.remove(&self.module_id);
507 }
508 }
509}
510
511#[derive(Debug, Default)]
512struct ModuleBreakerState {
513 consecutive_timeouts: u32,
515 cooldown_until: Option<Instant>,
518 probe_in_flight: bool,
523}
524
525enum RouteBindAdmission<'a> {
527 Admitted {
528 guard: RouteBindBreakerGuard<'a>,
529 probe: bool,
532 },
533 Refused {
534 consecutive_timeouts: u32,
535 retry_in: Duration,
539 probe_in_flight: bool,
540 },
541}
542
543struct RouteBindBreakerGuard<'a> {
550 breakers: RouteBindBreakers,
551 module_id: &'a str,
552 settled: bool,
553}
554
555impl RouteBindBreakerGuard<'_> {
556 fn record_accepted(&mut self) -> bool {
560 self.settled = true;
561 self.breakers.record_accepted(self.module_id)
562 }
563
564 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
567 self.settled = true;
568 self.breakers
569 .record_timeout(self.module_id, threshold, cooldown)
570 }
571
572 fn record_inconclusive(&mut self) {
581 self.settled = true;
582 self.breakers.record_inconclusive(self.module_id);
583 }
584}
585
586impl Drop for RouteBindBreakerGuard<'_> {
587 fn drop(&mut self) {
588 if !self.settled {
589 self.breakers.record_inconclusive(self.module_id);
590 }
591 }
592}
593
594struct BreakerOpened {
598 consecutive_timeouts: u32,
599 reopened_after_probe: bool,
602}
603
604impl RouteBindBreakers {
605 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
606 self.modules
607 .lock()
608 .expect("route.bind breaker mutex poisoned")
609 }
610
611 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
614 let admitted = |probe| RouteBindAdmission::Admitted {
615 guard: RouteBindBreakerGuard {
616 breakers: self.clone(),
617 module_id,
618 settled: false,
619 },
620 probe,
621 };
622
623 let mut modules = self.lock();
624 let Some(state) = modules.get_mut(module_id) else {
625 return admitted(false);
626 };
627 let Some(cooldown_until) = state.cooldown_until else {
628 return admitted(false);
629 };
630 if state.probe_in_flight {
631 return RouteBindAdmission::Refused {
632 consecutive_timeouts: state.consecutive_timeouts,
633 retry_in: Duration::ZERO,
634 probe_in_flight: true,
635 };
636 }
637 let now = Instant::now();
638 if now < cooldown_until {
639 return RouteBindAdmission::Refused {
640 consecutive_timeouts: state.consecutive_timeouts,
641 retry_in: cooldown_until - now,
642 probe_in_flight: false,
643 };
644 }
645 state.probe_in_flight = true;
646 admitted(true)
647 }
648
649 fn record_accepted(&self, module_id: &str) -> bool {
650 self.lock()
651 .remove(module_id)
652 .is_some_and(|state| state.cooldown_until.is_some())
653 }
654
655 fn record_timeout(
656 &self,
657 module_id: &str,
658 threshold: u32,
659 cooldown: Duration,
660 ) -> Option<BreakerOpened> {
661 let mut modules = self.lock();
662 let state = modules.entry(module_id.to_string()).or_default();
663 let was_open = state.cooldown_until.is_some();
664 let was_probe = state.probe_in_flight;
665 state.probe_in_flight = false;
666 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
667 if state.consecutive_timeouts < threshold {
668 return None;
669 }
670 state.cooldown_until = Some(Instant::now() + cooldown);
671 Some(BreakerOpened {
672 consecutive_timeouts: state.consecutive_timeouts,
673 reopened_after_probe: was_open && was_probe,
674 })
675 }
676
677 fn record_inconclusive(&self, module_id: &str) {
678 if let Some(state) = self.lock().get_mut(module_id) {
679 state.probe_in_flight = false;
680 }
681 }
682
683 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
698 self.lock()
699 .remove(module_id)
700 .map(|state| state.consecutive_timeouts)
701 .filter(|discarded| *discarded > 0)
702 }
703
704 fn open_snapshot(&self) -> Option<serde_json::Value> {
711 let now = Instant::now();
712 let modules = self.lock();
713 let open = modules
714 .iter()
715 .filter_map(|(module_id, state)| {
716 let cooldown_until = state.cooldown_until?;
717 Some((
718 module_id.clone(),
719 serde_json::json!({
720 "consecutive_timeouts": state.consecutive_timeouts,
721 "cooldown_remaining_ms":
722 cooldown_until.saturating_duration_since(now).as_millis() as u64,
723 "probe_in_flight": state.probe_in_flight,
724 }),
725 ))
726 })
727 .collect::<serde_json::Map<String, serde_json::Value>>();
728 (!open.is_empty()).then_some(serde_json::Value::Object(open))
729 }
730}
731
732impl ControlHandler {
733 pub fn new(registry: Arc<Registry>) -> Self {
734 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
735 }
736
737 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
738 let counters = forwarding.counters();
739 let route_bind_breakers = forwarding.route_bind_breakers();
743 let route_bind_concurrency = forwarding.route_bind_concurrency();
744 let route_outages = forwarding.route_outages();
745 Self {
746 registry,
747 forwarding,
748 process_liveness: None,
749 supervisor: SupervisorHandle::new(),
750 subc_capabilities: Arc::from([
751 CAP_MANIFEST_REGISTRATION.to_string(),
752 CAP_CHANNEL_LIFECYCLE.to_string(),
753 CAP_PING_PONG.to_string(),
754 CAP_SESSION_ATTACH.to_string(),
755 CAP_ADMISSION_FACTS_RELAY.to_string(),
756 CAP_SCOPES_V1.to_string(),
757 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
758 ]),
759 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
760 route_bind_relay_timeouts: BTreeMap::new(),
761 route_bind_breakers,
762 route_bind_concurrency,
763 route_outages,
764 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
765 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
766 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
767 storage_config: None,
768 machine_id: None,
769 admission_facts_carrier_module_id: None,
770 admission_facts_targets: None,
771 scopes: Arc::new(RwLock::new(ScopeTable::new(
772 crate::daemon_config::default_scope_authority_owners(),
773 ))),
774 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
775 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
776 rescan: None,
777 connected_clients: ConnectedClients::new(),
778 counters,
779 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
780 daemon_provenance: DaemonProvenanceFacts::default(),
781 #[cfg(test)]
782 control_dispatch_delay: None,
783 #[cfg(test)]
784 provenance_probe_override: None,
785 }
786 }
787
788 pub fn with_storage_config(
791 mut self,
792 storage_config: Option<crate::daemon_config::StorageConfig>,
793 ) -> Self {
794 self.storage_config = storage_config;
795 self
796 }
797
798 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
801 self.machine_id = machine_id;
802 self
803 }
804
805 pub fn with_admission_facts_config(
809 mut self,
810 carrier_module_id: Option<String>,
811 targets: Option<Vec<String>>,
812 ) -> Self {
813 self.admission_facts_carrier_module_id = carrier_module_id;
814 self.admission_facts_targets = targets;
815 self
816 }
817
818 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
822 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
823 self.scope_authority_owners = owners;
824 self
825 }
826
827 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
830 self.route_bind_relay_timeout = timeout;
831 self
832 }
833
834 pub fn with_route_bind_relay_timeouts(
840 mut self,
841 timeouts: impl IntoIterator<Item = (String, Duration)>,
842 ) -> Self {
843 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
844 self
845 }
846
847 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
853 self.route_bind_relay_timeouts
854 .get(module_id)
855 .copied()
856 .unwrap_or(self.route_bind_relay_timeout)
857 }
858
859 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
867 self.route_bind_breaker_threshold = threshold.max(1);
868 self.route_bind_breaker_cooldown = cooldown;
869 self
870 }
871
872 #[cfg(test)]
873 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
874 self.health_probe_timeout = timeout;
875 self
876 }
877
878 #[cfg(test)]
879 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
880 self.control_dispatch_delay = Some(delay);
881 self
882 }
883
884 pub fn with_process_liveness(
885 mut self,
886 process_liveness: Arc<dyn ModuleProcessLiveness>,
887 ) -> Self {
888 self.process_liveness = Some(process_liveness);
889 self
890 }
891
892 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
893 self.supervisor = supervisor;
894 self
895 }
896
897 pub fn with_daemon_provenance(
898 mut self,
899 pid: u32,
900 started_at_ms: u64,
901 executable_path: Option<PathBuf>,
902 build_git_sha: Option<String>,
903 build_lock_digest: Option<String>,
904 ) -> Self {
905 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
906 let process_start_time = process_start_time(pid);
907 self.daemon_provenance = DaemonProvenanceFacts {
908 build: DaemonBuildProvenance {
909 build_git_sha,
910 build_lock_digest,
911 },
912 pid: Some(pid),
913 started_at_ms: Some(started_at_ms),
914 start_clock: None,
915 executable_path,
916 executable_identity,
917 process_start_time,
918 probe: ExecutableIdentityProbe::default(),
919 };
920 self
921 }
922
923 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
924 self.daemon_provenance.start_clock = Some(clock);
925 self
926 }
927
928 #[cfg(test)]
929 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
930 self.provenance_probe_override = Some(result);
931 self
932 }
933
934 pub fn with_capability_config(
938 self,
939 modules: impl IntoIterator<Item = (String, bool)>,
940 reserved_capabilities: BTreeMap<String, String>,
941 ) -> Self {
942 self.capability_evaluator
943 .configure(modules, reserved_capabilities);
944 self
945 }
946
947 pub fn with_supervisor_rescan(
948 mut self,
949 supervisor: Supervisor,
950 config_path: impl Into<PathBuf>,
951 configured_port: Option<u16>,
952 ) -> Self {
953 self.rescan = Some(SupervisorRescanContext {
954 supervisor,
955 config_path: config_path.into(),
956 configured_port,
957 storage_config: self.storage_config.clone(),
958 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
959 admission_facts_targets: self.admission_facts_targets.clone(),
960 scope_authority_owners: self.scope_authority_owners.clone(),
961 });
962 self
963 }
964
965 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
966 self.connected_clients = connected_clients;
967 self
968 }
969
970 pub fn forwarding(&self) -> Arc<ForwardingTable> {
971 Arc::clone(&self.forwarding)
972 }
973
974 pub(crate) fn counters(&self) -> DaemonCounters {
975 self.counters.clone()
976 }
977
978 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
981 tokio::spawn(async move {
982 loop {
983 self.capability_evaluator
984 .wait_for_change_or_deadline()
985 .await;
986 self.refresh_capability_requirements();
987 }
988 });
989 }
990
991 fn runtime_capability_snapshot(
992 &self,
993 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
994 let runtime = self
995 .supervisor
996 .list()
997 .into_iter()
998 .map(|module| {
999 let status = module.status().map_err(|err| {
1000 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1001 })?;
1002 Ok(RuntimeModule {
1003 module_id: status.module_id,
1004 state: status.state,
1005 enabled: status.enabled,
1006 })
1007 })
1008 .collect::<Result<Vec<_>, RouterError>>()?;
1009 let (_, registrations) = self.registry.list_modules().map_err(|err| {
1010 RouterError::backend(
1011 0,
1012 0,
1013 format!("failed to list capability registrations: {err}"),
1014 )
1015 })?;
1016 let registrations = registrations
1017 .into_iter()
1018 .map(|registration| RegisteredModule {
1019 module_id: registration.manifest.module_id,
1020 module_version: registration.manifest.module_version,
1021 capabilities: registration.manifest.capabilities,
1022 })
1023 .collect();
1024 Ok((runtime, registrations))
1025 }
1026
1027 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1034 let cached_registration = RegisteredModule {
1035 module_id: registration.manifest.module_id.clone(),
1036 module_version: registration.manifest.module_version.clone(),
1037 capabilities: registration.manifest.capabilities.clone(),
1038 };
1039 if self.capability_evaluator.record_hello(&cached_registration) {
1040 warn!(
1041 module_id = %cached_registration.module_id,
1042 "capability claims drifted from the cached manifest"
1043 );
1044 }
1045 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1046 self.enforce_capability_denies();
1047 }
1048 self.refresh_capability_requirements();
1049 }
1050
1051 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1055 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1056 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1057 self.supervisor.set_swap_promotion_observer(observer);
1058 }
1059
1060 pub fn refresh_capability_requirements(&self) {
1061 match self.runtime_capability_snapshot() {
1062 Ok((runtime, registrations)) => {
1063 log_requirement_events(
1064 self.capability_evaluator
1065 .evaluate_now(&runtime, ®istrations),
1066 );
1067 }
1068 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1069 }
1070 }
1071
1072 fn enforce_capability_denies(&self) {
1076 let (_, registrations) = match self.registry.list_modules() {
1077 Ok(snapshot) => snapshot,
1078 Err(err) => {
1079 warn!(error = %err, "failed to read registrations for capability deny census");
1080 return;
1081 }
1082 };
1083 let manifests = registrations
1084 .into_iter()
1085 .map(|registration| {
1086 (
1087 registration.manifest.module_id.clone(),
1088 registration.manifest,
1089 )
1090 })
1091 .collect::<BTreeMap<_, _>>();
1092 let census = match self.forwarding.route_census(None) {
1093 Ok(census) => census,
1094 Err(err) => {
1095 warn!(error = %err, "failed to read route census for capability deny enforcement");
1096 return;
1097 }
1098 };
1099
1100 for (target_module_id, routes) in census {
1101 let Some(target_manifest) = manifests.get(&target_module_id) else {
1102 continue;
1103 };
1104 let mut closed_routes = Vec::new();
1105 let mut module_goodbyes = Vec::new();
1106 for route in routes {
1107 let Principal::Reserved {
1108 module_id: opening_module_id,
1109 } = &route.principal
1110 else {
1111 continue;
1112 };
1113 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1114 continue;
1115 };
1116 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1117 continue;
1118 };
1119
1120 match self.forwarding.release_client_route(
1121 route.goodbye_target.connection_id,
1122 route.goodbye_target.channel,
1123 route.goodbye_target.epoch,
1124 ) {
1125 Ok(RouteRelease::Removed(module_goodbye)) => {
1126 warn!(
1127 opening_module_id,
1128 target_module_id,
1129 capability,
1130 "force-closing route because an attested capability deny edge now matches"
1131 );
1132 closed_routes.push(route);
1133 module_goodbyes.push(module_goodbye);
1134 }
1135 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1136 Err(err) => warn!(
1137 opening_module_id,
1138 target_module_id,
1139 capability,
1140 error = %err,
1141 "failed to force-close capability-denied route"
1142 ),
1143 }
1144 }
1145
1146 if closed_routes.is_empty() {
1147 continue;
1148 }
1149 send_route_control_pushes(
1150 &self.forwarding,
1151 closed_routes,
1152 ClientControlPush::RouteClosed {
1153 module_id: target_module_id,
1154 reason: RouteCloseReason::CapabilityDenied,
1155 drained: false,
1156 abandoned: 0,
1157 excluded_subscriptions: 0,
1158 terminal: Some(false),
1159 },
1160 );
1161 self.emit_route_goodbyes(module_goodbyes);
1162 }
1163 }
1164
1165 fn not_ready_reason(
1171 &self,
1172 registration: &crate::registry::ModuleRegistration,
1173 ) -> Option<NotReadyReason> {
1174 if !registration.ready {
1175 return Some(NotReadyReason {
1176 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1177 capability: None,
1178 });
1179 }
1180 self.first_unprovided_required_capability(registration)
1181 .map(|capability| NotReadyReason {
1182 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1183 capability: Some(capability),
1184 })
1185 }
1186
1187 fn first_unprovided_required_capability(
1205 &self,
1206 registration: &crate::registry::ModuleRegistration,
1207 ) -> Option<String> {
1208 let required = registration
1209 .manifest
1210 .capabilities
1211 .iter()
1212 .flat_map(|declarations| declarations.requires.iter())
1213 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1214 .map(|requirement| requirement.capability.as_str())
1215 .collect::<BTreeSet<_>>();
1216 if required.is_empty() {
1217 return None;
1218 }
1219 let module_id = registration.manifest.module_id.as_str();
1220 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1221 if required
1222 .iter()
1223 .any(|capability| verdict(capability).is_none())
1224 {
1225 self.refresh_capability_requirements();
1226 }
1227 required
1228 .into_iter()
1229 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1230 .map(str::to_string)
1231 }
1232
1233 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1234 self.capability_evaluator
1235 .statuses()
1236 .into_iter()
1237 .map(capability_requirement_status)
1238 .collect()
1239 }
1240
1241 fn deregister_connection(
1248 &self,
1249 connection_id: ConnectionId,
1250 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1251 self.registry.deregister_connection(connection_id)
1252 }
1253
1254 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1255 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1256 return None;
1257 }
1258 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1259 parse_client_control_request(&frame.body)
1260 else {
1261 return None;
1262 };
1263 Some(target_module_id(&target).to_string())
1264 }
1265
1266 pub(crate) fn route_open_capacity_refusal(
1267 &self,
1268 ctx: &RouteCtx,
1269 frame: &Frame,
1270 target_module_id: &str,
1271 in_flight: usize,
1272 limit: usize,
1273 ) -> Result<Frame, RouterError> {
1274 self.route_open_admission_refusal_frame(
1275 ctx,
1276 frame,
1277 target_module_id,
1278 "open_admission_full",
1279 (in_flight, limit),
1280 format!(
1281 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1282 ),
1283 )
1284 }
1285
1286 fn route_open_target_capacity_refusal(
1287 &self,
1288 ctx: &RouteCtx,
1289 frame: &Frame,
1290 target_module_id: &str,
1291 in_flight: usize,
1292 ) -> Result<Frame, RouterError> {
1293 self.route_open_admission_refusal_frame(
1294 ctx,
1295 frame,
1296 target_module_id,
1297 "target_binds_full",
1298 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1299 format!(
1300 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1301 ),
1302 )
1303 }
1304
1305 fn route_open_admission_refusal_frame(
1313 &self,
1314 ctx: &RouteCtx,
1315 frame: &Frame,
1316 target_module_id: &str,
1317 reason: &'static str,
1318 (in_flight, limit): (usize, usize),
1319 message: impl Into<String>,
1320 ) -> Result<Frame, RouterError> {
1321 let code = error_codes::TARGET_UNAVAILABLE;
1322 self.counters.increment_route_open_refused(code);
1323 info!(
1324 target: "control",
1325 code,
1326 reason,
1327 module_id = ?target_module_id,
1328 connection_id = ctx.connection_id.get(),
1329 in_flight,
1330 limit,
1331 "route.open refused"
1332 );
1333 control_error_frame(frame, code, message.into())
1334 }
1335
1336 #[cfg(test)]
1342 pub fn handle_control(
1343 &self,
1344 connection_id: ConnectionId,
1345 frame: Frame,
1346 ) -> Result<Vec<Frame>, RouterError> {
1347 match frame.header.ty {
1348 FrameType::Ping => Ok(vec![pong(&frame)?]),
1349 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1350 FrameType::Goodbye => self.handle_goodbye(connection_id),
1351 ty => Ok(vec![control_error_frame(
1352 &frame,
1353 "unsupported_control_frame",
1354 format!("unsupported channel-0 frame {ty:?}"),
1355 )?]),
1356 }
1357 }
1358
1359 pub async fn handle_control_frame(
1360 &self,
1361 ctx: &RouteCtx,
1362 frame: Frame,
1363 ) -> Result<Vec<Frame>, RouterError> {
1364 self.handle_control_frame_timed(ctx, frame, None).await
1365 }
1366
1367 pub(crate) async fn handle_control_frame_timed(
1368 &self,
1369 ctx: &RouteCtx,
1370 frame: Frame,
1371 dispatch_started_at: Option<StdInstant>,
1372 ) -> Result<Vec<Frame>, RouterError> {
1373 match frame.header.ty {
1374 FrameType::Ping => Ok(vec![pong(&frame)?]),
1375 FrameType::Hello => {
1376 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1377 }
1378 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1379 FrameType::Cancel => {
1380 if self
1381 .supervisor
1382 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1383 {
1384 Ok(Vec::new())
1385 } else {
1386 Ok(vec![control_error_frame(
1387 &frame,
1388 "unknown_subscription",
1389 "no supervisor spawn subscription has this correlation id",
1390 )?])
1391 }
1392 }
1393 FrameType::Request => {
1394 if self
1395 .forwarding
1396 .module_endpoint_for_connection(ctx.connection_id)
1397 .map_err(RouterError::Forwarding)?
1398 .is_some()
1399 {
1400 if !is_known_module_request_op(&frame.body) {
1401 return Ok(vec![control_error_frame(
1402 &frame,
1403 "unsupported_control_frame",
1404 "module-originated channel-0 REQUEST is not supported",
1405 )?]);
1406 }
1407 let request = match parse_module_control_request_from_module(&frame.body) {
1408 Ok(request) => request,
1409 Err((err, ControlRequestBodyError::UnknownOp)) => {
1410 return Ok(vec![control_error_frame(
1411 &frame,
1412 "unsupported_control_frame",
1413 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1414 )?])
1415 }
1416 Err((err, ControlRequestBodyError::InvalidBody)) => {
1417 return Ok(vec![control_error_frame(
1418 &frame,
1419 "invalid_control_body",
1420 format!("malformed module control body: {err}"),
1421 )?])
1422 }
1423 };
1424 let op = module_control_request_op(&request);
1425 let corr = frame.header.corr;
1426 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1427 let result =
1428 self.handle_module_control_request(ctx.connection_id, frame, request);
1429 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1430 return result;
1431 }
1432
1433 if is_known_module_request_op(&frame.body) {
1434 return Ok(vec![control_error_frame(
1435 &frame,
1436 "not_registered",
1437 "catalog.update requires an active module registration owned by this connection",
1438 )?]);
1439 }
1440
1441 let request = match parse_client_control_request(&frame.body) {
1442 Ok(request) => request,
1443 Err((err, ControlRequestBodyError::UnknownOp)) => {
1444 return Ok(vec![control_error_frame(
1445 &frame,
1446 "unknown_control_op",
1447 format!("unknown client control op: {err}"),
1448 )?])
1449 }
1450 Err((err, ControlRequestBodyError::InvalidBody)) => {
1451 return Ok(vec![control_error_frame(
1452 &frame,
1453 "invalid_control_body",
1454 format!("malformed client control body: {err}"),
1455 )?])
1456 }
1457 };
1458 let op = client_control_request_op(&request);
1459 let corr = frame.header.corr;
1460 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1461 #[cfg(test)]
1462 if let Some(delay) = self.control_dispatch_delay {
1463 tokio::time::sleep(delay).await;
1464 }
1465 let result = self
1466 .handle_client_control_request(ctx, frame, request)
1467 .await;
1468 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1469 result
1470 }
1471 FrameType::Push => {
1472 let Some(endpoint) = self
1473 .forwarding
1474 .module_endpoint_for_connection(ctx.connection_id)
1475 .map_err(RouterError::Forwarding)?
1476 else {
1477 return Ok(vec![control_error_frame(
1478 &frame,
1479 "unsupported_control_frame",
1480 "client-originated channel-0 PUSH is not supported",
1481 )?]);
1482 };
1483 self.handle_status_update(endpoint, frame)
1484 }
1485 FrameType::Response | FrameType::Error
1486 if self
1487 .forwarding
1488 .module_endpoint_for_connection(ctx.connection_id)
1489 .map_err(RouterError::Forwarding)?
1490 .is_some() =>
1491 {
1492 self.handle_module_relay_response(ctx.connection_id, frame)
1493 }
1494 ty => Ok(vec![control_error_frame(
1495 &frame,
1496 "unsupported_control_frame",
1497 format!("unsupported channel-0 frame {ty:?}"),
1498 )?]),
1499 }
1500 }
1501
1502 pub fn cleanup_connection(
1503 &self,
1504 connection_id: ConnectionId,
1505 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1506 let crash_closed = self
1507 .registry
1508 .get_module_by_connection(connection_id)?
1509 .and_then(|registration| {
1510 self.forwarding
1511 .module_endpoint_for_connection(connection_id)
1512 .ok()
1513 .flatten()
1514 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1515 .map(|routes| (registration.manifest.module_id, routes))
1516 });
1517 let crash_closed = crash_closed.map(|(module_id, routes)| {
1518 let terminal = match self.supervisor.get(&module_id) {
1519 None => false,
1520 Some(module) => match module.will_recover_after_connection_loss() {
1521 Ok(will_recover) => !will_recover,
1522 Err(err) => {
1523 warn!(
1524 %module_id,
1525 error = %err,
1526 "failed to read crash recovery verdict; reporting non-terminal conservatively"
1527 );
1528 false
1529 }
1530 },
1531 };
1532 let reason = match self.forwarding.is_daemon_draining() {
1536 Ok(true) => RouteCloseReason::Restart,
1537 Ok(false) => RouteCloseReason::Crash,
1538 Err(err) => {
1539 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
1540 RouteCloseReason::Crash
1541 }
1542 };
1543 (module_id, routes, reason, terminal)
1544 });
1545 let registrations = self.deregister_connection(connection_id);
1546 let cleanup = self.forwarding.cleanup_connection_counted(connection_id);
1547 if let Some((module_id, routes, reason, terminal)) = crash_closed {
1552 let abandoned = cleanup
1553 .as_ref()
1554 .map_or(0, |cleanup| cleanup.abandoned_relays);
1555 send_route_control_pushes(
1556 &self.forwarding,
1557 routes,
1558 ClientControlPush::RouteClosed {
1559 module_id,
1560 reason,
1561 drained: false,
1562 abandoned,
1563 excluded_subscriptions: 0,
1564 terminal: Some(terminal),
1565 },
1566 );
1567 }
1568 if let Ok(cleanup) = cleanup {
1569 self.emit_route_goodbyes(cleanup.released);
1570 }
1571 if matches!(®istrations, Ok(r) if !r.is_empty()) {
1575 crate::supervise::notify_registration_release();
1576 self.capability_evaluator.wake_deadline_loop();
1577 self.refresh_capability_requirements();
1578 }
1579 self.supervisor.remove_spawn_subscribers(connection_id);
1580 self.hello_launch_nonces
1583 .lock()
1584 .unwrap_or_else(|poisoned| poisoned.into_inner())
1585 .forget(connection_id);
1586 self.scopes
1587 .write()
1588 .unwrap_or_else(|poisoned| poisoned.into_inner())
1589 .release_connection(connection_id);
1590 registrations
1591 }
1592
1593 pub(crate) fn handle_route_goodbye(
1594 &self,
1595 connection_id: ConnectionId,
1596 route_channel: u16,
1597 route_epoch: u32,
1598 ) -> Result<bool, RouterError> {
1599 debug!(
1600 connection_id = connection_id.get(),
1601 route_channel, route_epoch, "handling route GOODBYE"
1602 );
1603 let RouteRelease::Removed(released_route) = self
1604 .forwarding
1605 .release_client_route(connection_id, route_channel, route_epoch)
1606 .map_err(RouterError::Forwarding)?
1607 else {
1608 return Ok(false);
1609 };
1610 self.emit_route_goodbyes(vec![released_route]);
1611 Ok(true)
1612 }
1613
1614 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
1615 for released in released_routes {
1616 let frame = match Frame::build_with_version(
1617 released.negotiated_ver,
1618 FrameType::Goodbye,
1619 control_flags(),
1620 released.channel,
1621 released.epoch,
1622 0,
1623 Vec::new(),
1624 ) {
1625 Ok(frame) => frame,
1626 Err(err) => {
1627 warn!(
1628 route_channel = released.channel,
1629 error = %err,
1630 "failed to build route GOODBYE frame"
1631 );
1632 continue;
1633 }
1634 };
1635 if !released.close_on_delivery_failure() {
1636 crate::forwarding::send_module_route_goodbye(
1637 &self.counters,
1638 &released.sink,
1639 frame,
1640 released.module_id.as_deref(),
1641 "client route released",
1642 );
1643 continue;
1644 }
1645 if let Err(err) = released.sink.try_send(frame) {
1646 warn!(
1647 target_connection_id = released.connection_id.get(),
1648 route_channel = released.channel,
1649 error = %err,
1650 "route GOODBYE was not delivered to client; closing target connection"
1651 );
1652 if self
1653 .forwarding
1654 .escalate_client_delivery_failure(
1655 released.connection_id,
1656 released.channel,
1657 released.epoch,
1658 CloseReason::new(
1659 "route_goodbye_delivery_failed",
1660 format!(
1661 "failed to enqueue route GOODBYE for channel {}: {err}",
1662 released.channel
1663 ),
1664 ),
1665 crate::forwarding::UndeliveredFrame {
1666 module_id: released.module_id.as_deref(),
1667 sink: &released.sink,
1668 },
1669 )
1670 .unwrap_or(false)
1671 {
1672 self.counters.increment_goodbye_relay_client_failed();
1673 }
1674 }
1675 }
1676 }
1677
1678 fn send_abandoned_route_bind_goodbye(
1690 &self,
1691 module_sink: &crate::FrameSink,
1692 negotiated_ver: u8,
1693 module_channel: u16,
1694 module_epoch: u32,
1695 ) {
1696 let frame = match Frame::build_with_version(
1697 negotiated_ver,
1698 FrameType::Goodbye,
1699 control_flags(),
1700 module_channel,
1701 module_epoch,
1702 0,
1703 Vec::new(),
1704 ) {
1705 Ok(frame) => frame,
1706 Err(err) => {
1707 warn!(
1708 route_channel = module_channel,
1709 error = %err,
1710 "failed to build GOODBYE for abandoned route.bind"
1711 );
1712 return;
1713 }
1714 };
1715 crate::forwarding::send_module_route_goodbye(
1716 &self.counters,
1717 module_sink,
1718 frame,
1719 None,
1720 "abandoned route.bind",
1721 );
1722 }
1723
1724 fn handle_hello(
1725 &self,
1726 connection_id: ConnectionId,
1727 sink: Option<crate::FrameSink>,
1728 frame: Frame,
1729 ) -> Result<Vec<Frame>, RouterError> {
1730 debug!(
1731 connection_id = connection_id.get(),
1732 corr = frame.header.corr,
1733 "handling HELLO"
1734 );
1735 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
1736 Ok(value) => value,
1737 Err(err) => {
1738 return Ok(vec![control_error_frame(
1739 &frame,
1740 "invalid_hello",
1741 format!("malformed HELLO body: {err}"),
1742 )?])
1743 }
1744 };
1745 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
1746 return Ok(vec![control_error_frame(
1747 &frame,
1748 "invalid_capability_grammar",
1749 err.to_string(),
1750 )?]);
1751 }
1752 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
1753 return Ok(vec![control_error_frame(
1754 &frame,
1755 "invalid_manifest",
1756 err.to_string(),
1757 )?]);
1758 }
1759 if let Some(provenance) = hello_value
1760 .get("manifest")
1761 .and_then(|manifest| manifest.get("provenance"))
1762 {
1763 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
1764 return Ok(vec![control_error_frame(
1765 &frame,
1766 "invalid_manifest",
1767 format!("malformed manifest provenance: {err}"),
1768 )?]);
1769 }
1770 }
1771 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
1772 Ok(hello) => hello,
1773 Err(err) => {
1774 return Ok(vec![control_error_frame(
1775 &frame,
1776 "invalid_hello",
1777 format!("malformed HELLO body: {err}"),
1778 )?])
1779 }
1780 };
1781
1782 if hello.protocol_ver != hello.manifest.protocol_ver {
1783 return Ok(vec![control_error_frame(
1784 &frame,
1785 "invalid_manifest",
1786 format!(
1787 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
1788 hello.protocol_ver, hello.manifest.protocol_ver
1789 ),
1790 )?]);
1791 }
1792
1793 if hello.manifest.module_id.trim().is_empty() {
1794 return Ok(vec![control_error_frame(
1795 &frame,
1796 "invalid_manifest",
1797 "manifest module_id must not be empty",
1798 )?]);
1799 }
1800
1801 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
1802 Ok(negotiated_ver) => negotiated_ver,
1803 Err(message) => {
1804 return Ok(vec![control_error_frame(
1805 &frame,
1806 "version_unsupported",
1807 message,
1808 )?])
1809 }
1810 };
1811
1812 let swap_admission = self
1820 .supervisor
1821 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
1822 if swap_admission == SwapHelloAdmission::Refused {
1823 warn!(
1824 module_id = %hello.manifest.module_id,
1825 connection_id = connection_id.get(),
1826 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
1827 );
1828 return Ok(vec![control_error_frame(
1829 &frame,
1830 "swap_token_invalid",
1831 format!(
1832 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
1833 hello.manifest.module_id
1834 ),
1835 )?]);
1836 }
1837 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
1838
1839 if let Some(rejection) = (!swap_candidate)
1847 .then(|| {
1848 self.supervisor.reserved_hello_rejection(
1849 &hello.manifest.module_id,
1850 hello.launch_nonce.as_deref(),
1851 )
1852 })
1853 .flatten()
1854 {
1855 let message = match rejection {
1856 ReservedHelloRejection::Exact { module_id } => format!(
1857 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
1858 ),
1859 ReservedHelloRejection::Prefix {
1860 prefix,
1861 owner_module_id,
1862 } => format!(
1863 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
1864 hello.manifest.module_id
1865 ),
1866 };
1867 return Ok(vec![control_error_frame(
1868 &frame,
1869 "reserved_module",
1870 message,
1871 )?]);
1872 }
1873
1874 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
1875 &hello.manifest.module_id,
1876 hello.manifest.capabilities.as_ref(),
1877 );
1878 if let Some(refusal) = reserved_capability_refusals.first() {
1879 let capability = refusal.capability.clone();
1880 let bound_module = refusal.claimants[0].clone();
1881 log_duplicate_claim_events(reserved_capability_refusals);
1882 return Ok(vec![control_error_frame(
1883 &frame,
1884 "reserved_capability",
1885 format!(
1886 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
1887 capability, bound_module, hello.manifest.module_id
1888 ),
1889 )?]);
1890 }
1891
1892 if self
1895 .forwarding
1896 .connection_has_client_routes(connection_id)
1897 .map_err(RouterError::Forwarding)?
1898 {
1899 return Ok(vec![control_error_frame(
1900 &frame,
1901 "invalid_hello",
1902 "connection has open client routes and cannot also register as a module",
1903 )?]);
1904 }
1905
1906 self.hello_launch_nonces
1911 .lock()
1912 .unwrap_or_else(|poisoned| poisoned.into_inner())
1913 .record(connection_id, hello.launch_nonce.as_deref());
1914 let control_ops = effective_module_control_ops(hello.control_ops);
1915 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
1918 if swap_candidate {
1919 return self.register_swap_candidate(
1920 connection_id,
1921 sink,
1922 &frame,
1923 hello.manifest,
1924 negotiated_ver,
1925 control_ops,
1926 hello_ack,
1927 );
1928 }
1929 let registration = match self.registry.register_with_control_ops(
1930 hello.manifest,
1931 negotiated_ver,
1932 connection_id,
1933 control_ops,
1934 ) {
1935 Ok(registration) => registration,
1936 Err(RegistryError::DuplicateModuleId { module_id }) => {
1937 return Ok(vec![control_error_frame(
1938 &frame,
1939 "duplicate_module_id",
1940 format!(
1941 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
1942 ),
1943 )?])
1944 }
1945 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
1946 return Ok(vec![control_error_frame(
1947 &frame,
1948 "invalid_module_id",
1949 err.to_string(),
1950 )?])
1951 }
1952 Err(err) => {
1953 return Ok(vec![control_error_frame(
1954 &frame,
1955 "registry_error",
1956 err.to_string(),
1957 )?])
1958 }
1959 };
1960
1961 let reply = if let Some(sink) = sink {
1962 let concurrency = manifest_concurrency(®istration.manifest);
1978 if let Err(err) = self.forwarding.register_module_connection_acked(
1979 connection_id,
1980 registration.manifest.module_id.clone(),
1981 negotiated_ver,
1982 concurrency,
1983 sink,
1984 hello_ack,
1985 ) {
1986 if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
1990 crate::supervise::notify_registration_release();
1991 }
1992 return Ok(vec![control_error_frame(
1993 &frame,
1994 forwarding_error_code(&err),
1995 err.to_string(),
1996 )?]);
1997 }
1998 Vec::new()
1999 } else {
2000 vec![hello_ack]
2003 };
2004
2005 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2014 info!(
2015 module_id = %registration.manifest.module_id,
2016 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2017 );
2018 }
2019
2020 self.apply_registration_capabilities(®istration);
2021
2022 info!(
2023 module_id = %registration.manifest.module_id,
2024 module_version = %registration.manifest.module_version,
2025 negotiated_ver,
2026 routable_provider = manifest_provides_routable_role(®istration.manifest),
2027 connection_id = connection_id.get(),
2028 "module registered"
2029 );
2030
2031 Ok(reply)
2032 }
2033
2034 #[allow(clippy::too_many_arguments)]
2044 fn register_swap_candidate(
2045 &self,
2046 connection_id: ConnectionId,
2047 sink: Option<crate::FrameSink>,
2048 frame: &Frame,
2049 manifest: ModuleManifest,
2050 negotiated_ver: u8,
2051 control_ops: Vec<String>,
2052 hello_ack: Frame,
2053 ) -> Result<Vec<Frame>, RouterError> {
2054 let module_id = manifest.module_id.clone();
2055 let registration = match self.registry.register_candidate_with_control_ops(
2056 manifest,
2057 negotiated_ver,
2058 connection_id,
2059 control_ops,
2060 ) {
2061 Ok(registration) => registration,
2062 Err(RegistryError::DuplicateModuleId { module_id }) => {
2063 return Ok(vec![control_error_frame(
2064 frame,
2065 "duplicate_module_id",
2066 format!(
2067 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2068 ),
2069 )?])
2070 }
2071 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2072 return Ok(vec![control_error_frame(
2073 frame,
2074 "invalid_module_id",
2075 err.to_string(),
2076 )?])
2077 }
2078 Err(err) => {
2079 return Ok(vec![control_error_frame(
2080 frame,
2081 "registry_error",
2082 err.to_string(),
2083 )?])
2084 }
2085 };
2086 let reply = if let Some(sink) = sink {
2087 let concurrency = manifest_concurrency(®istration.manifest);
2091 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2092 connection_id,
2093 module_id.clone(),
2094 negotiated_ver,
2095 concurrency,
2096 sink,
2097 hello_ack,
2098 ) {
2099 if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
2100 crate::supervise::notify_registration_release();
2101 }
2102 return Ok(vec![control_error_frame(
2103 frame,
2104 forwarding_error_code(&err),
2105 err.to_string(),
2106 )?]);
2107 }
2108 Vec::new()
2109 } else {
2110 vec![hello_ack]
2111 };
2112 self.supervisor.mark_swap_candidate_admitted(&module_id);
2113 info!(
2114 module_id = %module_id,
2115 module_version = %registration.manifest.module_version,
2116 negotiated_ver,
2117 ready = registration.ready,
2118 connection_id = connection_id.get(),
2119 "swap candidate registered; not routable until cutover"
2120 );
2121 Ok(reply)
2122 }
2123
2124 fn build_hello_ack(
2125 &self,
2126 frame: &Frame,
2127 negotiated_ver: u8,
2128 module_id: &str,
2129 ) -> Result<Frame, RouterError> {
2130 let ack = ModuleHelloAckBody {
2131 negotiated_ver,
2132 subc_ops: module_subc_ops(),
2133 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2134 storage: self
2135 .storage_config
2136 .as_ref()
2137 .map(|cfg| cfg.descriptor_for(module_id)),
2138 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2139 };
2140 let body = serde_json::to_vec(&ack).map_err(|err| {
2141 RouterError::backend(
2142 0,
2143 frame.header.corr,
2144 format!("failed to encode HELLO_ACK: {err}"),
2145 )
2146 })?;
2147
2148 Frame::build_with_version(
2149 negotiated_ver,
2150 FrameType::HelloAck,
2151 control_flags(),
2152 0,
2153 0,
2154 frame.header.corr,
2155 body,
2156 )
2157 .map_err(RouterError::FrameBuild)
2158 }
2159
2160 async fn handle_client_control_request(
2161 &self,
2162 ctx: &RouteCtx,
2163 frame: Frame,
2164 request: ClientControlRequest,
2165 ) -> Result<Vec<Frame>, RouterError> {
2166 match request {
2167 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2168 ClientControlRequest::CatalogList { module_id } => {
2169 self.handle_catalog_list(frame, module_id)
2170 }
2171 ClientControlRequest::RouteOpen {
2172 target,
2173 identity,
2174 consumer_identity,
2175 consumer_capabilities,
2176 role_versions,
2177 admission_facts,
2178 scope,
2179 } => {
2180 self.handle_route_open(
2181 ctx,
2182 frame,
2183 RouteOpenRequest {
2184 target,
2185 identity,
2186 consumer_identity,
2187 consumer_capabilities,
2188 role_versions,
2189 admission_facts,
2190 scope,
2191 },
2192 )
2193 .await
2194 }
2195 ClientControlRequest::RoutePoll {
2196 route_channel,
2197 route_epoch,
2198 kind,
2199 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2200 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2201 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2202 self.handle_supervisor_spawn_snapshot(frame)
2203 }
2204 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2205 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2206 }
2207 ClientControlRequest::SupervisorRestart {
2208 module_id,
2209 drain_timeout_ms,
2210 } => {
2211 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms)
2212 .await
2213 }
2214 ClientControlRequest::SupervisorSwap {
2215 module_id,
2216 ready_timeout_ms,
2217 } => {
2218 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms)
2219 .await
2220 }
2221 ClientControlRequest::SupervisorReload { module_id } => {
2222 self.handle_supervisor_reload(frame, module_id).await
2223 }
2224 ClientControlRequest::SupervisorRescan { preview } => {
2225 self.handle_supervisor_rescan(frame, preview).await
2226 }
2227 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2228 self.handle_supervisor_release_reserved(frame, module_id)
2229 .await
2230 }
2231 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2232 self.handle_supervisor_set_enabled(frame, module_id, enabled)
2233 .await
2234 }
2235 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2236 self.handle_supervisor_health_probe(frame, module_id).await
2237 }
2238 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2239 ClientControlRequest::SupervisorRoutes { module_id } => {
2240 self.handle_supervisor_routes(frame, module_id)
2241 }
2242 ClientControlRequest::SupervisorProvenance { module_id } => {
2243 self.handle_supervisor_provenance(frame, module_id).await
2244 }
2245 ClientControlRequest::SupervisorStderrTail {
2246 module_id,
2247 max_lines,
2248 max_bytes,
2249 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2250 ClientControlRequest::SupervisorTerminals { module_id } => {
2251 self.handle_supervisor_terminals(frame, module_id).await
2252 }
2253 }
2254 }
2255
2256 fn handle_module_control_request(
2257 &self,
2258 connection_id: ConnectionId,
2259 frame: Frame,
2260 request: ModuleControlRequestFromModule,
2261 ) -> Result<Vec<Frame>, RouterError> {
2262 match request {
2263 ModuleControlRequestFromModule::CatalogUpdate {
2264 provides,
2265 capabilities,
2266 ready,
2267 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2268 ModuleControlRequestFromModule::LiveRoots {} => {
2269 let registered = self
2270 .registry
2271 .get_module_by_connection(connection_id)
2272 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2273 let Some(registration) = registered else {
2274 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2275 };
2276 let response = self
2277 .forwarding
2278 .live_roots(®istration.manifest.module_id)
2279 .map_err(RouterError::Forwarding)?;
2280 Ok(vec![control_response_body_frame(
2281 &frame,
2282 &response,
2283 "ModuleControlResponseToModule::LiveRoots",
2284 )?])
2285 }
2286 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
2287 self.handle_scope_sync(connection_id, frame, generation, scopes)
2288 }
2289 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
2290 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
2291 }
2292 }
2293 }
2294
2295 fn handle_scope_sync(
2299 &self,
2300 connection_id: ConnectionId,
2301 frame: Frame,
2302 generation: u64,
2303 scopes: Vec<ScopeRecord>,
2304 ) -> Result<Vec<Frame>, RouterError> {
2305 let Some(registration) = self
2306 .registry
2307 .get_module_by_connection(connection_id)
2308 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2309 else {
2310 return Ok(vec![control_error_frame(
2311 &frame,
2312 "not_registered",
2313 "scope.sync requires an active module registration owned by this connection",
2314 )?]);
2315 };
2316 let owner = registration.manifest.module_id;
2317 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
2318 let is_current_launch = |connection: ConnectionId| {
2319 self.hello_launch_nonces
2320 .lock()
2321 .unwrap_or_else(|poisoned| poisoned.into_inner())
2322 .presented(connection, current_nonce.as_deref())
2323 };
2324 let mut table = self
2329 .scopes
2330 .write()
2331 .unwrap_or_else(|poisoned| poisoned.into_inner());
2332 let outcome = table.sync(&owner, connection_id, is_current_launch, generation, scopes);
2333 let drained = match &outcome {
2334 Ok(applied) => self
2335 .forwarding
2336 .publish_scope_changes(&applied.tag_changes)
2337 .map_err(RouterError::Forwarding)?,
2338 Err(_) => Vec::new(),
2339 };
2340 drop(table);
2341 match outcome {
2342 Ok(applied) => {
2343 let counts = ScopeOutcomeCounts::of(&applied.results);
2344 info!(
2345 owner = %owner,
2346 generation,
2347 records = applied.results.len(),
2348 created = counts.created,
2349 replaced = counts.replaced,
2350 updated = counts.updated,
2351 unchanged = counts.unchanged,
2352 refused = counts.refused,
2353 ended = applied.ended.len(),
2354 tag_changes = applied.tag_changes.len(),
2355 routes_closed = drained.len(),
2356 "scope sync accepted"
2357 );
2358 for refused in applied
2364 .results
2365 .iter()
2366 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
2367 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
2368 {
2369 warn!(
2370 owner = %owner,
2371 generation,
2372 scope_ref = %refused.scope_ref,
2373 scope_epoch = refused.scope_epoch,
2374 code = refused.code.as_deref().unwrap_or(""),
2375 "scope record refused"
2376 );
2377 }
2378 self.close_scope_drained_routes(drained);
2379 let response = ModuleControlResponseToModule::ScopeSync {
2380 generation,
2381 results: applied.results,
2382 ended: applied.ended,
2383 };
2384 Ok(vec![control_response_body_frame(
2385 &frame,
2386 &response,
2387 "ModuleControlResponseToModule::ScopeSync",
2388 )?])
2389 }
2390 Err(refusal) => {
2391 info!(
2392 owner = %owner,
2393 generation,
2394 code = refusal.code,
2395 "scope sync refused"
2396 );
2397 Ok(vec![control_error_frame(
2398 &frame,
2399 refusal.code,
2400 refusal.message,
2401 )?])
2402 }
2403 }
2404 }
2405
2406 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
2412 if drained.is_empty() {
2413 return;
2414 }
2415 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
2416 BTreeMap::new();
2417 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
2418 for route in drained {
2419 warn!(
2420 module_id = %route.module_id,
2421 reason = ?route.reason,
2422 client_connection_id = route.client.connection_id.get(),
2423 route_channel = route.client.channel,
2424 "closing route because its scope changed"
2425 );
2426 pushes
2427 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
2428 .or_insert_with(|| (route.reason, Vec::new()))
2429 .1
2430 .push(EndpointRoute {
2431 goodbye_target: route.client.clone(),
2432 principal: Principal::Unverified,
2433 bound_at: Instant::now(),
2434 draining: false,
2435 drain_reason: None,
2436 });
2437 goodbyes.push(route.module);
2438 goodbyes.push(route.client);
2439 }
2440 for ((module_id, _), (reason, routes)) in pushes {
2441 send_route_control_pushes(
2442 &self.forwarding,
2443 routes,
2444 ClientControlPush::RouteClosed {
2445 module_id,
2446 reason,
2447 drained: false,
2448 abandoned: 0,
2449 excluded_subscriptions: 0,
2450 terminal: Some(false),
2451 },
2452 );
2453 }
2454 self.emit_route_goodbyes(goodbyes);
2455 }
2456
2457 fn handle_scope_describe(
2460 &self,
2461 connection_id: ConnectionId,
2462 frame: Frame,
2463 owner: Principal,
2464 scope_ref: String,
2465 ) -> Result<Vec<Frame>, RouterError> {
2466 let registered = self
2467 .registry
2468 .get_module_by_connection(connection_id)
2469 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2470 if registered.is_none() {
2471 return Ok(vec![control_error_frame(
2472 &frame,
2473 "not_registered",
2474 "scope.describe requires an active module registration owned by this connection",
2475 )?]);
2476 }
2477 let description = self
2478 .scopes
2479 .read()
2480 .unwrap_or_else(|poisoned| poisoned.into_inner())
2481 .describe(&owner, &scope_ref);
2482 let owner_configured = match &owner {
2483 Principal::Reserved { module_id } => self.supervisor.get(module_id).is_some(),
2484 _ => false,
2485 };
2486 let response = ModuleControlResponseToModule::ScopeDescribe {
2487 status: description.status,
2488 scope_epoch: description.scope_epoch,
2489 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
2490 owner_synced: description.owner_synced,
2491 owner_configured,
2492 scope: description.stamp,
2493 };
2494 Ok(vec![control_response_body_frame(
2495 &frame,
2496 &response,
2497 "ModuleControlResponseToModule::ScopeDescribe",
2498 )?])
2499 }
2500
2501 fn handle_catalog_update(
2502 &self,
2503 connection_id: ConnectionId,
2504 frame: Frame,
2505 provides: Vec<ProviderRole>,
2506 capabilities: Option<CapabilityDeclarations>,
2507 ready: Option<bool>,
2508 ) -> Result<Vec<Frame>, RouterError> {
2509 self.refresh_capability_requirements();
2510 let Some(registration) = self
2511 .registry
2512 .get_module_by_connection(connection_id)
2513 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2514 else {
2515 return Ok(vec![control_error_frame(
2516 &frame,
2517 "not_registered",
2518 "catalog.update requires an active module registration owned by this connection",
2519 )?]);
2520 };
2521
2522 if let Some(message) =
2523 catalog_update_frozen_field_message(®istration.manifest, &provides)
2524 {
2525 return Ok(vec![control_error_frame(
2526 &frame,
2527 "catalog_update_frozen_field",
2528 message,
2529 )?]);
2530 }
2531
2532 let mut candidate = registration.manifest.clone();
2533 candidate.provides = provides.clone();
2534 candidate.capabilities = capabilities
2535 .clone()
2536 .or_else(|| registration.manifest.capabilities.clone());
2537 if let Err(err) = candidate.validate_capability_grammar() {
2538 return Ok(vec![control_error_frame(
2539 &frame,
2540 "invalid_capability_grammar",
2541 err.to_string(),
2542 )?]);
2543 }
2544
2545 let updated = self
2546 .registry
2547 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
2548 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2549 if updated.is_none() {
2550 return Ok(vec![control_error_frame(
2551 &frame,
2552 "not_registered",
2553 "catalog.update requires an active module registration owned by this connection",
2554 )?]);
2555 }
2556 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
2557 log_duplicate_claim_events(
2558 self.capability_evaluator
2559 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
2560 );
2561 }
2562 if capability_census_trigger(
2563 registration.manifest.capabilities.as_ref(),
2564 updated
2565 .as_ref()
2566 .and_then(|entry| entry.manifest.capabilities.as_ref()),
2567 ) {
2568 self.enforce_capability_denies();
2569 }
2570 self.refresh_capability_requirements();
2571
2572 let response = ModuleControlResponseToModule::CatalogUpdate {};
2573 control_response_body_frame(
2574 &frame,
2575 &response,
2576 "ModuleControlResponseToModule::CatalogUpdate",
2577 )
2578 .map(|frame| vec![frame])
2579 }
2580
2581 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
2582 self.refresh_capability_requirements();
2583 let mut counters = self.counters.snapshot();
2589 if let (Ok((connections_with_routes, max)), Some(obj)) = (
2590 self.forwarding.client_route_concentration(),
2591 counters.as_object_mut(),
2592 ) {
2593 obj.insert(
2594 "client_connections_with_routes".into(),
2595 connections_with_routes.into(),
2596 );
2597 obj.insert("max_routes_on_one_connection".into(), max.into());
2598 }
2599 if let (Some(open_breakers), Some(obj)) = (
2605 self.route_bind_breakers.open_snapshot(),
2606 counters.as_object_mut(),
2607 ) {
2608 obj.insert("route_bind_breakers_open".into(), open_breakers);
2609 }
2610 let response = ClientControlResponse::ServerDescribe {
2611 protocol_ver: PROTOCOL_VERSION,
2612 subc_ops: subc_ops(),
2613 capabilities: self.subc_capabilities.as_ref().to_vec(),
2614 connected_clients: self.connected_clients.count(),
2615 counters: Some(counters),
2616 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
2617 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
2618 capability_requirements: self.capability_requirement_statuses(),
2619 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2620 };
2621 Ok(vec![control_response_body_frame(
2622 &frame,
2623 &response,
2624 "ClientControlResponse::ServerDescribe",
2625 )?])
2626 }
2627
2628 fn handle_catalog_list(
2629 &self,
2630 frame: Frame,
2631 module_id: Option<String>,
2632 ) -> Result<Vec<Frame>, RouterError> {
2633 let (generation, modules) = self.registry.list_modules().map_err(|err| {
2634 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
2635 })?;
2636 let entries = modules
2637 .into_iter()
2638 .filter(|registration| {
2639 module_id
2640 .as_deref()
2641 .map(|wanted| registration.manifest.module_id == wanted)
2642 .unwrap_or(true)
2643 })
2644 .map(|registration| {
2645 let not_ready = self.not_ready_reason(®istration);
2646 let roles = registration.manifest.provides;
2647 CatalogEntry {
2648 module_id: registration.manifest.module_id,
2649 ready: not_ready.is_none(),
2650 not_ready,
2651 module_version: Some(registration.manifest.module_version),
2652 roles,
2653 control_ops: registration.control_ops,
2654 capabilities: registration.manifest.capabilities,
2655 self_signals: registration.manifest.self_signals,
2656 }
2657 })
2658 .collect();
2659 let response = ClientControlResponse::CatalogList {
2660 generation,
2661 modules: entries,
2662 subc_ops: subc_ops(),
2663 };
2664 Ok(vec![control_response_body_frame(
2665 &frame,
2666 &response,
2667 "ClientControlResponse::CatalogList",
2668 )?])
2669 }
2670
2671 fn route_open_principal(
2672 &self,
2673 frame: &Frame,
2674 consumer_identity: Option<ConsumerIdentity>,
2675 ) -> Result<Result<Principal, Frame>, RouterError> {
2676 let Some(consumer_identity) = consumer_identity else {
2677 return Ok(Ok(Principal::Direct));
2678 };
2679
2680 if self.supervisor.spawned_consumer_authorized(
2681 &consumer_identity.module_id,
2682 &consumer_identity.launch_nonce,
2683 ) {
2684 return Ok(Ok(Principal::Reserved {
2685 module_id: consumer_identity.module_id,
2686 }));
2687 }
2688
2689 Ok(Err(control_error_frame(
2690 frame,
2691 "bad_consumer_identity",
2692 format!(
2693 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
2694 consumer_identity.module_id
2695 ),
2696 )?))
2697 }
2698
2699 fn route_open_refusal_frame(
2706 &self,
2707 ctx: &RouteCtx,
2708 frame: &Frame,
2709 module_id: &str,
2710 reason: &'static str,
2711 code: &'static str,
2712 message: impl Into<String>,
2713 ) -> Result<Frame, RouterError> {
2714 self.observe_route_open_refusal(ctx, module_id, reason, code);
2715 control_error_frame(frame, code, message.into())
2716 }
2717
2718 fn route_open_breaker_refusal_frame(
2736 &self,
2737 ctx: &RouteCtx,
2738 frame: &Frame,
2739 module_id: &str,
2740 consecutive_timeouts: u32,
2741 retry_in: Duration,
2742 probe_in_flight: bool,
2743 ) -> Result<Frame, RouterError> {
2744 self.counters
2745 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
2746 debug!(
2747 target: "control",
2748 code = "module_timeout",
2749 module_id = ?module_id,
2750 connection_id = ctx.connection_id.get(),
2751 consecutive_timeouts,
2752 retry_in_ms = retry_in.as_millis() as u64,
2753 probe_in_flight,
2754 "route.open refused by open bind-relay breaker"
2755 );
2756 let detail = if probe_in_flight {
2757 "one probe bind is already in flight; retry once it settles".to_string()
2758 } else {
2759 format!("not relaying for another {retry_in:?}")
2760 };
2761 control_error_frame(
2762 frame,
2763 "module_timeout",
2764 format!(
2765 "module_id '{module_id}' failed {consecutive_timeouts} consecutive route.bind \
2766 relays; {detail}"
2767 ),
2768 )
2769 }
2770
2771 fn observe_route_open_refusal(
2784 &self,
2785 ctx: &RouteCtx,
2786 module_id: &str,
2787 reason: &'static str,
2788 code: &'static str,
2789 ) {
2790 self.counters.increment_route_open_refused(code);
2791 info!(
2792 target: "control",
2793 code,
2794 reason,
2795 module_id = ?module_id,
2796 connection_id = ctx.connection_id.get(),
2797 "route.open refused"
2798 );
2799 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
2800 self.route_outages.record_not_serving(module_id, reason);
2801 }
2802 }
2803
2804 fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
2849 self.route_outages.record_accepted(module_id);
2850 self.counters.increment_route_open_accepted(principal);
2851 info!(
2852 target: "control",
2853 principal,
2854 module_id,
2855 connection_id = ctx.connection_id.get(),
2856 "route.open accepted"
2857 );
2858 }
2859
2860 fn supervised_absent_route_open_refusal_frame(
2861 &self,
2862 ctx: &RouteCtx,
2863 frame: &Frame,
2864 module_id: &str,
2865 code: &'static str,
2866 status: &crate::supervise::ModuleStatus,
2867 ) -> Result<Frame, RouterError> {
2868 self.counters.increment_route_open_refused(code);
2869 info!(
2870 target: "control",
2871 code,
2872 reason = "supervised_not_registered",
2873 module_id = ?module_id,
2874 connection_id = ctx.connection_id.get(),
2875 state = %status.state,
2876 enabled = status.enabled,
2877 live = status.live,
2878 "route.open refused"
2879 );
2880 self.route_outages
2884 .record_not_serving(module_id, "supervised_not_registered");
2885 control_error_frame(
2886 frame,
2887 code,
2888 format!(
2889 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
2890 status.state, status.enabled, status.live
2891 ),
2892 )
2893 }
2894
2895 async fn handle_route_open(
2896 &self,
2897 ctx: &RouteCtx,
2898 frame: Frame,
2899 request: RouteOpenRequest,
2900 ) -> Result<Vec<Frame>, RouterError> {
2901 let RouteOpenRequest {
2902 target,
2903 mut identity,
2904 consumer_identity,
2905 consumer_capabilities,
2906 role_versions,
2907 admission_facts,
2908 scope,
2909 } = request;
2910 let target_module_id = target_module_id(&target).to_string();
2911 debug!(
2912 connection_id = ctx.connection_id.get(),
2913 corr = frame.header.corr,
2914 module_id = %target_module_id,
2915 "handling route.open"
2916 );
2917
2918 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
2924 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
2925 self.observe_route_open_refusal(
2926 ctx,
2927 &target_module_id,
2928 "invalid_role_versions",
2929 error_codes::INVALID_REQUEST,
2930 );
2931 return Ok(vec![control_error_body_frame(
2932 &frame,
2933 ErrorBody {
2934 code: error_codes::INVALID_REQUEST.to_string(),
2935 message: error.to_string(),
2936 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
2937 },
2938 )?]);
2939 }
2940
2941 let Some(registration) = self
2960 .registry
2961 .get_module(&target_module_id)
2962 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2963 else {
2964 if let Some((status, warming)) =
2965 self.supervisor_status(&target_module_id, frame.header.corr)?
2966 {
2967 if status.protocol == ModuleProtocol::None {
2974 return Ok(vec![self.route_open_refusal_frame(
2975 ctx,
2976 &frame,
2977 &target_module_id,
2978 "protocol_none",
2979 error_codes::MODULE_NO_PROTOCOL,
2980 format!(
2981 "module_id '{target_module_id}' is declared protocol: none; \
2982 it speaks no subc wire and serves no routes"
2983 ),
2984 )?]);
2985 }
2986 let code = if warming {
2987 "module_warming"
2988 } else {
2989 "target_unavailable"
2990 };
2991 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
2992 ctx,
2993 &frame,
2994 &target_module_id,
2995 code,
2996 &status,
2997 )?]);
2998 }
2999 if let Some(removed_ago_ms) =
3000 self.supervisor.removal_tombstone_age_ms(&target_module_id)
3001 {
3002 return Ok(vec![self.route_open_refusal_frame(
3003 ctx,
3004 &frame,
3005 &target_module_id,
3006 "removed",
3007 error_codes::MODULE_REMOVED,
3008 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
3009 )?]);
3010 }
3011 return Ok(vec![self.route_open_refusal_frame(
3012 ctx,
3013 &frame,
3014 &target_module_id,
3015 "not_registered",
3016 error_codes::UNKNOWN_MODULE,
3017 format!("module_id '{target_module_id}' is not registered"),
3018 )?]);
3019 };
3020
3021 if !registration.ready {
3025 self.counters
3026 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
3027 info!(
3028 target: "control",
3029 code = error_codes::MODULE_WARMING,
3030 module_id = ?target_module_id,
3031 connection_id = ctx.connection_id.get(),
3032 reason = "declared_not_ready",
3033 "route.open refused"
3034 );
3035 self.route_outages
3038 .record_not_serving(&target_module_id, "declared_not_ready");
3039 return Ok(vec![control_error_body_frame(
3040 &frame,
3041 ErrorBody {
3042 code: error_codes::MODULE_WARMING.to_string(),
3043 message: format!(
3044 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
3045 ),
3046 detail: Some(serde_json::json!({
3047 "reason": "declared_not_ready"
3048 })),
3049 },
3050 )?]);
3051 }
3052
3053 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
3072 self.counters
3073 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
3074 info!(
3075 target: "control",
3076 code = error_codes::MODULE_WARMING,
3077 module_id = ?target_module_id,
3078 connection_id = ctx.connection_id.get(),
3079 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3080 capability = %capability,
3081 "route.open refused"
3082 );
3083 return Ok(vec![control_error_body_frame(
3084 &frame,
3085 ErrorBody {
3086 code: error_codes::MODULE_WARMING.to_string(),
3087 message: format!(
3088 "module_id '{target_module_id}' requires capability '{capability}', \
3089 which no registered module provides; retry"
3090 ),
3091 detail: Some(serde_json::json!({
3092 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3093 "capability": capability,
3094 })),
3095 },
3096 )?]);
3097 }
3098
3099 if !target_has_required_role(&target, ®istration.manifest.provides) {
3100 return Ok(vec![self.route_open_refusal_frame(
3101 ctx,
3102 &frame,
3103 &target_module_id,
3104 "role_not_provided",
3105 "target_unavailable",
3106 format!("module_id '{target_module_id}' does not provide the requested target"),
3107 )?]);
3108 }
3109
3110 if registration.state != ChannelState::Active {
3111 return Ok(vec![self.route_open_refusal_frame(
3112 ctx,
3113 &frame,
3114 &target_module_id,
3115 "registration_not_active",
3116 "target_unavailable",
3117 format!("module_id '{target_module_id}' is not active"),
3118 )?]);
3119 }
3120
3121 if self
3122 .forwarding
3123 .module_is_draining(&target_module_id)
3124 .map_err(RouterError::Forwarding)?
3125 {
3126 return Ok(vec![self.route_open_refusal_frame(
3127 ctx,
3128 &frame,
3129 &target_module_id,
3130 "reloading",
3131 "module_reloading",
3132 format!("module_id '{target_module_id}' is reloading"),
3133 )?]);
3134 }
3135
3136 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
3137 process_liveness.process_live(&target_module_id) == Some(false)
3138 }) {
3139 if process_liveness.process_replacing(&target_module_id) {
3146 return Ok(vec![self.route_open_refusal_frame(
3147 ctx,
3148 &frame,
3149 &target_module_id,
3150 "reloading",
3151 "module_reloading",
3152 format!("module_id '{target_module_id}' is reloading"),
3153 )?]);
3154 }
3155 return Ok(vec![self.route_open_refusal_frame(
3156 ctx,
3157 &frame,
3158 &target_module_id,
3159 "supervisor_not_live",
3160 "target_unavailable",
3161 format!("module_id '{target_module_id}' is not live"),
3162 )?]);
3163 }
3164
3165 if !self
3166 .forwarding
3167 .has_live_module_connection(&target_module_id)
3168 .map_err(RouterError::Forwarding)?
3169 {
3170 return Ok(vec![self.route_open_refusal_frame(
3171 ctx,
3172 &frame,
3173 &target_module_id,
3174 "no_forwarding_connection",
3175 "target_unavailable",
3176 format!("module_id '{target_module_id}' has no live forwarding connection"),
3177 )?]);
3178 }
3179
3180 if let Some(error) =
3181 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
3182 {
3183 self.observe_route_open_refusal(
3184 ctx,
3185 &target_module_id,
3186 "op_not_allowed",
3187 "op_not_allowed",
3188 );
3189 return Ok(vec![error]);
3190 }
3191
3192 let principal = match self.route_open_principal(&frame, consumer_identity)? {
3193 Ok(principal) => principal,
3194 Err(error) => {
3195 self.observe_route_open_refusal(
3196 ctx,
3197 &target_module_id,
3198 "bad_consumer_identity",
3199 "bad_consumer_identity",
3200 );
3201 return Ok(vec![error]);
3202 }
3203 };
3204
3205 if let Principal::Reserved {
3209 module_id: opening_module_id,
3210 } = &principal
3211 {
3212 if let Some(opening_registration) = self
3213 .registry
3214 .get_module(opening_module_id)
3215 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3216 {
3217 if let Some(capability) =
3218 denied_capability(&opening_registration.manifest, ®istration.manifest)
3219 {
3220 warn!(
3221 opening_module_id,
3222 target_module_id,
3223 capability,
3224 "refusing route.open because an attested capability deny edge matches"
3225 );
3226 return Ok(vec![self.route_open_refusal_frame(
3227 ctx,
3228 &frame,
3229 &target_module_id,
3230 "capability_deny_edge",
3231 "capability_forbidden",
3232 format!(
3233 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
3234 ),
3235 )?]);
3236 }
3237 }
3238 }
3239
3240 if admission_facts.is_some() {
3241 let carrier_matches = matches!(
3242 &principal,
3243 Principal::Reserved { module_id }
3244 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
3245 );
3246 if !carrier_matches {
3247 return Ok(vec![self.route_open_refusal_frame(
3248 ctx,
3249 &frame,
3250 &target_module_id,
3251 "admission_facts_carrier_not_permitted",
3252 "admission_facts_not_permitted",
3253 "admission facts may only be carried by the configured reserved module",
3254 )?]);
3255 }
3256
3257 let target_allowed = self
3258 .admission_facts_targets
3259 .as_ref()
3260 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
3261 if !target_allowed {
3262 return Ok(vec![self.route_open_refusal_frame(
3263 ctx,
3264 &frame,
3265 &target_module_id,
3266 "admission_facts_target_not_listed",
3267 "admission_facts_target_not_allowed",
3268 format!(
3269 "admission facts are not permitted for target module_id '{target_module_id}'"
3270 ),
3271 )?]);
3272 }
3273
3274 }
3278
3279 let (bound_scope, scope_stamp) = match scope {
3285 None => (None, None),
3286 Some(selector) => {
3287 let owner_configured = match &selector.owner {
3288 Principal::Reserved { module_id } => self.supervisor.get(module_id).is_some(),
3289 _ => false,
3290 };
3291 let admitted = self
3292 .scopes
3293 .read()
3294 .unwrap_or_else(|poisoned| poisoned.into_inner())
3295 .admit(&principal, &target_module_id, &selector, owner_configured);
3296 match admitted {
3297 Ok(admission) => (
3298 Some(BoundScope {
3299 owner: admission.owner,
3300 scope_ref: admission.stamp.scope_ref.clone(),
3301 tag: admission.tag,
3302 }),
3303 Some(admission.stamp),
3304 ),
3305 Err(refusal) => {
3306 return Ok(vec![self.route_open_refusal_frame(
3307 ctx,
3308 &frame,
3309 &target_module_id,
3310 refusal.code,
3311 refusal.code,
3312 refusal.message,
3313 )?]);
3314 }
3315 }
3316 }
3317 };
3318
3319 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
3341 Ok(project_root) => project_root,
3342 Err(err) => {
3343 return Ok(vec![control_error_frame(
3344 &frame,
3345 "invalid_project_root",
3346 err.to_string(),
3347 )?])
3348 }
3349 };
3350 identity.project_root = project_root.as_path().to_path_buf();
3351
3352 let _concurrency_guard = match self
3363 .route_bind_concurrency
3364 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
3365 {
3366 Ok(guard) => guard,
3367 Err(in_flight) => {
3368 return Ok(vec![self.route_open_target_capacity_refusal(
3369 ctx,
3370 &frame,
3371 &target_module_id,
3372 in_flight,
3373 )?]);
3374 }
3375 };
3376
3377 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
3380 RouteBindAdmission::Admitted { guard, probe } => {
3381 if probe {
3382 info!(
3383 module_id = %target_module_id,
3384 connection_id = ctx.connection_id.get(),
3385 "route.bind breaker half-open: admitting one probe"
3386 );
3387 }
3388 guard
3389 }
3390 RouteBindAdmission::Refused {
3391 consecutive_timeouts,
3392 retry_in,
3393 probe_in_flight,
3394 } => {
3395 return Ok(vec![self.route_open_breaker_refusal_frame(
3396 ctx,
3397 &frame,
3398 &target_module_id,
3399 consecutive_timeouts,
3400 retry_in,
3401 probe_in_flight,
3402 )?]);
3403 }
3404 };
3405
3406 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
3411 let relay_deadline = Instant::now() + route_bind_relay_timeout;
3412 let pending = match self
3413 .forwarding
3414 .begin_route_bind_relay_for(
3415 ctx.connection_id,
3416 ctx.egress.clone(),
3417 response_version(&frame),
3418 frame.header.corr,
3419 &target_module_id,
3420 principal.clone(),
3421 bound_scope,
3422 Some(project_root),
3423 relay_deadline,
3424 )
3425 .await
3426 {
3427 Ok(pending) => pending,
3428 Err(err) => {
3429 return Ok(vec![self.route_open_refusal_frame(
3430 ctx,
3431 &frame,
3432 &target_module_id,
3433 "relay_reservation_failed",
3434 forwarding_error_code(&err),
3435 err.to_string(),
3436 )?])
3437 }
3438 };
3439 let crate::forwarding::PendingRouteBindRelay {
3440 endpoint,
3441 module_sink,
3442 negotiated_ver,
3443 client_channel,
3444 client_epoch,
3445 module_channel,
3446 module_epoch,
3447 corr: relay_corr,
3448 receiver,
3449 } = pending;
3450 let mut reservation =
3451 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
3452
3453 debug!(
3454 connection_id = ctx.connection_id.get(),
3455 client_channel,
3456 client_epoch,
3457 module_channel,
3458 module_epoch,
3459 "reserved route handle pair"
3460 );
3461 let principal_label = match &principal {
3464 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
3465 Principal::Direct => "direct".to_string(),
3466 other => format!("{other:?}"),
3467 };
3468 let relay = ModuleControlRequest::RouteBind {
3469 route_channel: module_channel,
3470 epoch: module_epoch,
3471 target,
3472 identity,
3473 principal: Some(principal),
3474 consumer_capabilities,
3475 role_versions,
3476 admission_facts,
3477 scope: scope_stamp,
3478 };
3479 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
3480 RouterError::backend(
3481 0,
3482 frame.header.corr,
3483 format!("failed to encode route.bind request: {err}"),
3484 )
3485 })?;
3486 let relay_frame = Frame::build_with_version(
3487 negotiated_ver,
3488 FrameType::Request,
3489 control_flags(),
3490 0,
3491 0,
3492 relay_corr,
3493 relay_body,
3494 )
3495 .map_err(RouterError::FrameBuild)?;
3496
3497 if let Err(err) = module_sink.send(relay_frame).await {
3498 reservation.release_and_disarm();
3499 return Ok(vec![self.route_open_refusal_frame(
3500 ctx,
3501 &frame,
3502 &target_module_id,
3503 "relay_send_failed",
3504 "target_unavailable",
3505 err.to_string(),
3506 )?]);
3507 }
3508
3509 if !self
3510 .forwarding
3511 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
3512 .map_err(RouterError::Forwarding)?
3513 {
3514 self.send_abandoned_route_bind_goodbye(
3515 &module_sink,
3516 negotiated_ver,
3517 module_channel,
3518 module_epoch,
3519 );
3520 }
3521
3522 match timeout_at(relay_deadline, receiver).await {
3523 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
3524 reservation.disarm();
3525 if breaker.record_accepted() {
3526 info!(
3527 module_id = %target_module_id,
3528 "route.bind breaker closed: the probe was accepted"
3529 );
3530 }
3531 self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
3532 Ok(Vec::new())
3533 }
3534 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
3535 reservation.release_and_disarm();
3536 breaker.record_inconclusive();
3540 let scope_code = match body.code.as_str() {
3544 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
3545 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
3546 _ => None,
3547 };
3548 if let Some(code) = scope_code {
3549 self.observe_route_open_refusal(
3550 ctx,
3551 &target_module_id,
3552 "scope_changed_before_commit",
3553 code,
3554 );
3555 return Ok(vec![control_error_body_frame(&frame, body)?]);
3556 }
3557 self.counters
3558 .increment_route_open_refused("module_rejected");
3559 info!(
3560 target: "control",
3561 code = "module_rejected",
3562 module_code = ?body.code,
3563 module_id = ?target_module_id,
3564 connection_id = ctx.connection_id.get(),
3565 "route.open refused"
3566 );
3567 Ok(vec![control_error_body_frame(&frame, body)?])
3568 }
3569 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
3570 reservation.release_and_disarm();
3571 breaker.record_inconclusive();
3572 tracing::warn!(
3579 module_id = %target_module_id,
3580 "route.bind relay abandoned: {message}"
3581 );
3582 Ok(vec![self.route_open_refusal_frame(
3583 ctx,
3584 &frame,
3585 &target_module_id,
3586 "relay_abandoned",
3587 "target_unavailable",
3588 message,
3589 )?])
3590 }
3591 Ok(Err(_)) => {
3592 reservation.release_and_disarm();
3593 breaker.record_inconclusive();
3594 Ok(vec![self.route_open_refusal_frame(
3595 ctx,
3596 &frame,
3597 &target_module_id,
3598 "relay_waiter_canceled",
3599 "target_unavailable",
3600 "route.bind relay waiter was canceled before the module responded",
3601 )?])
3602 }
3603 Err(_) => {
3604 reservation.release_and_disarm();
3605 if let Some(opened) = breaker.record_timeout(
3609 self.route_bind_breaker_threshold,
3610 self.route_bind_breaker_cooldown,
3611 ) {
3612 warn!(
3613 module_id = %target_module_id,
3614 consecutive_timeouts = opened.consecutive_timeouts,
3615 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
3616 reopened_after_probe = opened.reopened_after_probe,
3617 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
3618 );
3619 }
3620 tracing::warn!(
3628 module_id = %target_module_id,
3629 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
3630 "route.bind relay timed out: module did not ack within budget"
3631 );
3632 Ok(vec![self.route_open_refusal_frame(
3633 ctx,
3634 &frame,
3635 &target_module_id,
3636 "relay_timed_out",
3637 "module_timeout",
3638 format!(
3639 "module_id '{target_module_id}' did not answer route.bind within {:?}",
3640 route_bind_relay_timeout
3641 ),
3642 )?])
3643 }
3644 }
3645 }
3646
3647 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3648 let response = ClientControlResponse::SupervisorSpawnSnapshot {
3649 snapshot: self.supervisor.spawn_snapshot(),
3650 };
3651 Ok(vec![control_response_body_frame(
3652 &frame,
3653 &response,
3654 "ClientControlResponse::SupervisorSpawnSnapshot",
3655 )?])
3656 }
3657
3658 fn handle_supervisor_spawn_subscribe(
3659 &self,
3660 ctx: &RouteCtx,
3661 frame: Frame,
3662 since: Option<SpawnCursor>,
3663 ) -> Result<Vec<Frame>, RouterError> {
3664 match self.supervisor.subscribe_spawns(
3665 ctx.connection_id,
3666 frame.header.corr,
3667 response_version(&frame),
3668 since,
3669 ctx.egress.clone(),
3670 ) {
3671 Ok(()) => Ok(Vec::new()),
3672 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
3673 Ok(vec![control_error_body_frame(
3674 &frame,
3675 ErrorBody {
3676 code: "spawn_cursor_incarnation_mismatch".to_string(),
3677 message: "spawn cursor belongs to a different daemon incarnation"
3678 .to_string(),
3679 detail: Some(serde_json::json!({
3680 "current_daemon_incarnation": current
3681 })),
3682 },
3683 )?])
3684 }
3685 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
3686 &frame,
3687 ErrorBody {
3688 code: "spawn_cursor_too_old".to_string(),
3689 message: "spawn cursor predates the retained event ring".to_string(),
3690 detail: Some(serde_json::json!({
3691 "oldest_retained_cursor": oldest
3692 })),
3693 },
3694 )?]),
3695 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
3696 0,
3697 frame.header.corr,
3698 format!("failed to open supervisor spawn subscription: {error}"),
3699 )),
3700 }
3701 }
3702
3703 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3704 let generation = self
3705 .registry
3706 .generation()
3707 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3708 let mut modules = Vec::new();
3709 for module in self.supervisor.list() {
3710 let status = module.status_for_control("list").map_err(|err| {
3711 RouterError::backend(
3712 0,
3713 frame.header.corr,
3714 format!("failed to read supervisor status: {err}"),
3715 )
3716 })?;
3717 let (configured, _) = module.configuration().map_err(|err| {
3718 RouterError::backend(
3719 0,
3720 frame.header.corr,
3721 format!("failed to read module configuration: {err}"),
3722 )
3723 })?;
3724 let image = module.running_image_agreement().await;
3726 let resources = Some(module.child_resource_usage());
3729 let pending_reload = Some(reload_verdict(
3730 &configured.program,
3731 status.spawned_from.as_deref(),
3732 image,
3733 ));
3734 modules.push(SupervisorEntry {
3735 launch_nonce_env: Some(
3736 configured.protocol != subc_control::ModuleProtocol::None
3737 && (!cfg!(unix) || configured.launch_nonce_env),
3738 ),
3739 module_id: status.module_id,
3740 state: status.state.to_string(),
3741 enabled: status.enabled,
3742 live: status.live,
3743 protocol: status.protocol,
3744 health: status.health.status,
3745 pending_reload,
3746 last_probe_ms: status.health.last_probe_ms,
3747 last_exit_code: status.last_exit.as_ref().and_then(|e| e.code),
3748 last_exit_signal: status.last_exit.as_ref().and_then(|e| e.signal),
3749 last_exit_ms: status.last_exit.as_ref().map(|e| e.at_ms),
3750 last_exit_kind: status.last_exit.as_ref().map(|e| e.kind.into()),
3751 restart_count: Some(status.restart_count),
3752 max_restarts: Some(status.max_restarts),
3753 lifetime_restarts: Some(status.lifetime_restarts),
3754 spawn_generation: Some(status.spawn_generation),
3755 restart_window_secs: Some(status.restart_window.as_secs()),
3756 drain_timeout_ms: Some(status.drain_timeout.as_millis() as u64),
3757 restart_backoff_ms: Some(status.restart_backoff.as_millis() as u64),
3758 restart_max_backoff_ms: Some(status.restart_max_backoff.as_millis() as u64),
3759 resources,
3760 });
3761 }
3762 let response = ClientControlResponse::SupervisorList {
3763 generation,
3764 modules,
3765 };
3766 Ok(vec![control_response_body_frame(
3767 &frame,
3768 &response,
3769 "ClientControlResponse::SupervisorList",
3770 )?])
3771 }
3772
3773 fn handle_supervisor_stderr_tail(
3774 &self,
3775 frame: Frame,
3776 module_id: String,
3777 max_lines: Option<u32>,
3778 max_bytes: Option<u32>,
3779 ) -> Result<Vec<Frame>, RouterError> {
3780 let Some(module) = self.supervisor.get(&module_id) else {
3781 return Ok(vec![control_error_frame(
3782 &frame,
3783 "unknown_module",
3784 format!("module_id '{module_id}' is not supervised"),
3785 )?]);
3786 };
3787
3788 let snapshot = module.stderr_tail(
3789 max_lines.map(|value| value as usize),
3790 max_bytes.map(|value| value as usize),
3791 );
3792
3793 let response = ClientControlResponse::SupervisorStderrTail {
3794 module_id,
3795 tail: StderrTail {
3796 capture: match snapshot.capture {
3797 CaptureState::Captured => StderrCaptureState::Captured,
3798 CaptureState::Incomplete { reason } => {
3799 StderrCaptureState::Incomplete { reason }
3800 }
3801 CaptureState::NotCaptured { reason } => {
3802 StderrCaptureState::NotCaptured { reason }
3803 }
3804 },
3805 entries: snapshot
3806 .entries
3807 .into_iter()
3808 .map(|entry| match entry {
3809 TailEntry::Line {
3810 text,
3811 truncated,
3812 at_ms,
3813 } => StderrTailEntry::Line {
3814 text,
3815 truncated,
3816 at_ms,
3817 },
3818 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
3819 })
3820 .collect(),
3821 dropped_lines: snapshot.dropped_lines,
3822 },
3823 };
3824 Ok(vec![control_response_body_frame(
3825 &frame,
3826 &response,
3827 "ClientControlResponse::SupervisorStderrTail",
3828 )?])
3829 }
3830
3831 async fn handle_supervisor_terminals(
3832 &self,
3833 frame: Frame,
3834 module_id: String,
3835 ) -> Result<Vec<Frame>, RouterError> {
3836 let Some(module) = self.supervisor.get(&module_id) else {
3837 return Ok(vec![control_error_frame(
3838 &frame,
3839 "unknown_module",
3840 format!("module_id '{module_id}' is not supervised"),
3841 )?]);
3842 };
3843
3844 let terminals = module
3847 .read_durable_terminal_history()
3848 .await
3849 .map_err(|error| {
3850 RouterError::backend(
3851 0,
3852 frame.header.corr,
3853 format!("failed to read terminal history: {error}"),
3854 )
3855 })?;
3856 let response = ClientControlResponse::SupervisorTerminals {
3857 module_id,
3858 terminals,
3859 };
3860 Ok(vec![control_response_body_frame(
3861 &frame,
3862 &response,
3863 "ClientControlResponse::SupervisorTerminals",
3864 )?])
3865 }
3866
3867 fn handle_supervisor_routes(
3868 &self,
3869 frame: Frame,
3870 module_id: Option<String>,
3871 ) -> Result<Vec<Frame>, RouterError> {
3872 let modules = self
3873 .forwarding
3874 .route_census(module_id.as_deref())
3875 .map_err(RouterError::Forwarding)?
3876 .into_iter()
3877 .map(|(module_id, routes)| SupervisorRouteModule {
3878 module_id,
3879 routes: routes
3880 .into_iter()
3881 .map(|route| SupervisorRoute {
3882 consumer: match route.principal {
3883 Principal::Reserved { module_id } => {
3884 SupervisorRouteConsumer::Reserved { module_id }
3885 }
3886 Principal::Direct | Principal::Unverified => {
3887 SupervisorRouteConsumer::Direct {
3888 connection_id: route.goodbye_target.connection_id.get(),
3889 }
3890 }
3891 },
3892 age_ms: Instant::now()
3893 .saturating_duration_since(route.bound_at)
3894 .as_millis()
3895 .try_into()
3896 .unwrap_or(u64::MAX),
3897 draining: route.draining,
3898 drain_reason: route.drain_reason,
3899 })
3900 .collect(),
3901 })
3902 .collect();
3903 let response = ClientControlResponse::SupervisorRoutes { modules };
3904 Ok(vec![control_response_body_frame(
3905 &frame,
3906 &response,
3907 "ClientControlResponse::SupervisorRoutes",
3908 )?])
3909 }
3910
3911 async fn handle_supervisor_provenance(
3912 &self,
3913 frame: Frame,
3914 module_id: Option<String>,
3915 ) -> Result<Vec<Frame>, RouterError> {
3916 let mut selected = if let Some(module_id) = module_id {
3917 let Some(module) = self.supervisor.get(&module_id) else {
3918 return Ok(vec![control_error_frame(
3919 &frame,
3920 "unknown_module",
3921 format!("module_id '{module_id}' is not supervised"),
3922 )?]);
3923 };
3924 vec![module]
3925 } else {
3926 self.supervisor.list()
3927 };
3928
3929 let mut modules = Vec::with_capacity(selected.len());
3930 for module in selected.drain(..) {
3931 let status = module.status().map_err(|err| {
3932 RouterError::backend(
3933 0,
3934 frame.header.corr,
3935 format!("failed to read supervisor status: {err}"),
3936 )
3937 })?;
3938 let module_declared = self
3939 .registry
3940 .get_module(&status.module_id)
3941 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3942 .and_then(|registration| registration.manifest.provenance)
3943 .map(|build| ModuleDeclaredProvenance::Reported { build })
3944 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
3945 #[cfg(test)]
3946 let running_image = match &self.provenance_probe_override {
3947 Some(result) => result.clone(),
3948 None => module.running_image_agreement().await,
3949 };
3950 #[cfg(not(test))]
3951 let running_image = module.running_image_agreement().await;
3952 modules.push(SupervisorModuleProvenance {
3953 module_id: status.module_id,
3954 module_declared,
3955 daemon_observed: SupervisorObservedProcess {
3956 pid: status.pid,
3957 spawned_at_ms: status.spawned_at_ms,
3958 spawned_from: status.spawned_from,
3959 running_image,
3960 },
3961 });
3962 }
3963 let daemon = SupervisorDaemonProvenance {
3964 daemon_build: self.daemon_provenance.build.clone(),
3965 daemon_observed: DaemonObservedProcess {
3966 pid: self.daemon_provenance.pid,
3967 started_at_ms: self
3968 .daemon_provenance
3969 .start_clock
3970 .map(|clock| clock.started_at_ms())
3971 .or(self.daemon_provenance.started_at_ms),
3972 running_image: self
3973 .daemon_provenance
3974 .probe
3975 .observe(
3976 self.daemon_provenance.pid,
3977 self.daemon_provenance.executable_path.as_deref(),
3978 self.daemon_provenance.executable_identity,
3979 self.daemon_provenance.process_start_time,
3980 )
3981 .await,
3982 },
3983 };
3984 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
3985 Ok(vec![control_response_body_frame(
3986 &frame,
3987 &response,
3988 "ClientControlResponse::SupervisorProvenance",
3989 )?])
3990 }
3991
3992 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3993 self.refresh_capability_requirements();
3994 let generation = self
3995 .registry
3996 .generation()
3997 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3998 let modules = self
3999 .supervisor
4000 .list()
4001 .into_iter()
4002 .map(|module| {
4003 let status = module.status_for_control("health").map_err(|err| {
4004 RouterError::backend(
4005 0,
4006 frame.header.corr,
4007 format!("failed to read supervisor health: {err}"),
4008 )
4009 })?;
4010 let module_id = status.module_id;
4011 let capability_detail = self
4012 .capability_evaluator
4013 .required_problem_detail(&module_id);
4014 Ok(SupervisorHealthEntry {
4015 module_id,
4016 status: status.health.status,
4017 detail: append_capability_problem_detail(
4018 status.health.detail,
4019 capability_detail,
4020 ),
4021 metrics: status.health.metrics,
4022 consecutive_failures: status.health.consecutive_failures,
4023 late_answer_count: status.health.late_answer_count,
4024 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
4025 last_action: status.health.last_action,
4026 last_action_ms: status.health.last_action_ms,
4027 last_probe_ms: status.health.last_probe_ms,
4028 })
4029 })
4030 .collect::<Result<Vec<_>, RouterError>>()?;
4031 let response = ClientControlResponse::SupervisorHealth {
4032 generation,
4033 modules,
4034 };
4035 Ok(vec![control_response_body_frame(
4036 &frame,
4037 &response,
4038 "ClientControlResponse::SupervisorHealth",
4039 )?])
4040 }
4041
4042 async fn handle_supervisor_restart(
4043 &self,
4044 frame: Frame,
4045 module_id: String,
4046 drain_timeout_ms: Option<u64>,
4047 ) -> Result<Vec<Frame>, RouterError> {
4048 let operation_lock = self.supervisor.operation_lock();
4049 let _operation_guard = operation_lock.lock().await;
4050 let Some(module) = self.supervisor.get(&module_id) else {
4051 return Ok(vec![control_error_frame(
4052 &frame,
4053 "unknown_module",
4054 format!("module_id '{module_id}' is not supervised"),
4055 )?]);
4056 };
4057
4058 self.route_outages.mark_operator_action(&module_id);
4059 if let Err(err) = module.restart(drain_timeout_ms).await {
4060 self.route_outages
4061 .operator_action_ended_unrefused(&module_id);
4062 let (code, message) = match err {
4063 crate::supervise::SuperviseError::Disabled { .. } => {
4064 ("module_disabled", err.to_string())
4065 }
4066 crate::supervise::SuperviseError::SwapInProgress { .. } => {
4067 ("swap_in_progress", err.to_string())
4068 }
4069 _ => (
4070 "target_unavailable",
4071 format!("failed to restart module_id '{module_id}': {err}"),
4072 ),
4073 };
4074 return Ok(vec![control_error_frame(&frame, code, message)?]);
4075 }
4076
4077 let response = ClientControlResponse::SupervisorAck {
4078 module_id,
4079 applied: true,
4080 };
4081 Ok(vec![control_response_body_frame(
4082 &frame,
4083 &response,
4084 "ClientControlResponse::SupervisorAck",
4085 )?])
4086 }
4087
4088 async fn handle_supervisor_swap(
4092 &self,
4093 frame: Frame,
4094 module_id: String,
4095 ready_timeout_ms: Option<u64>,
4096 ) -> Result<Vec<Frame>, RouterError> {
4097 let module = {
4104 let operation_lock = self.supervisor.operation_lock();
4105 let _operation_guard = operation_lock.lock().await;
4106 self.supervisor.get(&module_id)
4107 };
4108 let Some(module) = module else {
4109 return Ok(vec![control_error_frame(
4110 &frame,
4111 "unknown_module",
4112 format!("module_id '{module_id}' is not supervised"),
4113 )?]);
4114 };
4115
4116 self.route_outages.mark_operator_action(&module_id);
4117 if let Err(err) = module
4118 .swap(ready_timeout_ms.map(Duration::from_millis))
4119 .await
4120 {
4121 self.route_outages
4122 .operator_action_ended_unrefused(&module_id);
4123 use crate::supervise::SuperviseError;
4124 let message = err.to_string();
4125 let error = match err {
4126 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
4127 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
4128 code: "swap_refused".to_string(),
4129 message,
4130 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
4131 },
4132 SuperviseError::SwapFailed {
4133 arm,
4134 candidate_exit,
4135 ..
4136 } => ErrorBody {
4137 code: "swap_failed".to_string(),
4138 message,
4139 detail: Some(serde_json::json!({
4140 "arm": arm.as_str(),
4141 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
4142 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
4143 })),
4144 },
4145 _ => ErrorBody::new(
4146 "target_unavailable",
4147 format!("failed to swap module_id '{module_id}': {message}"),
4148 ),
4149 };
4150 return Ok(vec![control_error_body_frame(&frame, error)?]);
4151 }
4152 self.route_outages
4156 .operator_action_ended_unrefused(&module_id);
4157
4158 let response = ClientControlResponse::SupervisorAck {
4159 module_id,
4160 applied: true,
4161 };
4162 Ok(vec![control_response_body_frame(
4163 &frame,
4164 &response,
4165 "ClientControlResponse::SupervisorAck",
4166 )?])
4167 }
4168
4169 async fn handle_supervisor_reload(
4170 &self,
4171 frame: Frame,
4172 module_id: String,
4173 ) -> Result<Vec<Frame>, RouterError> {
4174 let operation_lock = self.supervisor.operation_lock();
4175 let _operation_guard = operation_lock.lock().await;
4176 let Some(module) = self.supervisor.get(&module_id) else {
4177 return Ok(vec![control_error_frame(
4178 &frame,
4179 "unknown_module",
4180 format!("module_id '{module_id}' is not supervised"),
4181 )?]);
4182 };
4183
4184 self.route_outages.mark_operator_action(&module_id);
4185 if let Err(err) = module.reload().await {
4186 self.route_outages
4187 .operator_action_ended_unrefused(&module_id);
4188 let (code, message) = match err {
4189 crate::supervise::SuperviseError::Disabled { .. } => {
4190 ("module_disabled", err.to_string())
4191 }
4192 crate::supervise::SuperviseError::SwapInProgress { .. } => {
4193 ("swap_in_progress", err.to_string())
4194 }
4195 _ => (
4196 "reload_failed",
4197 format!("failed to reload module_id '{module_id}': {err}"),
4198 ),
4199 };
4200 return Ok(vec![control_error_frame(&frame, code, message)?]);
4201 }
4202
4203 let response = ClientControlResponse::SupervisorAck {
4204 module_id,
4205 applied: true,
4206 };
4207 Ok(vec![control_response_body_frame(
4208 &frame,
4209 &response,
4210 "ClientControlResponse::SupervisorAck",
4211 )?])
4212 }
4213
4214 async fn handle_supervisor_rescan(
4215 &self,
4216 frame: Frame,
4217 preview: bool,
4218 ) -> Result<Vec<Frame>, RouterError> {
4219 let Some(context) = self.rescan.clone() else {
4220 return Ok(vec![control_error_frame(
4221 &frame,
4222 "rescan_unavailable",
4223 "the daemon was not started with a reloadable config path".to_string(),
4224 )?]);
4225 };
4226
4227 let operation_lock = self.supervisor.operation_lock();
4228 let _operation_guard = operation_lock.lock().await;
4229 let loaded = match crate::daemon_config::load(&context.config_path) {
4230 Ok(config) => config,
4231 Err(err) => {
4232 return Ok(vec![control_error_frame(
4233 &frame,
4234 "invalid_daemon_config",
4235 format!("supervisor rescan rejected daemon config: {err}"),
4236 )?])
4237 }
4238 };
4239 let Some(config) = loaded else {
4247 return Ok(vec![control_error_frame(
4248 &frame,
4249 "invalid_daemon_config",
4250 format!(
4251 "daemon config not found at {}; refusing to rescan (an absent config would \
4252 retire every supervised module)",
4253 context.config_path.display()
4254 ),
4255 )?]);
4256 };
4257 let (
4258 configured_port,
4259 storage_config,
4260 admission_facts_carrier_module_id,
4261 admission_facts_targets,
4262 scope_authority_owners,
4263 modules,
4264 reserved_capabilities,
4265 ) = (
4266 config.port,
4267 config.storage,
4268 config.admission_facts_carrier_module_id,
4269 config.admission_facts_targets,
4270 config.scope_authority_owners,
4271 config.modules,
4272 config.reserved_capabilities,
4273 );
4274
4275 let mut restart_required = Vec::new();
4284 for section in RestartRequiredSection::ALL {
4285 let changed = match section {
4286 RestartRequiredSection::Port => configured_port != context.configured_port,
4287 RestartRequiredSection::Storage => storage_config != context.storage_config,
4288 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
4289 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
4290 }
4291 RestartRequiredSection::AdmissionFactsTargets => {
4292 admission_facts_targets != context.admission_facts_targets
4293 }
4294 RestartRequiredSection::ScopeAuthorityOwners => {
4295 scope_authority_owners != context.scope_authority_owners
4296 }
4297 };
4298 if changed {
4299 restart_required.push(section.label().to_string());
4300 }
4301 }
4302 if !restart_required.is_empty() {
4303 warn!(
4304 config_path = %context.config_path.display(),
4305 sections = %restart_required.join(", "),
4306 "daemon config changed outside the modules section; restart the daemon to apply those changes"
4307 );
4308 }
4309
4310 for configured in &modules {
4311 if let Err(err) = validate_spec(&configured.module_spec()) {
4312 return Ok(vec![control_error_frame(
4313 &frame,
4314 "invalid_daemon_config",
4315 format!("supervisor rescan rejected daemon config: {err}"),
4316 )?]);
4317 }
4318 }
4319
4320 let configured_capabilities = modules
4321 .iter()
4322 .map(|module| (module.module_id.clone(), module.enabled))
4323 .collect::<Vec<_>>();
4324 let preview_capability_warnings = if preview {
4325 let (_, registrations) = self.runtime_capability_snapshot()?;
4326 let current_modules = self
4327 .supervisor
4328 .list()
4329 .into_iter()
4330 .map(|module| module.module_id().to_string())
4331 .collect::<BTreeSet<_>>();
4332 let resulting_modules = configured_capabilities.clone();
4333 let removed = current_modules
4334 .into_iter()
4335 .filter(|module_id| {
4336 !resulting_modules
4337 .iter()
4338 .any(|(configured_id, _)| configured_id == module_id)
4339 })
4340 .collect::<Vec<_>>();
4341 self.capability_evaluator.preview_removal_warnings(
4342 resulting_modules,
4343 &removed,
4344 ®istrations,
4345 )
4346 } else {
4347 Vec::new()
4348 };
4349 let result = match self
4350 .reconcile_supervised_modules(&context.supervisor, modules, preview)
4351 .await
4352 {
4353 Ok(result) => result,
4354 Err(message) => {
4355 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
4356 }
4357 };
4358 if !preview {
4359 self.capability_evaluator
4360 .configure(configured_capabilities, reserved_capabilities);
4361 self.capability_evaluator.wake_deadline_loop();
4362 self.refresh_capability_requirements();
4363 }
4364 let mut result = result;
4365 result.restart_required = restart_required;
4366 result.capability_warnings = preview_capability_warnings;
4367 let response = ClientControlResponse::SupervisorRescan { result };
4368 Ok(vec![control_response_body_frame(
4369 &frame,
4370 &response,
4371 "ClientControlResponse::SupervisorRescan",
4372 )?])
4373 }
4374
4375 async fn handle_supervisor_release_reserved(
4376 &self,
4377 frame: Frame,
4378 module_id: String,
4379 ) -> Result<Vec<Frame>, RouterError> {
4380 let Some(context) = self.rescan.clone() else {
4381 return Ok(vec![control_error_frame(
4382 &frame,
4383 "release_unavailable",
4384 "reserved-id release requires a daemon started with a reloadable config path",
4385 )?]);
4386 };
4387 let operation_lock = self.supervisor.operation_lock();
4388 let _operation_guard = operation_lock.lock().await;
4389 let loaded = match crate::daemon_config::load(&context.config_path) {
4390 Ok(Some(config)) => config,
4391 Ok(None) => {
4392 return Ok(vec![control_error_frame(
4393 &frame,
4394 "invalid_daemon_config",
4395 format!(
4396 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
4397 context.config_path.display()
4398 ),
4399 )?])
4400 }
4401 Err(err) => {
4402 return Ok(vec![control_error_frame(
4403 &frame,
4404 "invalid_daemon_config",
4405 format!("unable to verify reserved-id release against daemon config: {err}"),
4406 )?])
4407 }
4408 };
4409 if loaded
4410 .modules
4411 .iter()
4412 .any(|configured| configured.module_id == module_id)
4413 {
4414 return Ok(vec![control_error_frame(
4415 &frame,
4416 "reserved_module_configured",
4417 format!(
4418 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
4419 ),
4420 )?]);
4421 }
4422 if !self.supervisor.release_retained_reserved_gate(&module_id) {
4423 return Ok(vec![control_error_frame(
4424 &frame,
4425 "reserved_gate_not_retained",
4426 format!(
4427 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
4428 ),
4429 )?]);
4430 }
4431
4432 let response = ClientControlResponse::SupervisorAck {
4433 module_id,
4434 applied: true,
4435 };
4436 Ok(vec![control_response_body_frame(
4437 &frame,
4438 &response,
4439 "ClientControlResponse::SupervisorAck",
4440 )?])
4441 }
4442
4443 async fn reconcile_supervised_modules(
4452 &self,
4453 supervisor: &Supervisor,
4454 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
4455 preview: bool,
4456 ) -> Result<SupervisorRescanResult, String> {
4457 let mut current = BTreeMap::new();
4458 for module in self.supervisor.list() {
4459 let (spec, health) = module.configuration().map_err(|err| {
4460 format!(
4461 "failed to read configuration for module_id '{}': {err}",
4462 module.module_id()
4463 )
4464 })?;
4465 let enabled = module
4466 .status()
4467 .map_err(|err| {
4468 format!(
4469 "failed to read status for module_id '{}': {err}",
4470 module.module_id()
4471 )
4472 })?
4473 .enabled;
4474 current.insert(
4475 module.module_id().to_string(),
4476 (module, spec, health, enabled),
4477 );
4478 }
4479 let configured = configured_modules
4480 .into_iter()
4481 .map(|module| (module.module_id.clone(), module))
4482 .collect::<BTreeMap<_, _>>();
4483
4484 let added = configured
4485 .keys()
4486 .filter(|module_id| !current.contains_key(*module_id))
4487 .cloned()
4488 .collect::<Vec<_>>();
4489 let removed = current
4490 .keys()
4491 .filter(|module_id| !configured.contains_key(*module_id))
4492 .cloned()
4493 .collect::<Vec<_>>();
4494 let mut changed_pending_reload = Vec::new();
4495 let mut configuration_changes = BTreeSet::new();
4496 let mut enabled_changes = BTreeSet::new();
4497 let mut unchanged = 0_u32;
4498
4499 for (module_id, configured_module) in &configured {
4500 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
4501 else {
4502 continue;
4503 };
4504 let configuration_changed = *current_spec != configured_module.module_spec()
4505 || *current_health != configured_module.health;
4506 let enabled_changed = *current_enabled != configured_module.enabled;
4507 if configuration_changed {
4508 configuration_changes.insert(module_id.clone());
4509 changed_pending_reload.push(module_id.clone());
4510 }
4511 if enabled_changed {
4512 enabled_changes.insert(module_id.clone());
4513 }
4514 if !configuration_changed && !enabled_changed {
4515 unchanged = unchanged.saturating_add(1);
4516 }
4517 }
4518
4519 if preview {
4525 return Ok(SupervisorRescanResult {
4526 added,
4527 removed,
4528 changed_pending_reload,
4529 enabled_changes: enabled_changes.iter().cloned().collect(),
4530 unchanged,
4531 preview: true,
4532 restart_required: Vec::new(),
4536 capability_warnings: Vec::new(),
4537 });
4538 }
4539
4540 for module_id in &removed {
4541 let module = ¤t
4542 .get(module_id)
4543 .expect("removed module came from current supervisor state")
4544 .0;
4545 module.retire().await.map_err(|err| {
4546 format!("failed to retire module_id '{module_id}' during rescan: {err}")
4547 })?;
4548 self.supervisor.record_rescan_removal(module_id);
4582 self.supervisor.retire(module_id);
4583 self.route_outages.forget(module_id);
4584 }
4585
4586 for module_id in configured.keys() {
4587 let Some((module, _, _, _)) = current.get(module_id) else {
4588 continue;
4589 };
4590 let configured_module = configured
4591 .get(module_id)
4592 .expect("configured module id came from configured map");
4593 if configuration_changes.contains(module_id) {
4594 module
4595 .update_configuration(
4596 configured_module.module_spec(),
4597 configured_module.health,
4598 configured_module.drain_timeout_ms,
4599 )
4600 .await
4601 .map_err(|err| {
4602 format!(
4603 "failed to update module_id '{module_id}' configuration during rescan: {err}"
4604 )
4605 })?;
4606 }
4607 if enabled_changes.contains(module_id) {
4608 self.route_outages.mark_operator_action(module_id);
4611 module
4612 .set_enabled(configured_module.enabled)
4613 .await
4614 .map_err(|err| {
4615 self.route_outages.operator_action_ended_unrefused(module_id);
4616 format!(
4617 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
4618 configured_module.enabled
4619 )
4620 })?;
4621 }
4622 }
4623
4624 for module_id in &added {
4625 let configured_module = configured
4626 .get(module_id)
4627 .expect("added module id came from configured map");
4628 supervisor
4629 .supervise_configured_with_health(
4630 configured_module.module_spec(),
4631 configured_module.enabled,
4632 configured_module.health,
4633 configured_module.drain_timeout_ms,
4634 configured_module.restart,
4635 )
4636 .map_err(|err| {
4637 format!("failed to add module_id '{module_id}' during rescan: {err}")
4638 })?;
4639 }
4640
4641 Ok(SupervisorRescanResult {
4642 added,
4643 removed,
4644 changed_pending_reload,
4645 enabled_changes: enabled_changes.iter().cloned().collect(),
4646 unchanged,
4647 preview: false,
4648 restart_required: Vec::new(),
4651 capability_warnings: Vec::new(),
4652 })
4653 }
4654
4655 async fn handle_supervisor_set_enabled(
4656 &self,
4657 frame: Frame,
4658 module_id: String,
4659 enabled: bool,
4660 ) -> Result<Vec<Frame>, RouterError> {
4661 let operation_lock = self.supervisor.operation_lock();
4662 let _operation_guard = operation_lock.lock().await;
4663 let Some(module) = self.supervisor.get(&module_id) else {
4664 return Ok(vec![control_error_frame(
4665 &frame,
4666 "unknown_module",
4667 format!("module_id '{module_id}' is not supervised"),
4668 )?]);
4669 };
4670
4671 self.route_outages.mark_operator_action(&module_id);
4674 let applied = match module.set_enabled(enabled).await {
4675 Ok(applied) => applied,
4676 Err(err) => {
4677 self.route_outages
4678 .operator_action_ended_unrefused(&module_id);
4679 return Ok(vec![control_error_frame(
4680 &frame,
4681 "target_unavailable",
4682 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
4683 )?]);
4684 }
4685 };
4686 if !applied {
4687 self.route_outages
4690 .operator_action_ended_unrefused(&module_id);
4691 }
4692
4693 self.capability_evaluator.wake_deadline_loop();
4694 self.refresh_capability_requirements();
4695 let response = ClientControlResponse::SupervisorAck { module_id, applied };
4696 Ok(vec![control_response_body_frame(
4697 &frame,
4698 &response,
4699 "ClientControlResponse::SupervisorAck",
4700 )?])
4701 }
4702
4703 async fn handle_supervisor_health_probe(
4704 &self,
4705 frame: Frame,
4706 module_id: String,
4707 ) -> Result<Vec<Frame>, RouterError> {
4708 self.refresh_capability_requirements();
4709 let Some(registration) = self
4710 .registry
4711 .get_module(&module_id)
4712 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4713 else {
4714 return Ok(vec![control_error_frame(
4715 &frame,
4716 "unknown_module",
4717 format!("module_id '{module_id}' is not registered"),
4718 )?]);
4719 };
4720
4721 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
4734 {
4735 return Ok(vec![control_error_frame(
4736 &frame,
4737 "health_not_advertised",
4738 format!("module_id '{module_id}' did not advertise health.check"),
4739 )?]);
4740 }
4741
4742 let deadline = Instant::now() + self.health_probe_timeout;
4743 let pending = match self.forwarding.begin_module_control_rpc_for(
4744 &module_id,
4745 MODULE_CONTROL_OP_HEALTH_CHECK,
4746 deadline,
4747 ) {
4748 Ok(pending) => pending,
4749 Err(err) => {
4750 return Ok(vec![control_error_frame(
4751 &frame,
4752 forwarding_error_code(&err),
4753 err.to_string(),
4754 )?])
4755 }
4756 };
4757
4758 let PendingModuleControlRpc {
4759 endpoint,
4760 module_sink,
4761 negotiated_ver,
4762 corr: probe_corr,
4763 receiver,
4764 } = pending;
4765 let mut guard =
4766 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
4767 let probe_body =
4768 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
4769 RouterError::backend(
4770 0,
4771 frame.header.corr,
4772 format!("failed to encode health.check request: {err}"),
4773 )
4774 })?;
4775 let probe_frame = Frame::build_with_version(
4776 negotiated_ver,
4777 FrameType::Request,
4778 control_flags(),
4779 0,
4780 0,
4781 probe_corr,
4782 probe_body,
4783 )
4784 .map_err(RouterError::FrameBuild)?;
4785
4786 if let Err(err) = module_sink.send(probe_frame).await {
4787 return Ok(vec![control_error_frame(
4788 &frame,
4789 "target_unavailable",
4790 err.to_string(),
4791 )?]);
4792 }
4793
4794 match timeout_at(deadline, receiver).await {
4795 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
4796 guard.disarm();
4797 let Some(report) = response.health_report() else {
4798 return Ok(vec![control_error_frame(
4799 &frame,
4800 "invalid_control_body",
4801 "health.check RPC returned a non-health response",
4802 )?]);
4803 };
4804 let HealthReport {
4809 status,
4810 detail,
4811 metrics,
4812 } = report;
4813 let capability_detail = self
4814 .capability_evaluator
4815 .required_problem_detail(&module_id);
4816 let response = ClientControlResponse::SupervisorHealthProbe {
4817 module_id,
4818 status,
4819 detail: append_capability_problem_detail(detail, capability_detail),
4820 metrics,
4821 };
4822 Ok(vec![control_response_body_frame(
4823 &frame,
4824 &response,
4825 "ClientControlResponse::SupervisorHealthProbe",
4826 )?])
4827 }
4828 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
4829 guard.disarm();
4830 Ok(vec![control_error_body_frame(&frame, body)?])
4831 }
4832 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
4833 guard.disarm();
4834 Ok(vec![control_error_frame(
4835 &frame,
4836 "target_unavailable",
4837 message,
4838 )?])
4839 }
4840 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
4841 guard.disarm();
4842 Ok(vec![control_error_frame(
4843 &frame,
4844 "invalid_control_body",
4845 message,
4846 )?])
4847 }
4848 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
4849 guard.disarm();
4850 Ok(vec![control_error_frame(
4851 &frame,
4852 "invalid_control_body",
4853 format!("expected module-control op '{expected}', got '{actual}'"),
4854 )?])
4855 }
4856 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
4857 guard.disarm();
4858 Ok(vec![control_error_frame(
4859 &frame,
4860 "module_timeout",
4861 format!(
4862 "module_id '{module_id}' answered health.check after {:?}",
4863 self.health_probe_timeout
4864 ),
4865 )?])
4866 }
4867 Ok(Err(_)) => Ok(vec![control_error_frame(
4868 &frame,
4869 "target_unavailable",
4870 "health.check waiter was canceled before the module responded",
4871 )?]),
4872 Err(_) => Ok(vec![control_error_frame(
4873 &frame,
4874 "module_timeout",
4875 format!(
4876 "module_id '{module_id}' did not answer health.check within {:?}",
4877 self.health_probe_timeout
4878 ),
4879 )?]),
4880 }
4881 }
4882
4883 fn supervisor_status(
4884 &self,
4885 module_id: &str,
4886 corr: u64,
4887 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
4888 self.supervisor
4889 .get(module_id)
4890 .map(|module| {
4891 let warming = module.is_warming_for_control("status").map_err(|err| {
4892 RouterError::backend(
4893 0,
4894 corr,
4895 format!(
4896 "failed to read supervisor warming state for module_id '{module_id}': {err}"
4897 ),
4898 )
4899 })?;
4900 module.status_for_control("status").map_err(|err| {
4901 RouterError::backend(
4902 0,
4903 corr,
4904 format!(
4905 "failed to read supervisor status for module_id '{module_id}': {err}"
4906 ),
4907 )
4908 }).map(|status| (status, warming))
4909 })
4910 .transpose()
4911 }
4912
4913 fn guard_module_control_op(
4914 &self,
4915 frame: &Frame,
4916 module_id: &str,
4917 op: &str,
4918 ) -> Result<Option<Frame>, RouterError> {
4919 if self.module_grants_op(module_id, op, frame.header.corr)? {
4920 return Ok(None);
4921 }
4922
4923 Ok(Some(control_error_frame(
4924 frame,
4925 "op_not_allowed",
4926 format!("module_id '{module_id}' did not grant control op '{op}'"),
4927 )?))
4928 }
4929
4930 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
4931 let Some(registration) = self
4932 .registry
4933 .get_module(module_id)
4934 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
4935 else {
4936 return Ok(false);
4937 };
4938 Ok(module_registration_grants_op(®istration.control_ops, op))
4939 }
4940
4941 fn handle_status_update(
4942 &self,
4943 endpoint: ModuleEndpointId,
4944 frame: Frame,
4945 ) -> Result<Vec<Frame>, RouterError> {
4946 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
4947 Ok(update) => update,
4948 Err(err) => {
4949 if is_known_module_push_op(&frame.body) {
4954 return Ok(vec![control_error_frame(
4955 &frame,
4956 "invalid_control_body",
4957 format!("malformed module control push body: {err}"),
4958 )?]);
4959 }
4960 return Ok(Vec::new());
4961 }
4962 };
4963
4964 match update {
4965 ModuleControlPush::RouteStatus {
4966 route_channel,
4967 route_epoch,
4968 status,
4969 } => {
4970 self.forwarding
4971 .cache_status(endpoint, route_channel, route_epoch, status)
4972 .map_err(RouterError::Forwarding)?;
4973 }
4974 }
4975 Ok(Vec::new())
4976 }
4977
4978 fn handle_route_poll(
4979 &self,
4980 ctx: &RouteCtx,
4981 frame: Frame,
4982 route_channel: u16,
4983 route_epoch: u32,
4984 kind: PollKind,
4985 ) -> Result<Vec<Frame>, RouterError> {
4986 let snapshot = self
4987 .forwarding
4988 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
4989 .map_err(RouterError::Forwarding)?;
4990 let response = match (kind, snapshot) {
4991 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
4992 ClientControlResponse::RoutePoll {
4993 route_channel,
4994 route_epoch,
4995 status,
4996 live: None,
4997 }
4998 }
4999 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
5000 route_channel,
5001 route_epoch,
5002 status: None,
5003 live: None,
5004 },
5005 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
5006 let live = self
5032 .process_liveness
5033 .as_ref()
5034 .and_then(|source| source.process_live(&module_id))
5035 .unwrap_or(true);
5036 ClientControlResponse::RoutePoll {
5037 route_channel,
5038 route_epoch,
5039 status: None,
5040 live: Some(live),
5041 }
5042 }
5043 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
5044 route_channel,
5045 route_epoch,
5046 status: None,
5047 live: Some(false),
5048 },
5049 };
5050
5051 Ok(vec![control_response_body_frame(
5052 &frame,
5053 &response,
5054 "ClientControlResponse::RoutePoll",
5055 )?])
5056 }
5057
5058 pub(crate) fn observe_module_control_completion(
5059 &self,
5060 completion: ModuleControlRpcCompletion,
5061 ) -> bool {
5062 match completion {
5063 ModuleControlRpcCompletion::Unknown => false,
5064 ModuleControlRpcCompletion::Settled => true,
5065 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
5066 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
5067 info!(
5068 module_id = %module_id,
5069 latency_ms,
5070 "late health.check answer proves the module is alive"
5071 );
5072 match self
5073 .supervisor
5074 .record_late_health_answer(&module_id, latency_ms)
5075 {
5076 Ok(true) => {}
5077 Ok(false) => debug!(
5078 module_id = %module_id,
5079 latency_ms,
5080 "late health.check answer has no active supervisor snapshot"
5081 ),
5082 Err(err) => warn!(
5083 module_id = %module_id,
5084 latency_ms,
5085 error = %err,
5086 "failed to record late health.check answer"
5087 ),
5088 }
5089 true
5090 }
5091 }
5092 }
5093
5094 fn refuse_to_end_module_connection_for_a_client(
5115 &self,
5116 module_connection_id: ConnectionId,
5117 corr: u64,
5118 err: ForwardingError,
5119 ) -> Result<(), RouterError> {
5120 if let ForwardingError::ConnectionClosing { connection_id } = err {
5121 if connection_id != module_connection_id {
5122 warn!(
5123 module_connection_id = module_connection_id.get(),
5124 client_connection_id = connection_id.get(),
5125 corr,
5126 "dropping a route.bind response for a closing client; the module connection keeps serving"
5127 );
5128 return Ok(());
5129 }
5130 }
5131 Err(RouterError::Forwarding(err))
5132 }
5133
5134 fn handle_module_relay_response(
5135 &self,
5136 connection_id: ConnectionId,
5137 frame: Frame,
5138 ) -> Result<Vec<Frame>, RouterError> {
5139 let mut secondary_error = None;
5140 let outcome = match frame.header.ty {
5141 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
5142 Ok(probe) if probe.op == "route.bind" => {
5143 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
5144 Ok(ModuleControlResponse::RouteBindAck {}) => {
5145 RouteBindRelayOutcome::Accepted
5146 }
5147 Ok(other) => {
5148 let message =
5149 format!("route.bind response carried unexpected body: {other:?}");
5150 secondary_error = Some(control_error_frame(
5151 &frame,
5152 "invalid_control_body",
5153 message.clone(),
5154 )?);
5155 RouteBindRelayOutcome::ModuleGone(message)
5156 }
5157 Err(err) => {
5158 let message = format!("malformed route.bind response body: {err}");
5159 secondary_error = Some(control_error_frame(
5160 &frame,
5161 "invalid_control_body",
5162 message.clone(),
5163 )?);
5164 RouteBindRelayOutcome::ModuleGone(message)
5165 }
5166 }
5167 }
5168 Ok(probe) => {
5169 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
5170 {
5171 Ok(response) => ModuleControlRpcOutcome::Response(response),
5172 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
5173 "malformed {} response body: {err}",
5174 probe.op
5175 )),
5176 };
5177 let completion = self
5178 .forwarding
5179 .complete_module_control_rpc(
5180 connection_id,
5181 frame.header.corr,
5182 Some(&probe.op),
5183 outcome,
5184 )
5185 .map_err(RouterError::Forwarding)?;
5186 if !self.observe_module_control_completion(completion) {
5187 debug!(
5188 connection_id = connection_id.get(),
5189 corr = frame.header.corr,
5190 op = %probe.op,
5191 "dropping late or unknown module-control RPC response"
5192 );
5193 }
5194 return Ok(Vec::new());
5195 }
5196 Err(err) => {
5197 if let Some(expected_op) = self
5198 .forwarding
5199 .pending_module_control_op(connection_id, frame.header.corr)
5200 .map_err(RouterError::Forwarding)?
5201 {
5202 let completion = self
5203 .forwarding
5204 .complete_module_control_rpc(
5205 connection_id,
5206 frame.header.corr,
5207 None,
5208 ModuleControlRpcOutcome::MalformedResponse(format!(
5209 "malformed {expected_op} response body: {err}"
5210 )),
5211 )
5212 .map_err(RouterError::Forwarding)?;
5213 if !self.observe_module_control_completion(completion) {
5214 debug!(
5215 connection_id = connection_id.get(),
5216 corr = frame.header.corr,
5217 "dropping late malformed module-control RPC response"
5218 );
5219 }
5220 return Ok(Vec::new());
5221 }
5222 let message = format!("malformed route.bind response body: {err}");
5223 secondary_error = Some(control_error_frame(
5224 &frame,
5225 "invalid_control_body",
5226 message.clone(),
5227 )?);
5228 RouteBindRelayOutcome::ModuleGone(message)
5229 }
5230 },
5231 FrameType::Error => {
5232 if self
5233 .forwarding
5234 .pending_module_control_op(connection_id, frame.header.corr)
5235 .map_err(RouterError::Forwarding)?
5236 .is_some()
5237 {
5238 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
5239 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
5240 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
5241 "malformed module-control ERROR body: {err}"
5242 )),
5243 };
5244 let completion = self
5245 .forwarding
5246 .complete_module_control_rpc(
5247 connection_id,
5248 frame.header.corr,
5249 None,
5250 outcome,
5251 )
5252 .map_err(RouterError::Forwarding)?;
5253 if !self.observe_module_control_completion(completion) {
5254 debug!(
5255 connection_id = connection_id.get(),
5256 corr = frame.header.corr,
5257 "dropping late or unknown module-control RPC error"
5258 );
5259 }
5260 return Ok(Vec::new());
5261 }
5262 match serde_json::from_slice::<ErrorBody>(&frame.body) {
5263 Ok(body) => RouteBindRelayOutcome::Rejected(body),
5264 Err(err) => {
5265 let message = format!("malformed route.bind ERROR body: {err}");
5266 secondary_error = Some(control_error_frame(
5267 &frame,
5268 "invalid_control_body",
5269 message.clone(),
5270 )?);
5271 RouteBindRelayOutcome::ModuleGone(message)
5272 }
5273 }
5274 }
5275 ty => {
5276 return Ok(vec![control_error_frame(
5277 &frame,
5278 "unsupported_control_frame",
5279 format!("unsupported module channel-0 frame {ty:?}"),
5280 )?])
5281 }
5282 };
5283
5284 let settled =
5285 self.forwarding
5286 .complete_pending_relay(connection_id, frame.header.corr, outcome);
5287 let completion = match settled {
5288 Ok(completion) => completion,
5289 Err(err) => {
5290 self.refuse_to_end_module_connection_for_a_client(
5291 connection_id,
5292 frame.header.corr,
5293 err,
5294 )?;
5295 return Ok(secondary_error.into_iter().collect());
5296 }
5297 };
5298 if let Some(target) = completion.abandoned.as_ref() {
5299 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
5300 }
5301 if !completion.settled {
5302 debug!(
5303 connection_id = connection_id.get(),
5304 corr = frame.header.corr,
5305 frame_type = ?frame.header.ty,
5306 "dropping late or unknown route.bind relay response"
5307 );
5308 }
5309 Ok(secondary_error.into_iter().collect())
5310 }
5311
5312 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
5313 debug!(connection_id = connection_id.get(), "handling GOODBYE");
5314 let registrations = self
5315 .deregister_connection(connection_id)
5316 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
5317 let released_routes = self
5318 .forwarding
5319 .cleanup_connection(connection_id)
5320 .map_err(RouterError::Forwarding)?;
5321 self.emit_route_goodbyes(released_routes);
5322 if !registrations.is_empty() {
5324 crate::supervise::notify_registration_release();
5325 }
5326 Ok(Vec::new())
5327 }
5328}
5329
5330impl Default for ControlHandler {
5331 fn default() -> Self {
5332 Self::new(Arc::new(Registry::default()))
5333 }
5334}
5335
5336impl crate::supervise::SwapPromotionObserver for ControlHandler {
5337 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
5338 self.apply_registration_capabilities(registration);
5339 }
5340}
5341
5342fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
5343 CapabilityRequirementStatus {
5344 consumer: status.consumer,
5345 capability: status.capability,
5346 need: match status.need {
5347 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
5348 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
5349 },
5350 verdict: status.verdict.as_str().to_string(),
5351 episode_seq: status.episode_seq,
5352 config_satisfiable: status.config_satisfiable,
5353 runtime_available: status.runtime_available,
5354 detail: status.detail,
5355 }
5356}
5357
5358fn append_capability_problem_detail(
5359 detail: Option<String>,
5360 capability_detail: Option<String>,
5361) -> Option<String> {
5362 match (detail, capability_detail) {
5363 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
5364 (Some(detail), None) => Some(detail),
5365 (None, Some(capability_detail)) => Some(capability_detail),
5366 (None, None) => None,
5367 }
5368}
5369
5370fn subc_ops() -> Vec<String> {
5371 SUBC_CONTROL_OPS
5372 .iter()
5373 .map(|op| (*op).to_string())
5374 .collect()
5375}
5376
5377fn module_subc_ops() -> Vec<String> {
5378 SUBC_CONTROL_OPS
5379 .iter()
5380 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
5381 .map(|op| (*op).to_string())
5382 .collect()
5383}
5384
5385#[cfg(test)]
5386fn module_baseline_control_ops() -> Vec<String> {
5387 MODULE_BASELINE_CONTROL_OPS
5388 .iter()
5389 .map(|op| (*op).to_string())
5390 .collect()
5391}
5392
5393fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
5394 let mut seen = HashSet::new();
5395 let mut effective = Vec::new();
5396 for op in MODULE_BASELINE_CONTROL_OPS {
5397 if seen.insert((*op).to_string()) {
5398 effective.push((*op).to_string());
5399 }
5400 }
5401 for op in declared.unwrap_or_default() {
5402 if seen.insert(op.clone()) {
5403 effective.push(op);
5404 }
5405 }
5406 effective
5407}
5408
5409fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
5410 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
5411}
5412
5413fn target_module_id(target: &RouteTarget) -> &str {
5414 match target {
5415 RouteTarget::ToolProvider { module_id }
5416 | RouteTarget::ManagementSurface { module_id }
5417 | RouteTarget::InternalService { module_id, .. } => module_id,
5418 }
5419}
5420
5421fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
5422 roles.iter().any(|role| match (target, role) {
5423 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
5424 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
5425 (
5426 RouteTarget::InternalService { service_id, .. },
5427 ProviderRole::InternalService {
5428 service_id: provided,
5429 ..
5430 },
5431 ) => service_id == provided,
5432 _ => false,
5433 })
5434}
5435
5436fn is_routable_role(role: &ProviderRole) -> bool {
5437 matches!(
5438 role,
5439 ProviderRole::ToolProvider { .. }
5440 | ProviderRole::ManagementSurface { .. }
5441 | ProviderRole::InternalService { .. }
5442 )
5443}
5444
5445#[derive(Debug, Clone, Copy, PartialEq, Eq)]
5446enum ControlRequestBodyError {
5447 UnknownOp,
5448 InvalidBody,
5449}
5450
5451#[derive(Debug, Deserialize)]
5452struct ControlOpProbe {
5453 op: String,
5454}
5455
5456const MODULE_PUSH_OPS: &[&str] = &["route.status"];
5459
5460fn is_known_module_push_op(body: &[u8]) -> bool {
5461 serde_json::from_slice::<ControlOpProbe>(body)
5462 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
5463 .unwrap_or(false)
5464}
5465
5466fn is_known_module_request_op(body: &[u8]) -> bool {
5467 serde_json::from_slice::<ControlOpProbe>(body)
5468 .map(|probe| is_module_to_subc_op(&probe.op))
5469 .unwrap_or(false)
5470}
5471
5472fn is_module_to_subc_op(op: &str) -> bool {
5473 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
5474}
5475
5476fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
5477 debug!(
5478 op = %op,
5479 connection_id = connection_id.get(),
5480 corr,
5481 "control dispatch"
5482 );
5483}
5484
5485fn log_slow_control_dispatch(
5486 dispatch_started_at: Option<StdInstant>,
5487 op: &'static str,
5488 connection_id: ConnectionId,
5489 corr: u64,
5490) {
5491 let Some(dispatch_started_at) = dispatch_started_at else {
5492 return;
5493 };
5494 let elapsed = dispatch_started_at.elapsed();
5495 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
5496 warn!(
5497 op = %op,
5498 connection_id = connection_id.get(),
5499 corr,
5500 elapsed_ms = elapsed.as_millis() as u64,
5501 "slow control dispatch"
5502 );
5503 }
5504}
5505
5506fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
5507 match request {
5508 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
5509 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
5510 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
5511 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
5512 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
5513 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
5514 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
5515 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
5516 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
5517 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
5518 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
5519 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
5520 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
5521 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
5522 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
5523 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
5524 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
5525 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
5526 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
5527 }
5528}
5529
5530fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
5531 match request {
5532 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
5533 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
5534 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
5535 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
5536 }
5537}
5538
5539fn parse_client_control_request(
5540 body: &[u8],
5541) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
5542 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
5543 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5544 Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
5545 ControlRequestBodyError::InvalidBody
5546 }
5547 Ok(_) => ControlRequestBodyError::UnknownOp,
5548 Err(_) => ControlRequestBodyError::InvalidBody,
5549 };
5550 (err, classification)
5551 })
5552}
5553
5554fn parse_module_control_request_from_module(
5555 body: &[u8],
5556) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
5557 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
5558 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5559 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
5560 Ok(_) => ControlRequestBodyError::UnknownOp,
5561 Err(_) => ControlRequestBodyError::InvalidBody,
5562 };
5563 (err, classification)
5564 })
5565}
5566
5567#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
5568enum ProviderRoleKind {
5569 ToolProvider,
5570 PipelineStage,
5571 ManagementSurface,
5572 InternalService,
5573}
5574
5575fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
5576 match role {
5577 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
5578 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
5579 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
5580 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
5581 }
5582}
5583
5584fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
5585 roles.iter().map(provider_role_kind).collect()
5586}
5587
5588const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
5591
5592#[derive(Debug, Default, PartialEq, Eq)]
5594struct ScopeOutcomeCounts {
5595 created: usize,
5596 replaced: usize,
5597 updated: usize,
5598 unchanged: usize,
5599 refused: usize,
5600}
5601
5602impl ScopeOutcomeCounts {
5603 fn of(results: &[ScopeRecordResult]) -> Self {
5604 let mut counts = Self::default();
5605 for result in results {
5606 let slot = match result.outcome {
5607 ScopeRecordOutcome::Created => &mut counts.created,
5608 ScopeRecordOutcome::Replaced => &mut counts.replaced,
5609 ScopeRecordOutcome::Updated => &mut counts.updated,
5610 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
5611 ScopeRecordOutcome::Refused => &mut counts.refused,
5612 };
5613 *slot += 1;
5614 }
5615 counts
5616 }
5617}
5618
5619#[cfg(test)]
5620mod scope_outcome_count_tests {
5621 use super::*;
5622
5623 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
5624 ScopeRecordResult {
5625 scope_ref: "r".to_string(),
5626 scope_epoch: 1,
5627 outcome,
5628 code: None,
5629 message: None,
5630 version: None,
5631 parent_state: None,
5632 }
5633 }
5634
5635 #[test]
5638 fn every_outcome_is_counted_in_its_own_field() {
5639 let results = [
5640 result(ScopeRecordOutcome::Created),
5641 result(ScopeRecordOutcome::Created),
5642 result(ScopeRecordOutcome::Replaced),
5643 result(ScopeRecordOutcome::Updated),
5644 result(ScopeRecordOutcome::Unchanged),
5645 result(ScopeRecordOutcome::Refused),
5646 result(ScopeRecordOutcome::Refused),
5647 result(ScopeRecordOutcome::Refused),
5648 ];
5649 assert_eq!(
5650 ScopeOutcomeCounts::of(&results),
5651 ScopeOutcomeCounts {
5652 created: 2,
5653 replaced: 1,
5654 updated: 1,
5655 unchanged: 1,
5656 refused: 3,
5657 }
5658 );
5659 }
5660}
5661
5662fn capability_census_trigger(
5666 old: Option<&CapabilityDeclarations>,
5667 new: Option<&CapabilityDeclarations>,
5668) -> bool {
5669 let old_provides = old
5670 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
5671 .unwrap_or_default();
5672 let old_denies = old
5673 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
5674 .unwrap_or_default();
5675 let new = new.cloned().unwrap_or(CapabilityDeclarations {
5676 provides: Vec::new(),
5677 requires: Vec::new(),
5678 must_never_reach: Vec::new(),
5679 });
5680
5681 new.provides
5682 .iter()
5683 .any(|capability| !old_provides.contains(capability))
5684 || new
5685 .must_never_reach
5686 .iter()
5687 .any(|capability| !old_denies.contains(capability))
5688}
5689
5690fn denied_capability<'a>(
5693 opening_manifest: &'a ModuleManifest,
5694 target_manifest: &ModuleManifest,
5695) -> Option<&'a str> {
5696 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
5697 let target_capabilities = target_manifest.capabilities.as_ref()?;
5698 opening_capabilities
5699 .must_never_reach
5700 .iter()
5701 .find(|denied| {
5702 target_capabilities
5703 .provides
5704 .iter()
5705 .any(|provided| provided == *denied)
5706 })
5707 .map(String::as_str)
5708}
5709
5710fn catalog_update_frozen_field_message(
5711 registered: &ModuleManifest,
5712 provides: &[ProviderRole],
5713) -> Option<String> {
5714 let old_has_provides = !registered.provides.is_empty();
5715 let new_has_provides = !provides.is_empty();
5716 if old_has_provides != new_has_provides {
5717 return Some(format!(
5718 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
5719 registered.module_id
5720 ));
5721 }
5722
5723 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
5724 return Some(format!(
5725 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
5726 registered.module_id
5727 ));
5728 }
5729
5730 let registered_concurrency = manifest_concurrency(registered);
5731 let mut candidate = registered.clone();
5732 candidate.provides = provides.to_vec();
5733 let candidate_concurrency = manifest_concurrency(&candidate);
5734 if candidate_concurrency != registered_concurrency {
5735 return Some(format!(
5736 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
5737 registered.module_id, registered_concurrency, candidate_concurrency
5738 ));
5739 }
5740
5741 None
5744}
5745
5746fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
5747 manifest.provides.iter().any(is_routable_role)
5748}
5749
5750fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
5756 manifest
5757 .provides
5758 .iter()
5759 .find_map(|provider| match provider {
5760 ProviderRole::ToolProvider { concurrency, .. }
5761 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
5762 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
5763 })
5764 .unwrap_or(Concurrency::ModuleManaged)
5765}
5766
5767fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
5774 let has_management_surface = manifest
5775 .provides
5776 .iter()
5777 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
5778 if !has_management_surface {
5779 return false;
5780 }
5781 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
5782 return false;
5783 };
5784 let Some(provides) = raw
5785 .get("manifest")
5786 .and_then(|manifest| manifest.get("provides"))
5787 .and_then(serde_json::Value::as_array)
5788 else {
5789 return false;
5790 };
5791 provides.iter().any(|role| {
5796 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
5797 && role.get("concurrency").is_none()
5798 })
5799}
5800
5801fn negotiate_version(peer_version: u8) -> Result<u8, String> {
5802 if peer_version != PROTOCOL_VERSION {
5803 return Err(format!(
5804 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
5805 ));
5806 }
5807 Ok(PROTOCOL_VERSION)
5808}
5809
5810fn pong(frame: &Frame) -> Result<Frame, RouterError> {
5811 Frame::build_with_version(
5812 response_version(frame),
5813 FrameType::Pong,
5814 frame.header.flags,
5815 0,
5816 0,
5817 frame.header.corr,
5818 Vec::new(),
5819 )
5820 .map_err(RouterError::FrameBuild)
5821}
5822
5823fn control_error_frame(
5824 frame: &Frame,
5825 code: &'static str,
5826 message: impl Into<String>,
5827) -> Result<Frame, RouterError> {
5828 control_error_body_frame(
5829 frame,
5830 ErrorBody {
5831 code: code.to_string(),
5832 message: message.into(),
5833 detail: None,
5834 },
5835 )
5836}
5837
5838fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
5839 let body = serde_json::to_vec(&error).map_err(|err| {
5840 RouterError::backend(
5841 0,
5842 frame.header.corr,
5843 format!("failed to encode control ERROR: {err}"),
5844 )
5845 })?;
5846
5847 Frame::build_with_version(
5848 response_version(frame),
5849 FrameType::Error,
5850 control_flags(),
5851 0,
5852 0,
5853 frame.header.corr,
5854 body,
5855 )
5856 .map_err(RouterError::FrameBuild)
5857}
5858
5859fn control_response_body_frame<T: Serialize>(
5860 frame: &Frame,
5861 reply: &T,
5862 label: &'static str,
5863) -> Result<Frame, RouterError> {
5864 let body = serde_json::to_vec(reply).map_err(|err| {
5865 RouterError::backend(
5866 0,
5867 frame.header.corr,
5868 format!("failed to encode {label}: {err}"),
5869 )
5870 })?;
5871
5872 Frame::build_with_version(
5873 response_version(frame),
5874 FrameType::Response,
5875 control_flags(),
5876 0,
5877 0,
5878 frame.header.corr,
5879 body,
5880 )
5881 .map_err(RouterError::FrameBuild)
5882}
5883
5884fn forwarding_error_code(err: &ForwardingError) -> &'static str {
5914 match err {
5915 ForwardingError::NoModuleConnection => "target_unavailable",
5916 ForwardingError::ModuleReloading { .. } => "module_reloading",
5917 ForwardingError::ClientRouteChannelExhausted { .. }
5918 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
5919 ForwardingError::StaleModuleEndpoint
5920 | ForwardingError::UnknownReservation { .. }
5921 | ForwardingError::ConnectionClosing { .. }
5922 | ForwardingError::ClientEgressClosed { .. }
5923 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
5924 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
5927 ForwardingError::RelayCorrelationExhausted
5928 | ForwardingError::RouteOpenBuild(_)
5929 | ForwardingError::Poisoned => "forwarding_error",
5930 }
5931}
5932
5933fn response_version(frame: &Frame) -> u8 {
5934 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
5935 frame.header.ver
5936 } else {
5937 PROTOCOL_VERSION
5938 }
5939}
5940
5941fn control_flags() -> Flags {
5942 Flags::new(false, Priority::Passive, false)
5943}
5944
5945fn send_goodbye_target_best_effort(
5950 counters: &DaemonCounters,
5951 target: &GoodbyeTarget,
5952 context: &'static str,
5953) {
5954 let Ok(frame) = Frame::build_with_version(
5955 target.negotiated_ver,
5956 FrameType::Goodbye,
5957 control_flags(),
5958 target.channel,
5959 target.epoch,
5960 0,
5961 Vec::new(),
5962 ) else {
5963 return;
5964 };
5965 crate::forwarding::send_module_route_goodbye(
5966 counters,
5967 &target.sink,
5968 frame,
5969 target.module_id.as_deref(),
5970 context,
5971 );
5972}
5973
5974pub(crate) fn send_route_control_pushes(
5975 forwarding: &ForwardingTable,
5976 routes: Vec<EndpointRoute>,
5977 push: ClientControlPush,
5978) {
5979 let body = match serde_json::to_vec(&push) {
5980 Ok(body) => body,
5981 Err(err) => {
5982 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
5983 return;
5984 }
5985 };
5986 let mut targets = Vec::new();
5987 for route in routes {
5988 let target = route.goodbye_target;
5989 if let Some(existing) = targets
5990 .iter()
5991 .find(|existing: &&GoodbyeTarget| existing.connection_id == target.connection_id)
5992 {
5993 debug_assert_eq!(
5994 existing.negotiated_ver, target.negotiated_ver,
5995 "one connection cannot negotiate multiple frame versions"
5996 );
5997 continue;
5998 }
5999 targets.push(target);
6000 }
6001 for target in targets {
6002 let frame = match Frame::build_with_version(
6003 target.negotiated_ver,
6004 FrameType::Push,
6005 control_flags(),
6006 0,
6007 0,
6008 0,
6009 body.clone(),
6010 ) {
6011 Ok(frame) => frame,
6012 Err(err) => {
6013 warn!(
6014 route_channel = target.channel,
6015 error = %err,
6016 "failed to build route lifecycle control PUSH frame"
6017 );
6018 continue;
6019 }
6020 };
6021 if let Err(err) = target.sink.try_send(frame) {
6022 if target.close_on_delivery_failure() {
6023 warn!(
6024 target_connection_id = target.connection_id.get(),
6025 route_channel = target.channel,
6026 error = %err,
6027 "route lifecycle control PUSH was not delivered to client; closing target connection"
6028 );
6029 let _ = forwarding.escalate_client_delivery_failure(
6030 target.connection_id,
6031 target.channel,
6032 target.epoch,
6033 CloseReason::new(
6034 "route_lifecycle_push_delivery_failed",
6035 format!(
6036 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
6037 target.channel
6038 ),
6039 ),
6040 crate::forwarding::UndeliveredFrame {
6041 module_id: target.module_id.as_deref(),
6042 sink: &target.sink,
6043 },
6044 );
6045 }
6046 }
6047 }
6048}
6049
6050#[cfg(test)]
6051mod tests {
6052 use std::{
6053 collections::BTreeMap,
6054 fmt,
6055 path::PathBuf,
6056 sync::{Arc, Mutex},
6057 time::Duration,
6058 };
6059 use subc_test_support::TestTempDir;
6060
6061 use serde_json::{json, Value};
6062 use subc_protocol::{
6063 manifest::{
6064 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
6065 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
6066 },
6067 session::HealthStatus,
6068 FrameType,
6069 };
6070
6071 use super::*;
6072 use crate::{
6073 forwarding::{DataRoute, DataRouteState},
6074 registry::ChannelState,
6075 router::FrameSink,
6076 stderr_tail::DEFAULT_MAX_LINE_BYTES,
6077 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
6078 RouteCtx, Router,
6079 };
6080 use tokio::{
6081 sync::mpsc,
6082 time::{sleep, Instant},
6083 };
6084 use tracing::{
6085 field::{Field, Visit},
6086 Event, Subscriber,
6087 };
6088 use tracing_subscriber::{layer::Context, prelude::*, Layer};
6089
6090 fn fake_aft_stub_path() -> PathBuf {
6109 let mut path = std::env::current_exe().expect("current_exe available in tests");
6110 path.pop(); path.pop(); path.push(if cfg!(windows) {
6113 "fake-aft-stub.exe"
6114 } else {
6115 "fake-aft-stub"
6116 });
6117 assert!(
6118 path.exists(),
6119 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
6120 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
6121 path.display()
6122 );
6123 path
6124 }
6125
6126 fn client_retries(code: &str) -> bool {
6131 subc_protocol::error_codes::is_retryable_route_open(code)
6132 }
6133
6134 #[test]
6145 fn retryability_of_forwarding_codes_matches_the_failure() {
6146 let transient = [
6149 ForwardingError::NoModuleConnection,
6150 ForwardingError::ModuleReloading {
6151 module_id: "m".into(),
6152 },
6153 ForwardingError::StaleModuleEndpoint,
6154 ForwardingError::UnknownReservation {
6155 client_channel: 1,
6156 module_channel: 1,
6157 },
6158 ForwardingError::ConnectionClosing {
6159 connection_id: ConnectionId::new(1),
6160 },
6161 ForwardingError::ClientEgressClosed {
6162 connection_id: ConnectionId::new(1),
6163 },
6164 ForwardingError::ModuleEgressUnavailable {
6165 connection_id: ConnectionId::new(1),
6166 },
6167 ];
6168 for err in transient {
6169 let code = forwarding_error_code(&err);
6170 assert!(
6171 client_retries(code),
6172 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
6173 );
6174 }
6175
6176 let permanent = [
6181 ForwardingError::ClientRouteChannelExhausted {
6182 connection_id: ConnectionId::new(1),
6183 },
6184 ForwardingError::ModuleRouteChannelExhausted {
6185 endpoint: ModuleEndpointId {
6186 connection_id: ConnectionId::new(1),
6187 generation: 1,
6188 },
6189 },
6190 ForwardingError::RelayCorrelationExhausted,
6191 ForwardingError::RouteOpenBuild("x".into()),
6192 ForwardingError::Poisoned,
6193 ];
6194 for err in permanent {
6195 let code = forwarding_error_code(&err);
6196 assert!(
6197 !client_retries(code),
6198 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
6199 );
6200 }
6201 }
6202
6203 #[tokio::test]
6231 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
6232 let handler = ControlHandler::default();
6233 let frame =
6234 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
6235
6236 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
6240 assert_eq!(
6241 stamped,
6242 Principal::Direct,
6243 "a caller that proved nothing must not be stamped as a supervised module"
6244 );
6245
6246 let forged = handler
6251 .route_open_principal(
6252 &frame,
6253 Some(ConsumerIdentity {
6254 module_id: "aft".to_string(),
6255 launch_nonce: "not-a-real-nonce".to_string(),
6256 }),
6257 )
6258 .unwrap();
6259 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
6260 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
6261 }
6262
6263 #[test]
6280 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
6281 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
6282 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
6283 let ClientControlRequest::RouteOpen {
6284 consumer_identity, ..
6285 } = parsed
6286 else {
6287 panic!("route.open body must parse as RouteOpen");
6288 };
6289 assert_eq!(
6290 consumer_identity,
6291 Some(ConsumerIdentity {
6292 module_id: "aft".to_string(),
6293 launch_nonce: "n".to_string(),
6294 }),
6295 "the wire field name must reach the value route_open_principal reads"
6296 );
6297 }
6298
6299 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
6300 ModuleManifest::builder(module_id, "0.1.0")
6301 .protocol_ver(protocol_ver)
6302 .provides(vec![ProviderRole::ToolProvider {
6303 tools: vec![Tool {
6304 name: "read".to_string(),
6305 description: None,
6306 execution_mode: ExecutionMode::Pure,
6307 schema: json!({"type": "object"}),
6308 }],
6309 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
6310 concurrency: Concurrency::ModuleManaged,
6311 emits_push: true,
6312 sub_supervises: true,
6313 }])
6314 .build()
6315 }
6316
6317 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
6318 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
6319 }
6320
6321 fn hello_frame_with_control_ops(
6322 module_id: &str,
6323 protocol_ver: u8,
6324 corr: u64,
6325 control_ops: Option<Vec<String>>,
6326 ) -> Frame {
6327 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
6328 }
6329
6330 fn hello_frame_with_nonce(
6331 module_id: &str,
6332 protocol_ver: u8,
6333 corr: u64,
6334 launch_nonce: Option<&str>,
6335 ) -> Frame {
6336 hello_frame_full(
6337 module_id,
6338 protocol_ver,
6339 corr,
6340 None,
6341 launch_nonce.map(ToOwned::to_owned),
6342 )
6343 }
6344
6345 fn hello_frame_full(
6346 module_id: &str,
6347 protocol_ver: u8,
6348 corr: u64,
6349 control_ops: Option<Vec<String>>,
6350 launch_nonce: Option<String>,
6351 ) -> Frame {
6352 let body = serde_json::to_vec(&ModuleHelloBody {
6353 manifest: manifest(module_id, protocol_ver),
6354 protocol_ver,
6355 control_ops,
6356 launch_nonce,
6357 })
6358 .unwrap();
6359 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
6360 }
6361
6362 fn non_routable_hello_frame_with_control_ops(
6363 module_id: &str,
6364 corr: u64,
6365 control_ops: Option<Vec<String>>,
6366 ) -> Frame {
6367 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
6368 manifest.provides.clear();
6369 let body = serde_json::to_vec(&ModuleHelloBody {
6370 manifest,
6371 protocol_ver: PROTOCOL_VERSION,
6372 control_ops,
6373 launch_nonce: None,
6374 })
6375 .unwrap();
6376 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
6377 }
6378
6379 fn capability_grammar_hello_frame(
6380 capabilities: Value,
6381 runtime_computed: Option<Value>,
6382 corr: u64,
6383 ) -> Frame {
6384 let mut body = serde_json::to_value(ModuleHelloBody {
6385 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
6386 protocol_ver: PROTOCOL_VERSION,
6387 control_ops: None,
6388 launch_nonce: None,
6389 })
6390 .expect("HELLO body serializes");
6391 body["manifest"]["capabilities"] = capabilities;
6392 if let Some(runtime_computed) = runtime_computed {
6393 body["runtime_computed"] = runtime_computed;
6394 }
6395 Frame::build(
6396 FrameType::Hello,
6397 control_flags(),
6398 0,
6399 0,
6400 corr,
6401 serde_json::to_vec(&body).expect("HELLO body reserializes"),
6402 )
6403 .expect("HELLO frame builds")
6404 }
6405
6406 fn channel_request(channel: u16, corr: u64) -> Frame {
6407 Frame::build(
6408 FrameType::Request,
6409 Flags::new(true, Priority::Interactive, false),
6410 channel,
6411 0,
6412 corr,
6413 b"opaque".to_vec(),
6414 )
6415 .unwrap()
6416 }
6417
6418 fn route_ctx(
6419 connection_id: ConnectionId,
6420 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
6421 let (tx, rx) = mpsc::channel(8);
6422 (
6423 RouteCtx {
6424 connection_id,
6425 egress: FrameSink::new(tx),
6426 },
6427 rx,
6428 )
6429 }
6430
6431 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
6432 serde_json::from_slice(&frame.body).unwrap()
6433 }
6434
6435 async fn hello_via_sink(
6440 handler: &ControlHandler,
6441 ctx: &RouteCtx,
6442 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
6443 hello: Frame,
6444 ) -> Frame {
6445 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
6446 assert!(
6447 replies.is_empty(),
6448 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
6449 );
6450 let ack = rx
6451 .try_recv()
6452 .expect("HELLO_ACK is queued on the module sink")
6453 .frame;
6454 assert_eq!(ack.header.ty, FrameType::HelloAck);
6455 ack
6456 }
6457
6458 fn parse_error(frame: &Frame) -> Value {
6459 serde_json::from_slice(&frame.body).unwrap()
6460 }
6461
6462 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
6463 serde_json::from_slice(&frame.body).unwrap()
6464 }
6465
6466 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
6467 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
6468 route_channel,
6469 route_epoch: 0,
6470 kind,
6471 })
6472 .unwrap();
6473 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6474 }
6475
6476 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
6477 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
6478 module_id: module_id.to_string(),
6479 })
6480 .unwrap();
6481 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6482 }
6483
6484 fn route_open_frame(corr: u64, module_id: &str, project_root: TestTempDir) -> Frame {
6485 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
6486 }
6487
6488 fn route_open_frame_with_consumer_capabilities(
6489 corr: u64,
6490 module_id: &str,
6491 project_root: TestTempDir,
6492 consumer_capabilities: Option<Vec<String>>,
6493 ) -> Frame {
6494 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6495 target: RouteTarget::ToolProvider {
6496 module_id: module_id.to_string(),
6497 },
6498 identity: BindIdentity::new(
6499 project_root.path().to_path_buf(),
6500 "unit".to_string(),
6501 "session".to_string(),
6502 ),
6503 consumer_identity: None,
6504 consumer_capabilities,
6505 role_versions: None,
6506 admission_facts: None,
6507 scope: None,
6508 })
6509 .unwrap();
6510 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6511 }
6512
6513 fn route_open_frame_with_role_versions(
6514 corr: u64,
6515 module_id: &str,
6516 project_root: TestTempDir,
6517 role_versions: Option<BTreeMap<String, String>>,
6518 ) -> Frame {
6519 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6520 target: RouteTarget::ToolProvider {
6521 module_id: module_id.to_string(),
6522 },
6523 identity: BindIdentity::new(
6524 project_root.path().to_path_buf(),
6525 "unit".to_string(),
6526 format!("session-{corr}"),
6527 ),
6528 consumer_identity: None,
6529 consumer_capabilities: None,
6530 role_versions,
6531 admission_facts: None,
6532 scope: None,
6533 })
6534 .unwrap();
6535 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6536 }
6537
6538 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
6539 entries
6540 .iter()
6541 .map(|(role, version)| (role.to_string(), version.to_string()))
6542 .collect()
6543 }
6544
6545 fn route_open_frame_with_admission_facts(
6546 corr: u64,
6547 module_id: &str,
6548 project_root: TestTempDir,
6549 consumer_identity: Option<subc_control::ConsumerIdentity>,
6550 facts: Option<Value>,
6551 ) -> Frame {
6552 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6553 target: RouteTarget::ToolProvider {
6554 module_id: module_id.to_string(),
6555 },
6556 identity: BindIdentity::new(
6557 project_root.path().to_path_buf(),
6558 "unit".to_string(),
6559 format!("session-{corr}"),
6560 ),
6561 consumer_identity,
6562 consumer_capabilities: None,
6563 role_versions: None,
6564 admission_facts: facts,
6565 scope: None,
6566 })
6567 .unwrap();
6568 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6569 }
6570
6571 #[derive(Clone, Default)]
6572 struct EventCapture {
6573 events: Arc<Mutex<Vec<CapturedEvent>>>,
6574 }
6575
6576 #[derive(Clone, Debug)]
6577 struct CapturedEvent {
6578 target: String,
6579 level: tracing::Level,
6580 fields: BTreeMap<String, String>,
6581 }
6582
6583 impl EventCapture {
6584 fn events(&self) -> Vec<CapturedEvent> {
6585 self.events.lock().unwrap().clone()
6586 }
6587 }
6588
6589 impl<S> Layer<S> for EventCapture
6590 where
6591 S: Subscriber,
6592 {
6593 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
6594 let mut visitor = EventFieldVisitor::default();
6595 event.record(&mut visitor);
6596 self.events.lock().unwrap().push(CapturedEvent {
6597 target: event.metadata().target().to_string(),
6598 level: *event.metadata().level(),
6599 fields: visitor.fields,
6600 });
6601 }
6602 }
6603
6604 #[derive(Default)]
6605 struct EventFieldVisitor {
6606 fields: BTreeMap<String, String>,
6607 }
6608
6609 impl Visit for EventFieldVisitor {
6610 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
6611 self.fields
6612 .insert(field.name().to_string(), format!("{value:?}"));
6613 }
6614 }
6615
6616 fn health_response(corr: u64, status: HealthStatus) -> Frame {
6617 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
6618 status,
6619 detail: Some("warming".to_string()),
6620 metrics: Some(json!({"queue_depth": 3})),
6621 })
6622 .unwrap();
6623 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
6624 }
6625
6626 fn route_bind_ack(corr: u64) -> Frame {
6627 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
6628 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
6629 }
6630
6631 fn unique_project_root(label: &str) -> TestTempDir {
6632 TestTempDir::new(label)
6633 }
6634
6635 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
6636 match parse_route_poll(frame) {
6637 ClientControlResponse::RoutePoll {
6638 status: None,
6639 live: Some(live),
6640 ..
6641 } => assert_eq!(live, expected_live),
6642 other => panic!("unexpected route.poll response: {other:?}"),
6643 }
6644 }
6645
6646 fn bind_liveness_route(
6647 registry: &Registry,
6648 forwarding: &ForwardingTable,
6649 module_id: &str,
6650 ) -> (RouteCtx, u16, u32) {
6651 let module_connection = ConnectionId::new(101);
6652 let client_connection = ConnectionId::new(202);
6653 let registration = registry
6654 .register_with_control_ops(
6655 manifest(module_id, PROTOCOL_VERSION),
6656 PROTOCOL_VERSION,
6657 module_connection,
6658 module_baseline_control_ops(),
6659 )
6660 .unwrap();
6661 let (module_tx, _module_rx) = mpsc::channel(8);
6662 let endpoint = forwarding
6663 .register_module_connection(
6664 module_connection,
6665 module_id.to_string(),
6666 PROTOCOL_VERSION,
6667 manifest_concurrency(®istration.manifest),
6668 FrameSink::new(module_tx),
6669 )
6670 .unwrap();
6671 let (client_ctx, _client_rx) = route_ctx(client_connection);
6672 let pending = forwarding
6673 .begin_route_bind_relay_for_test(
6674 client_connection,
6675 client_ctx.egress.clone(),
6676 1,
6677 module_id,
6678 )
6679 .unwrap();
6680 assert_eq!(pending.endpoint, endpoint);
6681 let route_channel = pending.client_channel;
6682 let route_epoch = pending.client_epoch;
6683 forwarding
6684 .complete_pending_relay(
6685 module_connection,
6686 pending.corr,
6687 RouteBindRelayOutcome::Accepted,
6688 )
6689 .unwrap();
6690 (client_ctx, route_channel, route_epoch)
6691 }
6692
6693 struct FakeProcessLiveness {
6694 live: Option<bool>,
6695 }
6696
6697 impl ModuleProcessLiveness for FakeProcessLiveness {
6698 fn process_live(&self, _module_id: &str) -> Option<bool> {
6699 self.live
6700 }
6701 }
6702
6703 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6704 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
6705 {
6706 let registry = Arc::new(Registry::default());
6707 let supervisor_handle = SupervisorHandle::new();
6708 let supervisor = Supervisor::new(
6709 Arc::clone(®istry),
6710 RestartPolicy::new(1, Duration::from_millis(10)),
6711 )
6712 .with_handle(supervisor_handle.clone());
6713 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
6714 let module = supervisor
6715 .spawn(ModuleSpec {
6716 launch_nonce_env: true,
6717 module_id: "stderr-tail-wire".to_string(),
6718 program: fake_aft_stub_path(),
6719 args: Vec::new(),
6720 env: vec![
6721 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
6722 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
6723 ],
6724 reserved: false,
6725 reserved_prefixes: Vec::new(),
6726 protocol: ModuleProtocol::Subc,
6727 overlap: Default::default(),
6728 })
6729 .unwrap();
6730
6731 let deadline = Instant::now() + Duration::from_secs(5);
6732 loop {
6733 let tail = module.stderr_tail(None, None);
6734 if tail
6735 .entries
6736 .iter()
6737 .any(|entry| matches!(entry, TailEntry::ProcessStart))
6738 && tail.entries.iter().any(|entry| {
6739 matches!(
6740 entry,
6741 TailEntry::Line {
6742 truncated: true,
6743 ..
6744 }
6745 )
6746 })
6747 {
6748 break;
6749 }
6750 assert!(
6751 Instant::now() < deadline,
6752 "module did not produce a truncated line and restart boundary: {tail:?}"
6753 );
6754 sleep(Duration::from_millis(10)).await;
6755 }
6756
6757 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
6758 let request = ClientControlRequest::SupervisorStderrTail {
6759 module_id: "stderr-tail-wire".to_string(),
6760 max_lines: None,
6761 max_bytes: None,
6762 };
6763 let frame = Frame::build(
6764 FrameType::Request,
6765 control_flags(),
6766 0,
6767 0,
6768 1,
6769 serde_json::to_vec(&request).unwrap(),
6770 )
6771 .unwrap();
6772 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6773 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
6774 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
6775 serde_json::from_slice(&responses[0].body).unwrap()
6776 else {
6777 panic!("expected supervisor.stderr_tail response");
6778 };
6779
6780 assert!(
6781 tail.entries
6782 .iter()
6783 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
6784 "the control response lost the restart boundary"
6785 );
6786 let Some(StderrTailEntry::Line {
6787 text,
6788 truncated,
6789 at_ms,
6790 }) = tail.entries.iter().find(|entry| {
6791 matches!(
6792 entry,
6793 StderrTailEntry::Line {
6794 truncated: true,
6795 ..
6796 }
6797 )
6798 })
6799 else {
6800 panic!("the control response lost the truncated line");
6801 };
6802 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
6803 assert!(*truncated);
6804 assert!(
6805 at_ms.is_some(),
6806 "the control response lost the line's capture time"
6807 );
6808 }
6809
6810 #[tokio::test(flavor = "current_thread")]
6815 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
6816 let dir = TestTempDir::new("terminals-off-worker");
6817 let journal_path = dir.join("terminals.jsonl");
6818 let registry = Arc::new(Registry::default());
6819 let supervisor_handle = SupervisorHandle::new();
6820 let supervisor =
6821 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
6822 .with_handle(supervisor_handle.clone())
6823 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
6824 let module = supervisor
6825 .spawn(ModuleSpec {
6826 launch_nonce_env: true,
6827 module_id: "terminal-off-worker".to_string(),
6828 program: fake_aft_stub_path(),
6829 args: Vec::new(),
6830 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
6831 reserved: false,
6832 reserved_prefixes: Vec::new(),
6833 protocol: ModuleProtocol::Subc,
6834 overlap: Default::default(),
6835 })
6836 .unwrap();
6837 let deadline = Instant::now() + Duration::from_secs(5);
6838 while module.terminal_history().entries.len() != 2 {
6839 assert!(Instant::now() < deadline, "module did not record two exits");
6840 sleep(Duration::from_millis(10)).await;
6841 }
6842
6843 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
6844 let handler =
6845 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
6846 let frame = Frame::build(
6847 FrameType::Request,
6848 control_flags(),
6849 0,
6850 0,
6851 1,
6852 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
6853 module_id: "terminal-off-worker".to_string(),
6854 })
6855 .unwrap(),
6856 )
6857 .unwrap();
6858 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6859 let spawned_at = std::time::Instant::now();
6860 let read = tokio::spawn({
6861 let handler = Arc::clone(&handler);
6862 async move { handler.handle_control_frame(&ctx, frame).await }
6863 });
6864 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
6867 .await
6868 .unwrap()
6869 .expect("the history read reached its pause");
6870 let elapsed = spawned_at.elapsed();
6871 assert!(
6872 elapsed < Duration::from_secs(2) && !read.is_finished(),
6873 "this task could not run while the history read was paused \
6874 (resumed after {elapsed:?}, read finished: {})",
6875 read.is_finished()
6876 );
6877
6878 drop(release);
6879 let responses = read.await.unwrap().unwrap();
6880 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
6881 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
6882 panic!("expected supervisor.terminals response");
6883 };
6884 assert_eq!(terminals.entries.len(), 2);
6885 assert_eq!(terminals.journal_skipped_lines, 0);
6886 assert_eq!(terminals.journal_read_errors, 0);
6887 }
6888
6889 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6890 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
6891 let registry = Arc::new(Registry::default());
6892 let supervisor_handle = SupervisorHandle::new();
6893 let supervisor =
6894 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
6895 .with_handle(supervisor_handle.clone());
6896 let module = supervisor
6897 .spawn(ModuleSpec {
6898 launch_nonce_env: true,
6899 module_id: "terminal-golden".to_string(),
6900 program: fake_aft_stub_path(),
6901 args: Vec::new(),
6902 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
6903 reserved: false,
6904 reserved_prefixes: Vec::new(),
6905 protocol: ModuleProtocol::Subc,
6906 overlap: Default::default(),
6907 })
6908 .unwrap();
6909
6910 let deadline = Instant::now() + Duration::from_secs(5);
6911 while module.terminal_history().entries.len() != 2 {
6912 assert!(
6913 Instant::now() < deadline,
6914 "module did not retain two terminal exits: {:?}",
6915 module.terminal_history()
6916 );
6917 sleep(Duration::from_millis(10)).await;
6918 }
6919
6920 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
6921 let request = ClientControlRequest::SupervisorTerminals {
6922 module_id: "terminal-golden".to_string(),
6923 };
6924 let frame = Frame::build(
6925 FrameType::Request,
6926 control_flags(),
6927 0,
6928 0,
6929 1,
6930 serde_json::to_vec(&request).unwrap(),
6931 )
6932 .unwrap();
6933 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6934 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
6935 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
6936 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
6937 panic!("expected supervisor.terminals response");
6938 };
6939 assert_eq!(terminals.entries.len(), 2);
6940 assert_eq!(terminals.dropped, 0);
6941
6942 let mut rendered = serde_json::to_value(response).unwrap();
6943 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
6946 for (index, entry) in rendered["entries"]
6947 .as_array_mut()
6948 .expect("terminal response entries array")
6949 .iter_mut()
6950 .enumerate()
6951 {
6952 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
6953 }
6954
6955 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
6956 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
6957 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
6958 if std::env::var_os("UPDATE_GOLDEN").is_some() {
6959 std::fs::write(&golden_path, &serialized).unwrap();
6960 }
6961 let expected: Value =
6962 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
6963 assert_eq!(rendered, expected);
6964 }
6965
6966 #[test]
6967 fn hello_registers_manifest_and_returns_ack() {
6968 let registry = Arc::new(Registry::default());
6969 let handler = ControlHandler::new(Arc::clone(®istry));
6970 let conn = ConnectionId::new(1);
6971
6972 let responses = handler
6973 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
6974 .unwrap();
6975
6976 assert_eq!(responses.len(), 1);
6977 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
6978 assert_eq!(responses[0].header.channel, 0);
6979 assert_eq!(responses[0].header.corr, 7);
6980 let ack = parse_ack(&responses[0]);
6981 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
6982 assert!(ack
6983 .subc_capabilities
6984 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
6985 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
6986 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
6987 assert!(ack
6988 .subc_ops
6989 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
6990 assert!(ack
6991 .subc_ops
6992 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
6993
6994 let registration = registry.get_module("aft").unwrap().unwrap();
6995 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
6996 assert_eq!(registration.state, ChannelState::Active);
6997 assert_eq!(registration.connection_id, conn);
6998 assert_eq!(registration.control_ops, module_baseline_control_ops());
6999 }
7000
7001 #[test]
7002 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
7003 let invalid_identifiers = [
7004 ("case_change", "credentials-Provider/v1"),
7005 ("leading_zero", "credentials-provider/v01"),
7006 ("trailing_hyphen", "credentials-provider-/v1"),
7007 ("consecutive_hyphens", "credentials--provider/v1"),
7008 ("uppercase", "Credentials-provider/v1"),
7009 ("missing_v", "credentials-provider/1"),
7010 ("whitespace", "credentials provider/v1"),
7011 ("zero_version", "credentials-provider/v0"),
7012 ("out_of_range_version", "credentials-provider/v4294967296"),
7013 (
7014 "overlength_name",
7015 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
7016 ),
7017 ];
7018 let mut cases = invalid_identifiers
7019 .into_iter()
7020 .map(|(name, identifier)| {
7021 (
7022 format!("identifier_{name}"),
7023 "capabilities.provides[0]".to_string(),
7024 identifier.to_string(),
7025 json!({ "provides": [identifier] }),
7026 None,
7027 )
7028 })
7029 .collect::<Vec<_>>();
7030 cases.extend([
7031 (
7032 "unknown_need".to_string(),
7033 "capabilities.requires[0].need".to_string(),
7034 "deferred".to_string(),
7035 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
7036 None,
7037 ),
7038 (
7039 "duplicate_provides".to_string(),
7040 "capabilities.provides[1]".to_string(),
7041 "credentials-provider/v1".to_string(),
7042 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
7043 None,
7044 ),
7045 (
7046 "duplicate_must_never_reach".to_string(),
7047 "capabilities.must_never_reach[1]".to_string(),
7048 "credentials-provider/v1".to_string(),
7049 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
7050 None,
7051 ),
7052 (
7053 "duplicate_requires_same_need".to_string(),
7054 "capabilities.requires[1]".to_string(),
7055 "credentials-provider/v1".to_string(),
7056 json!({ "requires": [
7057 { "capability": "credentials-provider/v1", "need": "required" },
7058 { "capability": "credentials-provider/v1", "need": "required" }
7059 ] }),
7060 None,
7061 ),
7062 (
7063 "duplicate_requires_conflicting_need".to_string(),
7064 "capabilities.requires[1]".to_string(),
7065 "credentials-provider/v1".to_string(),
7066 json!({ "requires": [
7067 { "capability": "credentials-provider/v1", "need": "required" },
7068 { "capability": "credentials-provider/v1", "need": "optional" }
7069 ] }),
7070 None,
7071 ),
7072 (
7073 "capabilities_root_pointer".to_string(),
7074 "runtime_computed[0]".to_string(),
7075 "/capabilities".to_string(),
7076 json!({}),
7077 Some(json!(["/capabilities"])),
7078 ),
7079 (
7080 "capabilities_descendant_pointer".to_string(),
7081 "runtime_computed[0]".to_string(),
7082 "/capabilities/provides".to_string(),
7083 json!({}),
7084 Some(json!(["/capabilities/provides"])),
7085 ),
7086 (
7087 "malformed_pointer_without_leading_slash".to_string(),
7088 "runtime_computed[0]".to_string(),
7089 "capabilities".to_string(),
7090 json!({}),
7091 Some(json!(["capabilities"])),
7092 ),
7093 (
7094 "malformed_pointer_escape".to_string(),
7095 "runtime_computed[0]".to_string(),
7096 "/roles/~2/tools".to_string(),
7097 json!({}),
7098 Some(json!(["/roles/~2/tools"])),
7099 ),
7100 (
7101 "unknown_capabilities_field".to_string(),
7102 "capabilities.future".to_string(),
7103 "<array>".to_string(),
7104 json!({ "future": [] }),
7105 None,
7106 ),
7107 ]);
7108
7109 for (index, (name, field, value, capabilities, runtime_computed)) in
7110 cases.into_iter().enumerate()
7111 {
7112 let registry = Arc::new(Registry::default());
7113 let handler = ControlHandler::new(Arc::clone(®istry));
7114 let response = handler
7115 .handle_control(
7116 ConnectionId::new((index + 1) as u64),
7117 capability_grammar_hello_frame(
7118 capabilities,
7119 runtime_computed,
7120 index as u64 + 1,
7121 ),
7122 )
7123 .expect("invalid HELLO returns a refusal");
7124
7125 assert_eq!(response.len(), 1, "{name} must emit one refusal");
7126 let error = parse_error(&response[0]);
7127 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
7128 let message = error["message"]
7129 .as_str()
7130 .expect("error message is a string");
7131 assert!(
7132 message.contains(&field),
7133 "{name}: field missing from {message}"
7134 );
7135 assert!(
7136 message.contains(&value),
7137 "{name}: value missing from {message}"
7138 );
7139 assert_eq!(
7140 registry
7141 .active_registration_count()
7142 .expect("registry reads"),
7143 0,
7144 "{name}: refused HELLO must not create a catalog entry"
7145 );
7146 }
7147 }
7148
7149 #[test]
7150 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
7151 let registry = Arc::new(Registry::default());
7152 let handler = ControlHandler::new(Arc::clone(®istry));
7153 let capabilities = json!({
7154 "provides": ["credentials-provider/v1"],
7155 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
7156 "must_never_reach": ["federation-transport/v1"]
7157 });
7158 let response = handler
7159 .handle_control(
7160 ConnectionId::new(99),
7161 capability_grammar_hello_frame(
7162 capabilities.clone(),
7163 Some(json!(["/roles/0/tools"])),
7164 99,
7165 ),
7166 )
7167 .expect("valid HELLO registers");
7168 assert_eq!(response[0].header.ty, FrameType::HelloAck);
7169
7170 let request = Frame::build(
7171 FrameType::Request,
7172 control_flags(),
7173 0,
7174 0,
7175 100,
7176 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7177 .expect("catalog request serializes"),
7178 )
7179 .expect("catalog request frame builds");
7180 let response = handler
7181 .handle_catalog_list(request, None)
7182 .expect("catalog list succeeds");
7183 let ClientControlResponse::CatalogList { modules, .. } =
7184 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
7185 else {
7186 panic!("catalog request must return catalog.list");
7187 };
7188 assert_eq!(modules.len(), 1);
7189 assert_eq!(
7190 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
7191 capabilities
7192 );
7193 }
7194
7195 #[test]
7196 fn catalog_list_mirrors_management_operation_description() {
7197 let registry = Arc::new(Registry::default());
7198 let handler = ControlHandler::new(Arc::clone(®istry));
7199 let description = "List managed records and return their identifiers and metadata.";
7200 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
7201 manifest.provides = vec![ProviderRole::ManagementSurface {
7202 operations: vec![ManagementOperation {
7203 name: "records.list".to_string(),
7204 kind: ManagementOperationKind::Query,
7205 description: Some(description.to_string()),
7206 }],
7207 config_schema: json!({"type": "object"}),
7208 observability: vec![ObservabilitySurface {
7209 name: "records.stats".to_string(),
7210 kind: ObservabilityKind::Snapshot,
7211 }],
7212 identity_scope: vec![IdentityScope::Project],
7213 concurrency: Concurrency::ModuleManaged,
7214 }];
7215 registry
7216 .register_with_control_ops(
7217 manifest,
7218 PROTOCOL_VERSION,
7219 ConnectionId::new(99),
7220 Vec::new(),
7221 )
7222 .expect("described management manifest registers");
7223
7224 let request = Frame::build(
7225 FrameType::Request,
7226 control_flags(),
7227 0,
7228 0,
7229 100,
7230 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7231 .expect("catalog request serializes"),
7232 )
7233 .expect("catalog request frame builds");
7234 let response = handler
7235 .handle_catalog_list(request, None)
7236 .expect("catalog list succeeds");
7237 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
7238 assert_eq!(
7239 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
7240 "catalog.list must preserve the declared operation description verbatim"
7241 );
7242 }
7243
7244 #[test]
7245 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
7246 let registry = Arc::new(Registry::default());
7247 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
7248 [("vault".to_string(), true), ("squatter".to_string(), true)],
7249 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
7250 );
7251 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
7252 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
7253 provides: vec!["credentials-provider/v1".to_string()],
7254 requires: Vec::new(),
7255 must_never_reach: Vec::new(),
7256 });
7257 let frame = Frame::build(
7258 FrameType::Hello,
7259 control_flags(),
7260 0,
7261 0,
7262 77,
7263 serde_json::to_vec(&ModuleHelloBody {
7264 manifest: squatter,
7265 protocol_ver: PROTOCOL_VERSION,
7266 control_ops: None,
7267 launch_nonce: None,
7268 })
7269 .expect("HELLO serializes"),
7270 )
7271 .expect("HELLO frame builds");
7272 let response = handler
7273 .handle_control(ConnectionId::new(77), frame)
7274 .expect("reserved claim receives a typed refusal");
7275 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
7276 assert_eq!(
7277 registry
7278 .active_registration_count()
7279 .expect("registry reads"),
7280 0,
7281 "a reserved capability refusal must not leave a catalog entry"
7282 );
7283 }
7284
7285 #[test]
7286 fn server_describe_surfaces_required_capability_verdict_fields() {
7287 let registry = Arc::new(Registry::default());
7288 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
7289 [
7290 ("consumer".to_string(), true),
7291 ("provider".to_string(), false),
7292 ],
7293 BTreeMap::new(),
7294 );
7295 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
7296 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
7297 provides: Vec::new(),
7298 requires: vec![subc_protocol::manifest::CapabilityRequirement {
7299 capability: "credentials-provider/v1".to_string(),
7300 need: subc_protocol::manifest::CapabilityNeed::Required,
7301 }],
7302 must_never_reach: Vec::new(),
7303 });
7304 let hello = Frame::build(
7305 FrameType::Hello,
7306 control_flags(),
7307 0,
7308 0,
7309 78,
7310 serde_json::to_vec(&ModuleHelloBody {
7311 manifest: consumer,
7312 protocol_ver: PROTOCOL_VERSION,
7313 control_ops: None,
7314 launch_nonce: None,
7315 })
7316 .expect("HELLO serializes"),
7317 )
7318 .expect("HELLO frame builds");
7319 handler
7320 .handle_control(ConnectionId::new(78), hello)
7321 .expect("consumer registers");
7322 let describe = Frame::build(
7323 FrameType::Request,
7324 control_flags(),
7325 0,
7326 0,
7327 79,
7328 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
7329 .expect("request serializes"),
7330 )
7331 .expect("describe frame builds");
7332 let response = handler
7333 .handle_server_describe(describe)
7334 .expect("server.describe succeeds");
7335 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
7336 let requirement = &rendered["capability_requirements"][0];
7337 assert_eq!(requirement["consumer"], "consumer");
7338 assert_eq!(requirement["verdict"], "never_provided");
7339 assert_eq!(requirement["episode_seq"], 1);
7340 assert_eq!(requirement["config_satisfiable"], false);
7341 assert_eq!(requirement["runtime_available"], false);
7342 assert!(requirement["detail"]
7343 .as_str()
7344 .expect("detail string")
7345 .contains("credentials-provider/v1"));
7346 }
7347
7348 #[test]
7349 fn catalog_list_omits_capabilities_for_legacy_manifest() {
7350 let registry = Arc::new(Registry::default());
7351 let handler = ControlHandler::new(Arc::clone(®istry));
7352 let hello = handler
7353 .handle_control(
7354 ConnectionId::new(101),
7355 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
7356 )
7357 .expect("legacy HELLO registers");
7358 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
7359
7360 let request = Frame::build(
7361 FrameType::Request,
7362 control_flags(),
7363 0,
7364 0,
7365 102,
7366 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7367 .expect("catalog request serializes"),
7368 )
7369 .expect("catalog request frame builds");
7370 let response = handler
7371 .handle_catalog_list(request, None)
7372 .expect("catalog list succeeds");
7373 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
7374 assert!(
7375 body["modules"][0].get("capabilities").is_none(),
7376 "legacy manifest must retain an absent capabilities field on catalog.list"
7377 );
7378 }
7379
7380 #[test]
7381 fn hello_ack_omits_storage_when_no_storage_config() {
7382 let registry = Arc::new(Registry::default());
7383 let handler = ControlHandler::new(Arc::clone(®istry));
7384 let responses = handler
7385 .handle_control(
7386 ConnectionId::new(1),
7387 hello_frame("aft", PROTOCOL_VERSION, 7),
7388 )
7389 .unwrap();
7390 let ack = parse_ack(&responses[0]);
7391 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
7392 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
7393 }
7394
7395 #[tokio::test]
7396 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
7397 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
7398 let registry = Arc::new(Registry::default());
7399 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
7400 let responses = handler
7401 .handle_control(
7402 ConnectionId::new(1),
7403 hello_frame("aft", PROTOCOL_VERSION, 7),
7404 )
7405 .unwrap();
7406 let ack = parse_ack(&responses[0]);
7407 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
7408
7409 let described = handler
7410 .handle_control_frame(
7411 &route_ctx(ConnectionId::new(2)).0,
7412 Frame::build(
7413 FrameType::Request,
7414 control_flags(),
7415 0,
7416 0,
7417 9,
7418 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
7419 )
7420 .unwrap(),
7421 )
7422 .await
7423 .unwrap();
7424 let ClientControlResponse::ServerDescribe { machine_id, .. } =
7425 serde_json::from_slice(&described[0].body).unwrap()
7426 else {
7427 panic!("server.describe answered with another shape");
7428 };
7429 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
7430 }
7431
7432 #[test]
7433 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
7434 let registry = Arc::new(Registry::default());
7437 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
7438 crate::daemon_config::StorageConfig::Sqlite {
7439 data_home: std::path::PathBuf::from("/data"),
7440 },
7441 ));
7442
7443 let responses = handler
7444 .handle_control(
7445 ConnectionId::new(1),
7446 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
7447 )
7448 .unwrap();
7449 let ack = parse_ack(&responses[0]);
7450 assert_eq!(
7451 ack.storage,
7452 Some(serde_json::json!({
7453 "module_id": "alfonso-routing",
7454 "storage_namespace": "default",
7455 "isolation": { "kind": "module" },
7456 "backend": {
7457 "backend": "sqlite",
7458 "path": "/data/cortexkit/alfonso-routing/store.db"
7459 }
7460 })),
7461 "the delivered descriptor is the module's own sqlite store path"
7462 );
7463 }
7464
7465 #[test]
7466 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
7467 let registry = Arc::new(Registry::default());
7468 let handler = ControlHandler::new(Arc::clone(®istry));
7469 let conn = ConnectionId::new(1);
7470 let responses = handler
7471 .handle_control(
7472 conn,
7473 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
7474 )
7475 .unwrap();
7476 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
7477 let registration = registry.get_module("aft").unwrap().unwrap();
7478 assert_eq!(registration.control_ops, module_baseline_control_ops());
7479
7480 let frame =
7481 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
7482 assert!(handler
7483 .guard_module_control_op(&frame, "aft", "route.bind")
7484 .unwrap()
7485 .is_none());
7486 let error = handler
7487 .guard_module_control_op(&frame, "aft", "test.synthetic")
7488 .unwrap()
7489 .expect("synthetic ungranted op should be rejected");
7490 assert_eq!(error.header.ty, FrameType::Error);
7491 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
7492 }
7493
7494 #[test]
7495 fn hello_control_ops_some_adds_optional_grants() {
7496 let registry = Arc::new(Registry::default());
7497 let handler = ControlHandler::new(Arc::clone(®istry));
7498 handler
7499 .handle_control(
7500 ConnectionId::new(1),
7501 hello_frame_with_control_ops(
7502 "aft",
7503 PROTOCOL_VERSION,
7504 7,
7505 Some(vec![
7506 "future.synthetic".to_string(),
7507 "route.bind".to_string(),
7508 ]),
7509 ),
7510 )
7511 .unwrap();
7512 let registration = registry.get_module("aft").unwrap().unwrap();
7513 assert_eq!(
7514 registration.control_ops,
7515 vec![
7516 "route.bind".to_string(),
7517 "route.status".to_string(),
7518 "future.synthetic".to_string(),
7519 ]
7520 );
7521 let frame =
7522 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
7523 assert!(handler
7524 .guard_module_control_op(&frame, "aft", "future.synthetic")
7525 .unwrap()
7526 .is_none());
7527 }
7528
7529 #[tokio::test]
7530 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
7531 let registry = Arc::new(Registry::default());
7532 let forwarding = Arc::new(ForwardingTable::default());
7533 let handler =
7534 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
7535 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
7536 hello_via_sink(
7537 &handler,
7538 &module_ctx,
7539 &mut module_rx,
7540 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
7541 )
7542 .await;
7543
7544 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
7545 let responses = handler
7546 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
7547 .await
7548 .unwrap();
7549 assert_eq!(responses.len(), 1);
7550 assert_eq!(responses[0].header.ty, FrameType::Error);
7551 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
7552 assert!(module_rx.try_recv().is_err());
7553 }
7554
7555 #[tokio::test]
7556 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
7557 let registry = Arc::new(Registry::default());
7558 let forwarding = Arc::new(ForwardingTable::default());
7559 let handler =
7560 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
7561 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
7562 hello_via_sink(
7563 &handler,
7564 &module_ctx,
7565 &mut module_rx,
7566 hello_frame_with_control_ops(
7567 "aft",
7568 PROTOCOL_VERSION,
7569 7,
7570 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
7571 ),
7572 )
7573 .await;
7574
7575 let project_root = unique_project_root("demux");
7576 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
7577 let route_handler = handler.clone();
7578 let route_task = tokio::spawn(async move {
7579 route_handler
7580 .handle_control_frame(
7581 &route_client_ctx,
7582 route_open_frame(100, "aft", project_root),
7583 )
7584 .await
7585 .unwrap()
7586 });
7587 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7588 .await
7589 .unwrap()
7590 .unwrap();
7591 assert!(matches!(
7592 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
7593 ModuleControlRequest::RouteBind { .. }
7594 ));
7595
7596 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
7597 let health_handler = handler.clone();
7598 let health_task = tokio::spawn(async move {
7599 health_handler
7600 .handle_control_frame(
7601 &health_client_ctx,
7602 supervisor_health_probe_frame(101, "aft"),
7603 )
7604 .await
7605 .unwrap()
7606 });
7607 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7608 .await
7609 .unwrap()
7610 .unwrap();
7611 assert_eq!(
7612 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
7613 ModuleControlRequest::HealthCheck {}
7614 );
7615
7616 handler
7617 .handle_control_frame(
7618 &module_ctx,
7619 health_response(health_frame.header.corr, HealthStatus::Degraded),
7620 )
7621 .await
7622 .unwrap();
7623 let health_response = health_task.await.unwrap();
7624 assert_eq!(health_response.len(), 1);
7625 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
7626 ClientControlResponse::SupervisorHealthProbe {
7627 module_id,
7628 status,
7629 detail,
7630 metrics,
7631 } => {
7632 assert_eq!(module_id, "aft");
7633 assert_eq!(status, HealthStatus::Degraded);
7634 assert_eq!(detail.as_deref(), Some("warming"));
7635 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
7636 }
7637 other => panic!("unexpected health response: {other:?}"),
7638 }
7639
7640 handler
7641 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
7642 .await
7643 .unwrap();
7644 let route_response = route_task.await.unwrap();
7645 assert!(route_response.is_empty());
7646 let published = route_client_rx.recv().await.unwrap();
7647 assert!(matches!(
7648 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
7649 ClientControlResponse::RouteOpen { .. }
7650 ));
7651 }
7652
7653 async fn relay_route_open(
7658 handler: &ControlHandler,
7659 client_connection: ConnectionId,
7660 client_egress: &FrameSink,
7661 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
7662 corr: u64,
7663 module_id: &str,
7664 project_root_label: &str,
7665 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
7666 let ctx = RouteCtx {
7667 connection_id: client_connection,
7668 egress: client_egress.clone(),
7669 };
7670 let handler = handler.clone();
7671 let project_root = unique_project_root(project_root_label);
7672 let module_id = module_id.to_string();
7673 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
7674 let task = tokio::spawn(async move {
7675 let _guard = tracing::dispatcher::set_default(&dispatch);
7676 handler
7677 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
7678 .await
7679 .unwrap()
7680 });
7681 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
7682 .await
7683 .expect("module receives the relayed route.bind")
7684 .expect("module egress is open");
7685 (task, bind.frame)
7686 }
7687
7688 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
7689 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
7690 ModuleControlRequest::RouteBind {
7691 route_channel,
7692 epoch,
7693 ..
7694 } => (route_channel, epoch),
7695 other => panic!("expected a route.bind request, got {other:?}"),
7696 }
7697 }
7698
7699 fn published_route(frame: &Frame) -> (u16, u32) {
7700 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
7701 ClientControlResponse::RouteOpen {
7702 route_channel,
7703 route_epoch,
7704 } => (route_channel, route_epoch),
7705 other => panic!("expected a route.open response, got {other:?}"),
7706 }
7707 }
7708
7709 #[tokio::test]
7729 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
7730 let registry = Arc::new(Registry::default());
7731 let forwarding = Arc::new(ForwardingTable::default());
7732 let handler =
7733 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
7734
7735 let module_connection = ConnectionId::new(30);
7736 let (module_ctx, mut module_rx) = route_ctx(module_connection);
7737 hello_via_sink(
7738 &handler,
7739 &module_ctx,
7740 &mut module_rx,
7741 hello_frame("aft", PROTOCOL_VERSION, 7),
7742 )
7743 .await;
7744
7745 let dying_client = ConnectionId::new(31);
7746 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
7747
7748 let (first_task, first_bind) = relay_route_open(
7751 &handler,
7752 dying_client,
7753 &dying_ctx.egress,
7754 &mut module_rx,
7755 100,
7756 "aft",
7757 "closing-first",
7758 )
7759 .await;
7760 handler
7761 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
7762 .await
7763 .unwrap();
7764 assert!(first_task.await.unwrap().is_empty());
7765 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
7766
7767 let (second_task, second_bind) = relay_route_open(
7769 &handler,
7770 dying_client,
7771 &dying_ctx.egress,
7772 &mut module_rx,
7773 101,
7774 "aft",
7775 "closing-second",
7776 )
7777 .await;
7778 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
7779
7780 assert!(forwarding
7783 .escalate_client_delivery_failure(
7784 dying_client,
7785 first_channel,
7786 first_epoch,
7787 CloseReason::new(
7788 "module_to_client_delivery_failed",
7789 "client egress refused a module frame",
7790 ),
7791 crate::forwarding::UndeliveredFrame {
7792 module_id: None,
7793 sink: &dying_ctx.egress,
7794 },
7795 )
7796 .unwrap());
7797 assert!(!dying_ctx.egress.is_closed());
7798
7799 let ack = handler
7801 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
7802 .await;
7803 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
7804 if module_loop_error.is_some() {
7805 handler.cleanup_connection(module_connection).unwrap();
7809 }
7810 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7814 .await
7815 .ok()
7816 .flatten();
7817
7818 assert!(
7820 registry
7821 .get_module_by_connection(module_connection)
7822 .unwrap()
7823 .is_some(),
7824 "one client's closing connection ended the shared module connection: \
7825 {module_loop_error:?}"
7826 );
7827 let cotenant = ConnectionId::new(32);
7829 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
7830 let (cotenant_task, cotenant_bind) = relay_route_open(
7831 &handler,
7832 cotenant,
7833 &cotenant_ctx.egress,
7834 &mut module_rx,
7835 102,
7836 "aft",
7837 "closing-cotenant",
7838 )
7839 .await;
7840 handler
7841 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
7842 .await
7843 .unwrap();
7844 assert!(cotenant_task.await.unwrap().is_empty());
7845 let (cotenant_channel, cotenant_epoch) =
7846 published_route(&cotenant_rx.recv().await.unwrap());
7847 assert!(matches!(
7848 forwarding
7849 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
7850 .unwrap(),
7851 DataRoute::Client(DataRouteState::Bound(_))
7852 ));
7853
7854 let goodbye = post_ack_module_frame
7857 .expect("module receives a GOODBYE for the abandoned route channel");
7858 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
7859 assert_eq!(goodbye.header.channel, abandoned_channel);
7860 assert_eq!(goodbye.header.epoch, abandoned_epoch);
7861
7862 assert!(dying_rx.try_recv().is_err());
7866 let second_response = second_task.await.unwrap();
7867 assert_eq!(second_response.len(), 1);
7868 assert_eq!(
7869 parse_error(&second_response[0])["code"],
7870 "target_unavailable"
7871 );
7872 }
7873
7874 #[test]
7881 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
7882 let handler = ControlHandler::default();
7883 let module_connection = ConnectionId::new(30);
7884 let client_connection = ConnectionId::new(31);
7885
7886 handler
7887 .refuse_to_end_module_connection_for_a_client(
7888 module_connection,
7889 77,
7890 ForwardingError::ConnectionClosing {
7891 connection_id: client_connection,
7892 },
7893 )
7894 .expect("a closing client must never end the module connection");
7895
7896 assert!(matches!(
7897 handler.refuse_to_end_module_connection_for_a_client(
7898 module_connection,
7899 78,
7900 ForwardingError::ConnectionClosing {
7901 connection_id: module_connection,
7902 },
7903 ),
7904 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
7905 connection_id
7906 })) if connection_id == module_connection
7907 ));
7908 assert!(matches!(
7909 handler.refuse_to_end_module_connection_for_a_client(
7910 module_connection,
7911 79,
7912 ForwardingError::Poisoned,
7913 ),
7914 Err(RouterError::Forwarding(ForwardingError::Poisoned))
7915 ));
7916 assert!(matches!(
7917 handler.refuse_to_end_module_connection_for_a_client(
7918 module_connection,
7919 80,
7920 ForwardingError::StaleModuleEndpoint,
7921 ),
7922 Err(RouterError::Forwarding(
7923 ForwardingError::StaleModuleEndpoint
7924 ))
7925 ));
7926 }
7927
7928 #[tokio::test]
7941 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
7942 let registry = Arc::new(Registry::default());
7943 let forwarding = Arc::new(ForwardingTable::default());
7944 let supervisor = SupervisorHandle::new();
7945 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7946 let handler =
7947 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
7948 .with_supervisor(supervisor);
7949
7950 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
7951 hello_via_sink(
7952 &handler,
7953 &target_ctx,
7954 &mut target_rx,
7955 hello_frame("target", PROTOCOL_VERSION, 1),
7956 )
7957 .await;
7958
7959 let wrong_nonce = handler
7963 .handle_control_frame(
7964 &route_ctx(ConnectionId::new(91)).0,
7965 route_open_frame_with_admission_facts(
7966 20,
7967 "target",
7968 unique_project_root("admission-facts"),
7969 Some(subc_control::ConsumerIdentity {
7970 module_id: "fed".to_string(),
7971 launch_nonce: "not-the-real-nonce".to_string(),
7972 }),
7973 None,
7974 ),
7975 )
7976 .await
7977 .unwrap();
7978 assert_eq!(
7979 parse_error(&wrong_nonce[0])["code"],
7980 "bad_consumer_identity",
7981 "a mismatched launch nonce must be refused, not stamped Reserved"
7982 );
7983
7984 let never_spawned = handler
7988 .handle_control_frame(
7989 &route_ctx(ConnectionId::new(92)).0,
7990 route_open_frame_with_admission_facts(
7991 21,
7992 "target",
7993 unique_project_root("admission-facts"),
7994 Some(subc_control::ConsumerIdentity {
7995 module_id: "never-spawned".to_string(),
7996 launch_nonce: "any-nonce".to_string(),
7997 }),
7998 None,
7999 ),
8000 )
8001 .await
8002 .unwrap();
8003 assert_eq!(
8004 parse_error(&never_spawned[0])["code"],
8005 "bad_consumer_identity",
8006 "an unspawned module_id must be refused rather than accepted for lack of a record"
8007 );
8008 }
8009
8010 #[tokio::test]
8033 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
8034 let registry = Arc::new(Registry::default());
8035 let forwarding = Arc::new(ForwardingTable::default());
8036 let supervisor = SupervisorHandle::new();
8037 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8038 let handler =
8039 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8040 .with_supervisor(supervisor);
8041
8042 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
8043 hello_via_sink(
8044 &handler,
8045 &target_ctx,
8046 &mut target_rx,
8047 hello_frame("target", PROTOCOL_VERSION, 1),
8048 )
8049 .await;
8050
8051 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
8052 let route_handler = handler.clone();
8053 let route_task = tokio::spawn(async move {
8054 route_handler
8055 .handle_control_frame(
8056 &client_ctx,
8057 route_open_frame_with_admission_facts(
8058 30,
8059 "target",
8060 unique_project_root("admission-facts"),
8061 Some(subc_control::ConsumerIdentity {
8062 module_id: "fed".to_string(),
8063 launch_nonce: "fed-nonce".to_string(),
8064 }),
8065 None,
8066 ),
8067 )
8068 .await
8069 .unwrap()
8070 });
8071
8072 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8079 .await
8080 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
8081 .expect("module control channel closed before route.bind");
8082 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8083 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
8084 panic!("expected route.bind")
8085 };
8086 assert_eq!(
8087 principal,
8088 Some(Principal::Reserved {
8089 module_id: "fed".to_string()
8090 }),
8091 "a correctly attested consumer must be stamped Reserved for its own id"
8092 );
8093
8094 handler
8095 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8096 .await
8097 .unwrap();
8098 assert!(route_task.await.unwrap().is_empty());
8099 assert!(
8100 matches!(
8101 serde_json::from_slice::<ClientControlResponse>(
8102 &client_rx.recv().await.unwrap().body
8103 )
8104 .unwrap(),
8105 ClientControlResponse::RouteOpen { .. }
8106 ),
8107 "the route must actually open, not merely avoid an error"
8108 );
8109 }
8110
8111 #[tokio::test(start_paused = true)]
8112 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
8113 let registry = Arc::new(Registry::default());
8114 let forwarding = Arc::new(ForwardingTable::default());
8115 let supervisor = SupervisorHandle::new();
8116 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8117 let handler =
8118 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8119 .with_supervisor(supervisor);
8120
8121 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
8122 hello_via_sink(
8123 &handler,
8124 &target_ctx,
8125 &mut target_rx,
8126 hello_frame("target", PROTOCOL_VERSION, 1),
8127 )
8128 .await;
8129
8130 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
8131 let direct_handler = handler.clone();
8132 let direct_open = tokio::spawn(async move {
8133 direct_handler
8134 .handle_control_frame(
8135 &direct_ctx,
8136 route_open_frame(2, "target", unique_project_root("route-census-direct")),
8137 )
8138 .await
8139 .unwrap()
8140 });
8141 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8142 .await
8143 .expect("no direct route.bind within 5s")
8144 .expect("target control channel closed before direct route.bind");
8145 handler
8146 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
8147 .await
8148 .unwrap();
8149 assert!(direct_open.await.unwrap().is_empty());
8150 let _ = direct_rx.recv().await.unwrap();
8151
8152 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
8153 let reserved_handler = handler.clone();
8154 let reserved_open = tokio::spawn(async move {
8155 reserved_handler
8156 .handle_control_frame(
8157 &reserved_ctx,
8158 route_open_frame_with_admission_facts(
8159 3,
8160 "target",
8161 unique_project_root("admission-facts"),
8162 Some(ConsumerIdentity {
8163 module_id: "fed".to_string(),
8164 launch_nonce: "fed-nonce".to_string(),
8165 }),
8166 None,
8167 ),
8168 )
8169 .await
8170 .unwrap()
8171 });
8172 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8173 .await
8174 .expect("no reserved route.bind within 5s")
8175 .expect("target control channel closed before reserved route.bind");
8176 handler
8177 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
8178 .await
8179 .unwrap();
8180 assert!(reserved_open.await.unwrap().is_empty());
8181 let _ = reserved_rx.recv().await.unwrap();
8182
8183 forwarding
8184 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
8185 .unwrap();
8186 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
8187 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
8188 module_id: Some("target".to_string()),
8189 })
8190 .unwrap();
8191 let census_frame =
8192 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
8193 let response = handler
8194 .handle_control_frame(&census_ctx, census_frame)
8195 .await
8196 .unwrap()
8197 .pop()
8198 .unwrap();
8199 let actual: Value = serde_json::from_slice(&response.body).unwrap();
8200 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
8201 assert!(matches!(
8202 decoded,
8203 ClientControlResponse::SupervisorRoutes { .. }
8204 ));
8205 let routes = actual["modules"][0]["routes"].as_array().unwrap();
8206 assert_eq!(routes.len(), 2);
8207 assert!(routes.iter().all(|route| route["draining"] == true));
8208 assert!(
8211 routes.iter().all(|route| route["drain_reason"] == "reload"),
8212 "draining routes must name the drain's reason: {routes:?}"
8213 );
8214 assert!(routes.iter().any(|route| {
8215 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
8216 }));
8217 assert!(routes.iter().any(|route| {
8218 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
8219 }));
8220
8221 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8222 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
8223 if std::env::var_os("UPDATE_GOLDEN").is_some() {
8224 std::fs::write(
8225 &golden_path,
8226 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
8227 )
8228 .unwrap();
8229 }
8230 let expected: Value =
8231 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
8232 assert_eq!(actual, expected);
8233 }
8234
8235 async fn query_live_roots(
8236 handler: &ControlHandler,
8237 module_ctx: &RouteCtx,
8238 ) -> ModuleControlResponseToModule {
8239 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
8240 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
8241 let response = handler
8242 .handle_control_frame(module_ctx, frame)
8243 .await
8244 .unwrap()
8245 .pop()
8246 .unwrap();
8247 serde_json::from_slice(&response.body).unwrap()
8248 }
8249
8250 #[tokio::test(start_paused = true)]
8251 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
8252 let registry = Arc::new(Registry::default());
8253 let forwarding = Arc::new(ForwardingTable::default());
8254 let handler = ControlHandler::with_forwarding(registry, forwarding);
8255 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
8256 hello_via_sink(
8257 &handler,
8258 &target_ctx,
8259 &mut target_rx,
8260 hello_frame("target", PROTOCOL_VERSION, 1),
8261 )
8262 .await;
8263 let root = unique_project_root("live-roots-known");
8264 let path = ProjectRootId::from_path_allowing_missing(root.path())
8265 .unwrap()
8266 .as_path()
8267 .to_path_buf();
8268 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
8269 let open_handler = handler.clone();
8270 let opened = tokio::spawn(async move {
8271 open_handler
8272 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
8273 .await
8274 .unwrap()
8275 });
8276 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8277 .await
8278 .unwrap()
8279 .unwrap();
8280 handler
8281 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
8282 .await
8283 .unwrap();
8284 assert!(opened.await.unwrap().is_empty());
8285 let _ = client_rx.recv().await.unwrap();
8286
8287 let root = unique_project_root("live-roots-pending");
8288 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
8289 .unwrap()
8290 .as_path()
8291 .to_path_buf();
8292 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
8293 let open_handler = handler.clone();
8294 let pending = tokio::spawn(async move {
8295 open_handler
8296 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
8297 .await
8298 .unwrap()
8299 });
8300 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8301 .await
8302 .unwrap()
8303 .unwrap();
8304 let actual = query_live_roots(&handler, &target_ctx).await;
8305 let ModuleControlResponseToModule::LiveRoots {
8306 roots,
8307 unknown_root_bindings,
8308 total_bindings,
8309 } = actual
8310 else {
8311 panic!("expected live roots")
8312 };
8313 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
8314 assert_eq!(unknown_root_bindings, 0);
8315 assert_eq!(
8316 roots.len(),
8317 2,
8318 "root-known arm must retain each canonical root"
8319 );
8320 assert_eq!(
8321 total_bindings,
8322 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
8323 );
8324 let counts = roots
8325 .iter()
8326 .map(|root| (root.project_root.clone(), root.bound, root.pending))
8327 .collect::<Vec<_>>();
8328 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
8329 expected.sort_by(|a, b| a.0.cmp(&b.0));
8330 assert_eq!(
8331 counts, expected,
8332 "roots must sort by path and count pending separately"
8333 );
8334 handler
8335 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
8336 .await
8337 .unwrap();
8338 assert!(pending.await.unwrap().is_empty());
8339 }
8340
8341 #[tokio::test(start_paused = true)]
8342 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
8343 let forwarding = Arc::new(ForwardingTable::default());
8344 let handler =
8345 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
8346 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
8347 hello_via_sink(
8348 &handler,
8349 &target_ctx,
8350 &mut target_rx,
8351 hello_frame("target", PROTOCOL_VERSION, 1),
8352 )
8353 .await;
8354 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
8355 let pending = forwarding
8356 .begin_route_bind_relay_for_test(
8357 client_ctx.connection_id,
8358 client_ctx.egress.clone(),
8359 2,
8360 "target",
8361 )
8362 .unwrap();
8363 forwarding
8364 .complete_pending_relay(
8365 target_ctx.connection_id,
8366 pending.corr,
8367 RouteBindRelayOutcome::Accepted,
8368 )
8369 .unwrap();
8370 let actual = query_live_roots(&handler, &target_ctx).await;
8371 let ModuleControlResponseToModule::LiveRoots {
8372 roots,
8373 unknown_root_bindings,
8374 total_bindings,
8375 } = actual
8376 else {
8377 panic!("expected live roots")
8378 };
8379 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
8380 assert_eq!(
8381 unknown_root_bindings, 1,
8382 "unknown-root arm must not read as no bindings"
8383 );
8384 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
8385 assert_eq!(
8386 total_bindings,
8387 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
8388 );
8389 }
8390
8391 #[tokio::test(start_paused = true)]
8399 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
8400 let forwarding = Arc::new(ForwardingTable::default());
8401 let handler =
8402 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
8403 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
8404 let replies = handler
8405 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
8406 .await
8407 .unwrap();
8408 let queued_by_hello = module_rx.len();
8409
8410 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
8411 let open_handler = handler.clone();
8412 let open = tokio::spawn(async move {
8413 open_handler
8414 .handle_control_frame(
8415 &client_ctx,
8416 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
8417 )
8418 .await
8419 .unwrap()
8420 });
8421 let mut spins = 0;
8423 while module_rx.len() == queued_by_hello {
8424 spins += 1;
8425 assert!(spins < 10_000, "route.open never queued a route.bind");
8426 tokio::task::yield_now().await;
8427 }
8428
8429 for reply in replies {
8431 module_ctx.egress.send(reply).await.unwrap();
8432 }
8433
8434 let first = module_rx.recv().await.unwrap().frame;
8435 assert_eq!(
8436 first.header.ty,
8437 FrameType::HelloAck,
8438 "the first frame a registering module reads must be its HELLO_ACK"
8439 );
8440 assert_eq!(first.header.corr, 7);
8441 let second = module_rx.recv().await.unwrap().frame;
8442 assert_eq!(second.header.ty, FrameType::Request);
8443 assert!(
8444 matches!(
8445 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
8446 ModuleControlRequest::RouteBind { .. }
8447 ),
8448 "the route.bind follows the ack"
8449 );
8450 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
8451
8452 handler
8453 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
8454 .await
8455 .unwrap();
8456 assert!(open.await.unwrap().is_empty());
8457 let _ = client_rx.recv().await.unwrap();
8458 }
8459
8460 #[tokio::test(start_paused = true)]
8461 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
8462 let handler = ControlHandler::with_forwarding(
8463 Arc::new(Registry::default()),
8464 Arc::new(ForwardingTable::default()),
8465 );
8466 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
8467 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
8468 hello_via_sink(
8469 &handler,
8470 &first_ctx,
8471 &mut first_rx,
8472 hello_frame("first", PROTOCOL_VERSION, 1),
8473 )
8474 .await;
8475 hello_via_sink(
8476 &handler,
8477 &second_ctx,
8478 &mut second_rx,
8479 hello_frame("second", PROTOCOL_VERSION, 2),
8480 )
8481 .await;
8482 let root = unique_project_root("second-only");
8483 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
8484 let cloned = handler.clone();
8485 let open = tokio::spawn(async move {
8486 cloned
8487 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
8488 .await
8489 .unwrap()
8490 });
8491 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
8492 .await
8493 .unwrap()
8494 .unwrap();
8495 let first = query_live_roots(&handler, &first_ctx).await;
8496 let second = query_live_roots(&handler, &second_ctx).await;
8497 assert!(
8498 matches!(
8499 first,
8500 ModuleControlResponseToModule::LiveRoots {
8501 total_bindings: 0,
8502 ..
8503 }
8504 ),
8505 "cross-module scope must not expose another module's roots"
8506 );
8507 assert!(
8508 matches!(
8509 second,
8510 ModuleControlResponseToModule::LiveRoots {
8511 total_bindings: 1,
8512 ..
8513 }
8514 ),
8515 "second module must see its pending route"
8516 );
8517 handler
8518 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
8519 .await
8520 .unwrap();
8521 assert!(open.await.unwrap().is_empty());
8522 }
8523
8524 #[tokio::test(start_paused = true)]
8525 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
8526 let handler = ControlHandler::with_forwarding(
8527 Arc::new(Registry::default()),
8528 Arc::new(ForwardingTable::default()),
8529 );
8530 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
8531 hello_via_sink(
8532 &handler,
8533 &target_ctx,
8534 &mut target_rx,
8535 hello_frame("target", PROTOCOL_VERSION, 1),
8536 )
8537 .await;
8538 let actual = query_live_roots(&handler, &target_ctx).await;
8539 let ModuleControlResponseToModule::LiveRoots {
8540 roots,
8541 unknown_root_bindings,
8542 total_bindings,
8543 } = actual
8544 else {
8545 panic!("expected live roots")
8546 };
8547 assert!(roots.is_empty());
8548 assert_eq!(unknown_root_bindings, 0);
8549 assert_eq!(total_bindings, 0);
8550 assert_eq!(
8551 total_bindings,
8552 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
8553 );
8554 }
8555
8556 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
8563 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8564 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
8565 let text = std::fs::read_to_string(&path)
8566 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
8567 let vectors: Vec<(String, Value)> = text
8568 .lines()
8569 .filter(|line| !line.trim().is_empty())
8570 .map(|line| {
8571 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
8572 let id = entry["corpus_id"]
8573 .as_str()
8574 .expect("every vector carries a corpus_id")
8575 .to_string();
8576 (id, entry["package"].clone())
8577 })
8578 .collect();
8579 assert_eq!(
8583 vectors.len(),
8584 3,
8585 "vendored fed corpus changed size; re-sync from subc-federation"
8586 );
8587
8588 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
8603 let richest = vectors
8604 .iter()
8605 .filter_map(|(_, package)| package.as_object())
8606 .map(|object| {
8607 object
8608 .keys()
8609 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
8610 .count()
8611 })
8612 .max()
8613 .unwrap_or(0);
8614 assert!(
8615 richest >= 2,
8616 "vendored corpus no longer carries a package with unmodelled fields, \
8617 so the relay test can no longer distinguish a verbatim relay from a lossy one"
8618 );
8619
8620 vectors
8621 }
8622
8623 #[tokio::test]
8631 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
8632 for (corpus_id, package) in fed_admission_facts_vectors() {
8633 let registry = Arc::new(Registry::default());
8634 let forwarding = Arc::new(ForwardingTable::default());
8635 let supervisor = SupervisorHandle::new();
8636 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8637 let handler =
8638 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8639 .with_supervisor(supervisor)
8640 .with_admission_facts_config(
8641 Some("fed".to_string()),
8642 Some(vec!["target".to_string()]),
8643 );
8644
8645 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
8646 hello_via_sink(
8647 &handler,
8648 &target_ctx,
8649 &mut target_rx,
8650 hello_frame("target", PROTOCOL_VERSION, 1),
8651 )
8652 .await;
8653
8654 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
8655 let route_handler = handler.clone();
8656 let expected = package.clone();
8657 let route_task = tokio::spawn(async move {
8658 route_handler
8659 .handle_control_frame(
8660 &client_ctx,
8661 route_open_frame_with_admission_facts(
8662 20,
8663 "target",
8664 unique_project_root("admission-facts"),
8665 Some(subc_control::ConsumerIdentity {
8666 module_id: "fed".to_string(),
8667 launch_nonce: "fed-nonce".to_string(),
8668 }),
8669 Some(package),
8670 ),
8671 )
8672 .await
8673 .unwrap()
8674 });
8675
8676 let bind_frame = target_rx.recv().await.unwrap();
8677 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8678 let ModuleControlRequest::RouteBind {
8679 admission_facts, ..
8680 } = bind
8681 else {
8682 panic!("{corpus_id}: expected route.bind")
8683 };
8684 assert_eq!(
8685 admission_facts,
8686 Some(expected),
8687 "{corpus_id}: relay must not add, drop or reshape any field"
8688 );
8689
8690 handler
8691 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8692 .await
8693 .unwrap();
8694 route_task.await.unwrap();
8695 }
8696 }
8697
8698 #[tokio::test]
8699 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
8700 let registry = Arc::new(Registry::default());
8701 let forwarding = Arc::new(ForwardingTable::default());
8702 let supervisor = SupervisorHandle::new();
8703 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8704 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
8705 let handler =
8706 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8707 .with_supervisor(supervisor)
8708 .with_admission_facts_config(
8709 Some("fed".to_string()),
8710 Some(vec!["target".to_string()]),
8711 );
8712
8713 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
8714 hello_via_sink(
8715 &handler,
8716 &target_ctx,
8717 &mut target_rx,
8718 hello_frame("target", PROTOCOL_VERSION, 1),
8719 )
8720 .await;
8721 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
8722 hello_via_sink(
8723 &handler,
8724 &other_ctx,
8725 &mut other_rx,
8726 hello_frame("other", PROTOCOL_VERSION, 2),
8727 )
8728 .await;
8729
8730 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
8731 let expected_facts = facts.clone();
8732 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
8733 let route_handler = handler.clone();
8734 let route_task = tokio::spawn(async move {
8735 route_handler
8736 .handle_control_frame(
8737 &client_ctx,
8738 route_open_frame_with_admission_facts(
8739 10,
8740 "target",
8741 unique_project_root("admission-facts"),
8742 Some(subc_control::ConsumerIdentity {
8743 module_id: "fed".to_string(),
8744 launch_nonce: "fed-nonce".to_string(),
8745 }),
8746 Some(facts.clone()),
8747 ),
8748 )
8749 .await
8750 .unwrap()
8751 });
8752 let bind_frame = target_rx.recv().await.unwrap();
8753 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8754 let ModuleControlRequest::RouteBind {
8755 admission_facts, ..
8756 } = bind
8757 else {
8758 panic!("expected route.bind")
8759 };
8760 assert_eq!(admission_facts, Some(expected_facts));
8761 handler
8762 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8763 .await
8764 .unwrap();
8765 assert!(route_task.await.unwrap().is_empty());
8766 assert!(matches!(
8767 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
8768 .unwrap(),
8769 ClientControlResponse::RouteOpen { .. }
8770 ));
8771
8772 let direct = handler
8773 .handle_control_frame(
8774 &route_ctx(ConnectionId::new(73)).0,
8775 route_open_frame_with_admission_facts(
8776 11,
8777 "target",
8778 unique_project_root("admission-facts"),
8779 None,
8780 Some(json!({"x": 1})),
8781 ),
8782 )
8783 .await
8784 .unwrap();
8785 assert_eq!(
8786 parse_error(&direct[0])["code"],
8787 "admission_facts_not_permitted"
8788 );
8789
8790 let different_reserved = handler
8791 .handle_control_frame(
8792 &route_ctx(ConnectionId::new(77)).0,
8793 route_open_frame_with_admission_facts(
8794 15,
8795 "target",
8796 unique_project_root("admission-facts"),
8797 Some(subc_control::ConsumerIdentity {
8798 module_id: "other".to_string(),
8799 launch_nonce: "other-nonce".to_string(),
8800 }),
8801 Some(json!({"x": 1})),
8802 ),
8803 )
8804 .await
8805 .unwrap();
8806 assert_eq!(
8807 parse_error(&different_reserved[0])["code"],
8808 "admission_facts_not_permitted"
8809 );
8810
8811 let other_target = handler
8812 .handle_control_frame(
8813 &route_ctx(ConnectionId::new(74)).0,
8814 route_open_frame_with_admission_facts(
8815 12,
8816 "other",
8817 unique_project_root("admission-facts"),
8818 Some(subc_control::ConsumerIdentity {
8819 module_id: "fed".to_string(),
8820 launch_nonce: "fed-nonce".to_string(),
8821 }),
8822 Some(json!({"x": 1})),
8823 ),
8824 )
8825 .await
8826 .unwrap();
8827 assert_eq!(
8828 parse_error(&other_target[0])["code"],
8829 "admission_facts_target_not_allowed"
8830 );
8831
8832 let nonexistent = handler
8833 .handle_control_frame(
8834 &route_ctx(ConnectionId::new(75)).0,
8835 route_open_frame_with_admission_facts(
8836 13,
8837 "missing",
8838 unique_project_root("admission-facts"),
8839 None,
8840 Some(json!({"x": 1})),
8841 ),
8842 )
8843 .await
8844 .unwrap();
8845 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
8846
8847 let described = handler
8848 .handle_control_frame(
8849 &route_ctx(ConnectionId::new(76)).0,
8850 Frame::build(
8851 FrameType::Request,
8852 control_flags(),
8853 0,
8854 0,
8855 14,
8856 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
8857 )
8858 .unwrap(),
8859 )
8860 .await
8861 .unwrap();
8862 let ClientControlResponse::ServerDescribe { capabilities, .. } =
8863 serde_json::from_slice(&described[0].body).unwrap()
8864 else {
8865 panic!("expected server.describe response")
8866 };
8867 assert!(capabilities
8868 .iter()
8869 .any(|cap| cap == "admission_facts_relay_v1"));
8870 }
8871
8872 #[tokio::test]
8873 async fn admission_facts_without_configured_carrier_are_rejected() {
8874 let registry = Arc::new(Registry::default());
8875 let forwarding = Arc::new(ForwardingTable::default());
8876 let handler = ControlHandler::with_forwarding(registry, forwarding);
8877 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
8878 hello_via_sink(
8879 &handler,
8880 &target_ctx,
8881 &mut target_rx,
8882 hello_frame("target", PROTOCOL_VERSION, 1),
8883 )
8884 .await;
8885
8886 let responses = handler
8887 .handle_control_frame(
8888 &route_ctx(ConnectionId::new(79)).0,
8889 route_open_frame_with_admission_facts(
8890 16,
8891 "target",
8892 unique_project_root("admission-facts"),
8893 None,
8894 Some(json!({"x": 1})),
8895 ),
8896 )
8897 .await
8898 .unwrap();
8899 assert_eq!(
8900 parse_error(&responses[0])["code"],
8901 "admission_facts_not_permitted"
8902 );
8903 }
8904
8905 #[tokio::test]
8906 async fn route_open_relays_consumer_capabilities_verbatim() {
8907 let registry = Arc::new(Registry::default());
8908 let forwarding = Arc::new(ForwardingTable::default());
8909 let handler =
8910 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8911 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
8912 hello_via_sink(
8913 &handler,
8914 &module_ctx,
8915 &mut module_rx,
8916 hello_frame("aft", PROTOCOL_VERSION, 7),
8917 )
8918 .await;
8919
8920 let expected = vec!["elicitation".to_string(), "roots".to_string()];
8921 let expected_for_request = expected.clone();
8922 let project_root = unique_project_root("consumer-capabilities-present");
8923 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
8924 let route_handler = handler.clone();
8925 let route_task = tokio::spawn(async move {
8926 route_handler
8927 .handle_control_frame(
8928 &client_ctx,
8929 route_open_frame_with_consumer_capabilities(
8930 401,
8931 "aft",
8932 project_root,
8933 Some(expected_for_request),
8934 ),
8935 )
8936 .await
8937 .unwrap()
8938 });
8939 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8940 .await
8941 .unwrap()
8942 .unwrap();
8943 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8944 let ModuleControlRequest::RouteBind {
8945 consumer_capabilities,
8946 ..
8947 } = bind
8948 else {
8949 panic!("expected route.bind request, got {bind:?}");
8950 };
8951 assert_eq!(consumer_capabilities, Some(expected.clone()));
8952
8953 handler
8954 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8955 .await
8956 .unwrap();
8957 let route_response = route_task.await.unwrap();
8958 assert!(route_response.is_empty());
8959 let published = client_rx.recv().await.unwrap();
8960 assert!(matches!(
8961 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8962 ClientControlResponse::RouteOpen { .. }
8963 ));
8964 }
8965
8966 #[tokio::test]
8967 async fn route_open_without_consumer_capabilities_relays_none() {
8968 let registry = Arc::new(Registry::default());
8969 let forwarding = Arc::new(ForwardingTable::default());
8970 let handler =
8971 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8972 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
8973 hello_via_sink(
8974 &handler,
8975 &module_ctx,
8976 &mut module_rx,
8977 hello_frame("aft", PROTOCOL_VERSION, 7),
8978 )
8979 .await;
8980
8981 let project_root = unique_project_root("consumer-capabilities-absent");
8982 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
8983 let route_handler = handler.clone();
8984 let route_task = tokio::spawn(async move {
8985 route_handler
8986 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
8987 .await
8988 .unwrap()
8989 });
8990 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8991 .await
8992 .unwrap()
8993 .unwrap();
8994 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8995 let ModuleControlRequest::RouteBind {
8996 consumer_capabilities,
8997 ..
8998 } = bind
8999 else {
9000 panic!("expected route.bind request, got {bind:?}");
9001 };
9002 assert_eq!(consumer_capabilities, None);
9003
9004 handler
9005 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9006 .await
9007 .unwrap();
9008 let route_response = route_task.await.unwrap();
9009 assert!(route_response.is_empty());
9010 let published = client_rx.recv().await.unwrap();
9011 assert!(matches!(
9012 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9013 ClientControlResponse::RouteOpen { .. }
9014 ));
9015 }
9016
9017 async fn bind_role_versions_for(
9021 sent: Option<BTreeMap<String, String>>,
9022 connection: u64,
9023 ) -> Option<BTreeMap<String, String>> {
9024 let registry = Arc::new(Registry::default());
9025 let forwarding = Arc::new(ForwardingTable::default());
9026 let handler =
9027 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9028 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
9029 hello_via_sink(
9030 &handler,
9031 &module_ctx,
9032 &mut module_rx,
9033 hello_frame("aft", PROTOCOL_VERSION, 7),
9034 )
9035 .await;
9036 let project_root = unique_project_root("role-versions");
9037 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
9038 let route_handler = handler.clone();
9039 let route_task = tokio::spawn(async move {
9040 route_handler
9041 .handle_control_frame(
9042 &client_ctx,
9043 route_open_frame_with_role_versions(403, "aft", project_root, sent),
9044 )
9045 .await
9046 .unwrap()
9047 });
9048 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9049 .await
9050 .expect("a well-formed route.open reaches the module as a bind")
9051 .unwrap();
9052 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9053 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
9054 panic!("expected route.bind request, got {bind:?}");
9055 };
9056 handler
9057 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9058 .await
9059 .unwrap();
9060 assert!(route_task.await.unwrap().is_empty());
9061 let published = client_rx.recv().await.unwrap();
9062 assert!(matches!(
9063 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9064 ClientControlResponse::RouteOpen { .. }
9065 ));
9066 role_versions
9067 }
9068
9069 #[tokio::test]
9070 async fn route_open_relays_role_versions_verbatim() {
9071 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
9072 assert_eq!(
9073 bind_role_versions_for(Some(sent.clone()), 141).await,
9074 Some(sent)
9075 );
9076 }
9077
9078 #[tokio::test]
9081 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
9082 assert_eq!(bind_role_versions_for(None, 143).await, None);
9083 assert_eq!(
9084 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
9085 None
9086 );
9087 }
9088
9089 #[tokio::test]
9090 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
9091 let registry = Arc::new(Registry::default());
9092 let forwarding = Arc::new(ForwardingTable::default());
9093 let handler =
9094 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9095 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
9096 hello_via_sink(
9097 &handler,
9098 &module_ctx,
9099 &mut module_rx,
9100 hello_frame("aft", PROTOCOL_VERSION, 7),
9101 )
9102 .await;
9103
9104 let nine: BTreeMap<String, String> = (0..9)
9105 .map(|index| (format!("role-{index}"), "v1".to_string()))
9106 .collect();
9107 for (label, malformed) in [
9108 (
9109 "invalid role name",
9110 role_versions(&[("Tool_Provider", "v1")]),
9111 ),
9112 ("invalid version", role_versions(&[("tool-provider", "v0")])),
9113 ("nine entries", nine),
9114 ] {
9115 let responses = tokio::time::timeout(
9118 Duration::from_secs(1),
9119 handler.handle_control_frame(
9120 &route_ctx(ConnectionId::new(148)).0,
9121 route_open_frame_with_role_versions(
9122 404,
9123 "aft",
9124 unique_project_root("role-versions-malformed"),
9125 Some(malformed),
9126 ),
9127 ),
9128 )
9129 .await
9130 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
9131 .unwrap();
9132 assert_eq!(responses.len(), 1, "{label}");
9133 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
9134 let error = parse_error(&responses[0]);
9135 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
9136 assert_eq!(
9137 error["detail"]["field"], "role_versions",
9138 "{label}: {error}"
9139 );
9140 assert!(
9141 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
9142 "{label}: a malformed declaration is terminal"
9143 );
9144 assert!(
9145 module_rx.try_recv().is_err(),
9146 "{label}: the module must never see a bind"
9147 );
9148 }
9149 }
9150
9151 #[tokio::test]
9155 async fn route_role_versions_capability_is_advertised() {
9156 let handler = ControlHandler::new(Arc::new(Registry::default()));
9157 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
9158 let ack = hello_via_sink(
9159 &handler,
9160 &ctx,
9161 &mut rx,
9162 hello_frame("m", PROTOCOL_VERSION, 1),
9163 )
9164 .await;
9165 let ack = parse_ack(&ack);
9166 assert!(
9167 ack.subc_capabilities
9168 .iter()
9169 .any(|c| c == "route-role-versions/v1"),
9170 "{:?}",
9171 ack.subc_capabilities
9172 );
9173
9174 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
9175 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
9176 let reply = handler
9177 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
9178 .await
9179 .unwrap()
9180 .pop()
9181 .unwrap();
9182 let ClientControlResponse::ServerDescribe { capabilities, .. } =
9183 serde_json::from_slice(&reply.body).unwrap()
9184 else {
9185 panic!("not a server.describe reply");
9186 };
9187 assert!(
9188 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
9189 "{capabilities:?}"
9190 );
9191 }
9192
9193 #[tokio::test]
9194 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
9195 let registry = Arc::new(Registry::default());
9196 let forwarding = Arc::new(ForwardingTable::default());
9197 let handler =
9198 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9199 .with_health_probe_timeout(Duration::from_secs(5));
9200 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
9201 hello_via_sink(
9202 &handler,
9203 &module_ctx,
9204 &mut module_rx,
9205 non_routable_hello_frame_with_control_ops(
9206 "mcp",
9207 300,
9208 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9209 ),
9210 )
9211 .await;
9212 assert!(registry
9213 .get_module("mcp")
9214 .unwrap()
9215 .unwrap()
9216 .manifest
9217 .provides
9218 .is_empty());
9219
9220 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
9221 let route_response = handler
9222 .handle_control_frame(
9223 &route_client_ctx,
9224 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
9225 )
9226 .await
9227 .unwrap();
9228 assert_eq!(route_response[0].header.ty, FrameType::Error);
9229 assert_eq!(
9230 parse_error(&route_response[0])["code"],
9231 "target_unavailable"
9232 );
9233 assert!(parse_error(&route_response[0])["message"]
9234 .as_str()
9235 .unwrap()
9236 .contains("does not provide the requested target"));
9237 assert!(module_rx.try_recv().is_err());
9238
9239 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
9240 let health_handler = handler.clone();
9241 let health_task = tokio::spawn(async move {
9242 health_handler
9243 .handle_control_frame(
9244 &health_client_ctx,
9245 supervisor_health_probe_frame(302, "mcp"),
9246 )
9247 .await
9248 .unwrap()
9249 });
9250 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9251 .await
9252 .unwrap()
9253 .unwrap();
9254 assert_eq!(
9255 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
9256 ModuleControlRequest::HealthCheck {}
9257 );
9258 handler
9259 .handle_control_frame(
9260 &module_ctx,
9261 health_response(health_frame.header.corr, HealthStatus::Ok),
9262 )
9263 .await
9264 .unwrap();
9265 let health_response = health_task.await.unwrap();
9266 assert_eq!(health_response[0].header.ty, FrameType::Response);
9267 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
9268 ClientControlResponse::SupervisorHealthProbe {
9269 module_id, status, ..
9270 } => {
9271 assert_eq!(module_id, "mcp");
9272 assert_eq!(status, HealthStatus::Ok);
9273 }
9274 other => panic!("unexpected health response: {other:?}"),
9275 }
9276
9277 forwarding
9282 .cleanup_connection(module_ctx.connection_id)
9283 .unwrap();
9284 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
9285 let cleanup_response = tokio::time::timeout(
9286 Duration::from_millis(200),
9287 handler.handle_control_frame(
9288 &cleanup_probe_ctx,
9289 supervisor_health_probe_frame(303, "mcp"),
9290 ),
9291 )
9292 .await
9293 .expect("probe should fail immediately when the control lane is gone")
9294 .unwrap();
9295 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
9296 assert_eq!(
9297 parse_error(&cleanup_response[0])["code"],
9298 "target_unavailable"
9299 );
9300 assert!(parse_error(&cleanup_response[0])["message"]
9301 .as_str()
9302 .unwrap()
9303 .contains("no module connection"));
9304
9305 handler
9306 .cleanup_connection(module_ctx.connection_id)
9307 .unwrap();
9308 }
9309
9310 #[tokio::test]
9311 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
9312 let registry = Arc::new(Registry::default());
9313 let supervisor_handle = SupervisorHandle::new();
9314 let supervisor =
9315 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
9316 .with_handle(supervisor_handle.clone())
9317 .with_connection_file_path(
9318 std::env::temp_dir()
9319 .join(format!("subc-route-open-warming-{}", std::process::id())),
9320 );
9321 let module = supervisor
9322 .supervise_configured(
9323 ModuleSpec {
9324 launch_nonce_env: true,
9325 module_id: "warming".to_string(),
9326 program: fake_aft_stub_path(),
9327 args: Vec::new(),
9328 env: Vec::new(),
9329 reserved: false,
9330 reserved_prefixes: Vec::new(),
9331 protocol: ModuleProtocol::Subc,
9332 overlap: Default::default(),
9333 },
9334 true,
9335 )
9336 .unwrap();
9337 assert_eq!(module.state().unwrap(), ModuleState::Running);
9338
9339 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9340 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
9341 let response = handler
9342 .handle_control_frame(
9343 &ctx,
9344 route_open_frame(304, "warming", unique_project_root("warming")),
9345 )
9346 .await
9347 .unwrap();
9348 module.stop().await.unwrap();
9349
9350 assert_eq!(response[0].header.ty, FrameType::Error);
9351 let error = parse_error(&response[0]);
9352 assert_eq!(error["code"], "module_warming");
9353 assert!(error["message"]
9354 .as_str()
9355 .unwrap()
9356 .contains("state=running, enabled=true, live=false"));
9357 }
9358
9359 #[test]
9360 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
9361 let handler = ControlHandler::new(Arc::new(Registry::default()));
9362 let capture = EventCapture::default();
9363 let _subscriber =
9364 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9365 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
9366 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
9367 let pending = (0..limit).collect::<Vec<_>>();
9368 let response = handler
9369 .route_open_capacity_refusal(
9370 &ctx,
9371 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
9372 "busy",
9373 pending.len(),
9374 limit,
9375 )
9376 .unwrap();
9377 assert_eq!(parse_error(&response)["code"], "target_unavailable");
9378 let event = capture
9379 .events()
9380 .into_iter()
9381 .find(|event| {
9382 event.target == "control"
9383 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
9384 })
9385 .expect("connection admission refusal event");
9386 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
9387 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
9388 }
9389
9390 #[test]
9391 fn route_open_target_cap_logs_admission_reason_and_capacity() {
9392 let handler = ControlHandler::new(Arc::new(Registry::default()));
9393 let capture = EventCapture::default();
9394 let _subscriber =
9395 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9396 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
9397 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
9398 let guards = (0..limit)
9399 .map(|_| {
9400 handler
9401 .route_bind_concurrency
9402 .try_admit("busy", limit)
9403 .unwrap()
9404 })
9405 .collect::<Vec<_>>();
9406 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
9407 Err(in_flight) => in_flight,
9408 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
9409 };
9410 let response = handler
9411 .route_open_target_capacity_refusal(
9412 &ctx,
9413 &route_open_frame(397, "busy", unique_project_root("target-cap")),
9414 "busy",
9415 in_flight,
9416 )
9417 .unwrap();
9418 assert_eq!(parse_error(&response)["code"], "target_unavailable");
9419 let event = capture
9420 .events()
9421 .into_iter()
9422 .find(|event| {
9423 event.target == "control"
9424 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
9425 })
9426 .expect("target admission refusal event");
9427 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
9428 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
9429 drop(guards);
9430 }
9431
9432 #[tokio::test]
9437 async fn route_open_refusal_names_the_check_that_refused() {
9438 let handler = ControlHandler::new(Arc::new(Registry::default()));
9439 let capture = EventCapture::default();
9440 let _subscriber =
9441 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9442 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
9443 let response = handler
9444 .handle_control_frame(
9445 &ctx,
9446 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
9447 )
9448 .await
9449 .unwrap();
9450
9451 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
9452 let event = capture
9453 .events()
9454 .into_iter()
9455 .find(|event| {
9456 event.target == "control"
9457 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
9458 })
9459 .expect("route.open refusal event");
9460 assert_eq!(
9461 event.fields.get("reason"),
9462 Some(&"\"not_registered\"".to_string())
9463 );
9464 }
9465
9466 #[tokio::test]
9467 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
9468 let registry = Arc::new(Registry::default());
9469 let supervisor_handle = SupervisorHandle::new();
9470 let supervisor =
9471 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
9472 .with_handle(supervisor_handle.clone())
9473 .with_connection_file_path(std::env::temp_dir().join(format!(
9474 "subc-route-open-refusal-info-{}",
9475 std::process::id()
9476 )));
9477 let module = supervisor
9478 .supervise_configured(
9479 ModuleSpec {
9480 launch_nonce_env: true,
9481 module_id: "warming".to_string(),
9482 program: fake_aft_stub_path(),
9483 args: Vec::new(),
9484 env: Vec::new(),
9485 reserved: false,
9486 reserved_prefixes: Vec::new(),
9487 protocol: ModuleProtocol::Subc,
9488 overlap: Default::default(),
9489 },
9490 true,
9491 )
9492 .unwrap();
9493 assert_eq!(module.state().unwrap(), ModuleState::Running);
9494
9495 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9496 assert!(handler
9497 .counters()
9498 .snapshot()
9499 .get("route_open_refused_by_code")
9500 .is_none());
9501 let capture = EventCapture::default();
9502 let _subscriber =
9503 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9504 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
9505 let response = handler
9506 .handle_control_frame(
9507 &ctx,
9508 route_open_frame(394, "warming", unique_project_root("refusal-info")),
9509 )
9510 .await
9511 .unwrap();
9512 module.stop().await.unwrap();
9513
9514 assert_eq!(parse_error(&response[0])["code"], "module_warming");
9515 let event = capture
9516 .events()
9517 .into_iter()
9518 .find(|event| {
9519 event.target == "control"
9520 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
9521 })
9522 .expect("route.open refusal event");
9523 assert_eq!(
9524 event.fields.get("module_id"),
9525 Some(&"\"warming\"".to_string())
9526 );
9527 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
9528 assert_eq!(
9529 event.fields.get("reason"),
9530 Some(&"\"supervised_not_registered\"".to_string())
9531 );
9532 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
9533 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
9534 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
9535 assert_eq!(
9536 handler.counters().snapshot()["route_open_refused_by_code"],
9537 json!({ "module_warming": 1 })
9538 );
9539 }
9540
9541 const OUTAGE_START: &str = "route.open refusing module: not serving";
9542 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
9543
9544 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
9545 capture
9546 .events()
9547 .into_iter()
9548 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
9549 .collect()
9550 }
9551
9552 fn supervise_stub(
9553 registry: &Arc<Registry>,
9554 module_id: &str,
9555 enabled: bool,
9556 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
9557 let supervisor_handle = SupervisorHandle::new();
9558 let supervisor =
9559 Supervisor::new(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
9560 .with_handle(supervisor_handle.clone())
9561 .with_connection_file_path(std::env::temp_dir().join(format!(
9562 "subc-route-outage-{module_id}-{}",
9563 std::process::id()
9564 )));
9565 let module = supervisor
9566 .supervise_configured(
9567 ModuleSpec {
9568 launch_nonce_env: true,
9569 module_id: module_id.to_string(),
9570 program: fake_aft_stub_path(),
9571 args: Vec::new(),
9572 env: Vec::new(),
9573 reserved: false,
9574 reserved_prefixes: Vec::new(),
9575 protocol: ModuleProtocol::Subc,
9576 overlap: Default::default(),
9577 },
9578 enabled,
9579 )
9580 .unwrap();
9581 (supervisor_handle, module)
9582 }
9583
9584 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
9585 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
9586 module_id: module_id.to_string(),
9587 drain_timeout_ms: Some(50),
9588 })
9589 .unwrap();
9590 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9591 }
9592
9593 #[test]
9597 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
9598 let registry = Arc::new(Registry::default());
9599 let forwarding = Arc::new(ForwardingTable::default());
9600 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9601 let second = ControlHandler::with_forwarding(registry, forwarding);
9602 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
9603 }
9604
9605 #[tokio::test(flavor = "current_thread")]
9609 async fn route_open_unknown_module_refusals_add_no_outage_state() {
9610 let handler = ControlHandler::new(Arc::new(Registry::default()));
9611 let capture = EventCapture::default();
9612 let _subscriber =
9613 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9614 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
9615 for corr in 0..8 {
9616 let response = handler
9617 .handle_control_frame(
9618 &ctx,
9619 route_open_frame(
9620 380 + corr,
9621 &format!("nobody-{corr}"),
9622 unique_project_root("outage-unknown"),
9623 ),
9624 )
9625 .await
9626 .unwrap();
9627 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
9628 }
9629
9630 assert_eq!(handler.route_outages.tracked_module_count(), 0);
9631 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
9632 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
9633 }
9634
9635 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9640 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
9641 let registry = Arc::new(Registry::default());
9642 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
9643 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9644 let capture = EventCapture::default();
9645 let _subscriber =
9646 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9647 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
9648 handler.route_outages.record_accepted("outage-restart");
9651
9652 let response = handler
9653 .handle_control_frame(
9654 &ctx,
9655 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
9656 )
9657 .await
9658 .unwrap();
9659 assert_eq!(response[0].header.ty, FrameType::Error);
9660 let starts = outage_lines(&capture, OUTAGE_START);
9661 assert_eq!(starts.len(), 1, "{starts:?}");
9662 assert_eq!(starts[0].level, tracing::Level::WARN);
9663 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
9664 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
9665 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
9666 handler.route_outages.record_accepted("outage-restart");
9667 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
9668
9669 let restart = handler
9670 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
9671 .await
9672 .unwrap();
9673 assert_eq!(
9674 restart[0].header.ty,
9675 FrameType::Response,
9676 "{:?}",
9677 parse_error(&restart[0])
9678 );
9679 handler
9680 .handle_control_frame(
9681 &ctx,
9682 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
9683 )
9684 .await
9685 .unwrap();
9686 module.stop().await.unwrap();
9687
9688 let starts = outage_lines(&capture, OUTAGE_START);
9689 assert_eq!(starts.len(), 2, "{starts:?}");
9690 assert_eq!(starts[1].level, tracing::Level::INFO);
9691 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
9692 }
9693
9694 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9698 async fn failed_operator_restart_leaves_no_operator_mark() {
9699 let registry = Arc::new(Registry::default());
9700 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
9701 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9702 let capture = EventCapture::default();
9703 let _subscriber =
9704 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9705 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
9706 handler.route_outages.record_accepted("outage-disabled");
9707
9708 let restart = handler
9709 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
9710 .await
9711 .unwrap();
9712 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
9713 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
9714
9715 handler
9716 .handle_control_frame(
9717 &ctx,
9718 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
9719 )
9720 .await
9721 .unwrap();
9722 let starts = outage_lines(&capture, OUTAGE_START);
9723 assert_eq!(starts.len(), 1, "{starts:?}");
9724 assert_eq!(starts[0].level, tracing::Level::WARN);
9725 }
9726
9727 #[tokio::test(flavor = "current_thread")]
9728 async fn route_open_unknown_module_escapes_target_module_id() {
9729 let handler = ControlHandler::new(Arc::new(Registry::default()));
9730 let capture = EventCapture::default();
9731 let _subscriber =
9732 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9733 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
9734 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
9735 let response = handler
9736 .handle_control_frame(
9737 &ctx,
9738 route_open_frame(
9739 395,
9740 hostile_module_id,
9741 unique_project_root("hostile-target-module-id"),
9742 ),
9743 )
9744 .await
9745 .unwrap();
9746
9747 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
9748 let event = capture
9749 .events()
9750 .into_iter()
9751 .find(|event| {
9752 event.target == "control"
9753 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
9754 })
9755 .expect("route.open unknown-module refusal event");
9756 let logged = event.fields.get("module_id").expect("module_id field");
9757 assert!(!logged.bytes().any(|byte| byte < 0x20));
9758 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
9759 }
9760
9761 #[tokio::test(flavor = "current_thread")]
9762 async fn route_open_module_rejection_uses_daemon_counter_key() {
9763 let registry = Arc::new(Registry::default());
9764 let forwarding = Arc::new(ForwardingTable::default());
9765 let handler =
9766 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9767 let module_connection = ConnectionId::new(95);
9768 let (module_ctx, mut module_rx) = route_ctx(module_connection);
9769 hello_via_sink(
9770 &handler,
9771 &module_ctx,
9772 &mut module_rx,
9773 hello_frame("aft", PROTOCOL_VERSION, 395),
9774 )
9775 .await;
9776
9777 let client_connection = ConnectionId::new(96);
9778 let (client_ctx, _client_rx) = route_ctx(client_connection);
9779 let capture = EventCapture::default();
9780 let _subscriber =
9781 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9782 let (route_task, bind) = relay_route_open(
9783 &handler,
9784 client_connection,
9785 &client_ctx.egress,
9786 &mut module_rx,
9787 396,
9788 "aft",
9789 "hostile-module-code",
9790 )
9791 .await;
9792 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
9793 let rejection = Frame::build(
9794 FrameType::Error,
9795 control_flags(),
9796 0,
9797 0,
9798 bind.header.corr,
9799 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
9800 )
9801 .unwrap();
9802 handler
9803 .handle_control_frame(&module_ctx, rejection)
9804 .await
9805 .unwrap();
9806
9807 let response = route_task.await.unwrap();
9808 assert_eq!(parse_error(&response[0])["code"], hostile_code);
9809 let counters = handler.counters().snapshot();
9810 assert_eq!(
9811 counters["route_open_refused_by_code"],
9812 json!({ "module_rejected": 1 })
9813 );
9814 assert!(counters["route_open_refused_by_code"]
9815 .get(hostile_code)
9816 .is_none());
9817
9818 let event = capture
9819 .events()
9820 .into_iter()
9821 .find(|event| {
9822 event.target == "control"
9823 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
9824 })
9825 .expect("route.open module-rejection refusal event");
9826 let logged = event.fields.get("module_code").expect("module_code field");
9827 assert!(!logged.bytes().any(|byte| byte < 0x20));
9828 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
9829 }
9830
9831 #[tokio::test]
9832 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
9833 let registry = Arc::new(Registry::default());
9834 let supervisor_handle = SupervisorHandle::new();
9835 let missing_program = std::env::temp_dir().join(format!(
9836 "subc-route-open-missing-program-{}",
9837 std::process::id()
9838 ));
9839 let supervisor =
9840 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
9841 .with_handle(supervisor_handle.clone());
9842 let module = supervisor
9843 .supervise_configured(
9844 ModuleSpec {
9845 launch_nonce_env: true,
9846 module_id: "failed".to_string(),
9847 program: missing_program,
9848 args: Vec::new(),
9849 env: Vec::new(),
9850 reserved: false,
9851 reserved_prefixes: Vec::new(),
9852 protocol: ModuleProtocol::Subc,
9853 overlap: Default::default(),
9854 },
9855 true,
9856 )
9857 .unwrap();
9858 assert_eq!(module.state().unwrap(), ModuleState::Failed);
9859
9860 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9861 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
9862 let response = handler
9863 .handle_control_frame(
9864 &ctx,
9865 route_open_frame(305, "failed", unique_project_root("failed")),
9866 )
9867 .await
9868 .unwrap();
9869
9870 assert_eq!(response[0].header.ty, FrameType::Error);
9871 let error = parse_error(&response[0]);
9872 assert_eq!(error["code"], "target_unavailable");
9873 assert!(error["message"]
9874 .as_str()
9875 .unwrap()
9876 .contains("state=failed, enabled=true, live=false"));
9877 }
9878
9879 #[tokio::test]
9880 async fn route_open_role_mismatch_remains_target_unavailable() {
9881 let registry = Arc::new(Registry::default());
9882 let handler = ControlHandler::new(Arc::clone(®istry));
9883 handler
9884 .handle_control(
9885 ConnectionId::new(41),
9886 non_routable_hello_frame_with_control_ops("health-only", 306, None),
9887 )
9888 .unwrap();
9889
9890 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
9891 let response = handler
9892 .handle_control_frame(
9893 &ctx,
9894 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
9895 )
9896 .await
9897 .unwrap();
9898
9899 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
9900 assert!(parse_error(&response[0])["message"]
9901 .as_str()
9902 .unwrap()
9903 .contains("does not provide the requested target"));
9904 }
9905
9906 #[tokio::test]
9907 async fn route_open_inactive_registration_remains_target_unavailable() {
9908 let registry = Arc::new(Registry::default());
9909 let handler = ControlHandler::new(Arc::clone(®istry));
9910 handler
9911 .handle_control(
9912 ConnectionId::new(43),
9913 hello_frame("inactive", PROTOCOL_VERSION, 308),
9914 )
9915 .unwrap();
9916 assert!(registry
9917 .set_module_state_for_test("inactive", ChannelState::Closed)
9918 .unwrap());
9919
9920 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
9921 let response = handler
9922 .handle_control_frame(
9923 &ctx,
9924 route_open_frame(309, "inactive", unique_project_root("inactive")),
9925 )
9926 .await
9927 .unwrap();
9928
9929 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
9930 assert!(parse_error(&response[0])["message"]
9931 .as_str()
9932 .unwrap()
9933 .contains("is not active"));
9934 }
9935
9936 #[tokio::test]
9937 async fn late_health_reply_is_recorded_through_the_module_response_path() {
9938 let registry = Arc::new(Registry::default());
9939 let forwarding = Arc::new(ForwardingTable::default());
9940 let supervisor_handle = SupervisorHandle::new();
9941 let supervisor = Supervisor::new(Arc::clone(®istry), crate::RestartPolicy::default())
9942 .with_forwarding(Arc::clone(&forwarding))
9943 .with_handle(supervisor_handle.clone());
9944 let module = supervisor
9945 .supervise_configured(
9946 crate::ModuleSpec {
9947 launch_nonce_env: true,
9948 module_id: "late-health-response".to_string(),
9949 program: PathBuf::from("disabled-module"),
9950 args: Vec::new(),
9951 env: Vec::new(),
9952 reserved: false,
9953 reserved_prefixes: Vec::new(),
9954 protocol: ModuleProtocol::Subc,
9955 overlap: Default::default(),
9956 },
9957 false,
9958 )
9959 .unwrap();
9960 let handler =
9961 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9962 .with_supervisor(supervisor_handle);
9963 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
9964 handler
9965 .handle_control_frame(
9966 &module_ctx,
9967 hello_frame_with_control_ops(
9968 "late-health-response",
9969 PROTOCOL_VERSION,
9970 7,
9971 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9972 ),
9973 )
9974 .await
9975 .unwrap();
9976 let probe_started_at = Instant::now() - Duration::from_millis(80);
9977 let pending = forwarding
9978 .begin_health_probe_rpc_for(
9979 "late-health-response",
9980 MODULE_CONTROL_OP_HEALTH_CHECK,
9981 probe_started_at,
9982 Instant::now() - Duration::from_millis(1),
9983 )
9984 .unwrap();
9985 assert!(forwarding
9986 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
9987 .unwrap());
9988
9989 let responses = handler
9990 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
9991 .await
9992 .unwrap();
9993
9994 assert!(responses.is_empty());
9995 let health = module.status().unwrap().health;
9996 assert_eq!(health.late_answer_count, 1);
9997 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
9998 }
9999
10000 #[tokio::test]
10001 async fn health_probe_timeout_and_module_death_are_typed() {
10002 let registry = Arc::new(Registry::default());
10003 let forwarding = Arc::new(ForwardingTable::default());
10004 let handler =
10005 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10006 .with_health_probe_timeout(Duration::from_millis(50));
10007 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
10008 hello_via_sink(
10009 &handler,
10010 &module_ctx,
10011 &mut module_rx,
10012 hello_frame_with_control_ops(
10013 "aft",
10014 PROTOCOL_VERSION,
10015 7,
10016 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10017 ),
10018 )
10019 .await;
10020
10021 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10022 let responses = handler
10023 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
10024 .await
10025 .unwrap();
10026 assert_eq!(responses[0].header.ty, FrameType::Error);
10027 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
10028 let _ = module_rx.try_recv();
10029
10030 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
10031 let health_handler = handler.clone();
10032 let death_task = tokio::spawn(async move {
10033 health_handler
10034 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
10035 .await
10036 .unwrap()
10037 });
10038 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10039 .await
10040 .unwrap()
10041 .unwrap();
10042 handler
10043 .cleanup_connection(module_ctx.connection_id)
10044 .unwrap();
10045 let responses = death_task.await.unwrap();
10046 assert_eq!(responses[0].header.ty, FrameType::Error);
10047 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
10048 }
10049
10050 #[test]
10051 fn hello_requires_exact_protocol_version() {
10052 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
10053 let registry = Arc::new(Registry::default());
10054 let handler = ControlHandler::new(Arc::clone(®istry));
10055 let responses = handler
10056 .handle_control(
10057 ConnectionId::new(connection),
10058 hello_frame("aft", offered, 9),
10059 )
10060 .unwrap();
10061
10062 assert_eq!(responses.len(), 1);
10063 assert_eq!(responses[0].header.ty, FrameType::Error);
10064 let error = parse_error(&responses[0]);
10065 assert_eq!(error["code"], "version_unsupported");
10066 assert!(registry.get_module("aft").unwrap().is_none());
10067 assert_eq!(registry.active_registration_count().unwrap(), 0);
10068 }
10069 }
10070
10071 #[test]
10072 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
10073 let registry = Arc::new(Registry::default());
10074 let forwarding = Arc::new(ForwardingTable::default());
10075 let handler =
10076 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10077 let module_connection = ConnectionId::new(301);
10078 let registration = registry
10079 .register_with_control_ops(
10080 manifest("aft-push", PROTOCOL_VERSION),
10081 PROTOCOL_VERSION,
10082 module_connection,
10083 module_baseline_control_ops(),
10084 )
10085 .unwrap();
10086 let (module_tx, _module_rx) = mpsc::channel(8);
10087 let endpoint = forwarding
10088 .register_module_connection(
10089 module_connection,
10090 "aft-push".to_string(),
10091 PROTOCOL_VERSION,
10092 manifest_concurrency(®istration.manifest),
10093 FrameSink::new(module_tx),
10094 )
10095 .unwrap();
10096
10097 let unknown = Frame::build(
10099 FrameType::Push,
10100 control_flags(),
10101 0,
10102 0,
10103 5,
10104 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
10105 )
10106 .unwrap();
10107 let out = handler.handle_status_update(endpoint, unknown).unwrap();
10108 assert!(
10109 out.is_empty(),
10110 "unknown push op must be ignored, got {out:?}"
10111 );
10112
10113 let malformed = Frame::build(
10115 FrameType::Push,
10116 control_flags(),
10117 0,
10118 0,
10119 6,
10120 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
10121 )
10122 .unwrap();
10123 let out = handler.handle_status_update(endpoint, malformed).unwrap();
10124 assert_eq!(out.len(), 1);
10125 assert_eq!(out[0].header.ty, FrameType::Error);
10126 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
10127 }
10128
10129 #[test]
10130 fn hello_rejected_when_connection_already_owns_client_routes() {
10131 let registry = Arc::new(Registry::default());
10132 let forwarding = Arc::new(ForwardingTable::default());
10133 let handler =
10134 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10135 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
10137 let client_connection = ConnectionId::new(202);
10138
10139 let responses = handler
10142 .handle_control(
10143 client_connection,
10144 hello_frame("aft-second", PROTOCOL_VERSION, 9),
10145 )
10146 .unwrap();
10147 assert_eq!(responses[0].header.ty, FrameType::Error);
10148 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
10149 assert!(registry.get_module("aft-second").unwrap().is_none());
10150 }
10151
10152 #[test]
10153 fn reserved_module_hello_requires_matching_launch_nonce() {
10154 let registry = Arc::new(Registry::default());
10155 let supervisor = SupervisorHandle::new();
10156 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
10159 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
10160
10161 let no_nonce = handler
10163 .handle_control(
10164 ConnectionId::new(1),
10165 hello_frame("vault", PROTOCOL_VERSION, 1),
10166 )
10167 .unwrap();
10168 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
10169 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
10170 assert!(registry.get_module("vault").unwrap().is_none());
10171
10172 let wrong = handler
10174 .handle_control(
10175 ConnectionId::new(2),
10176 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
10177 )
10178 .unwrap();
10179 assert_eq!(wrong[0].header.ty, FrameType::Error);
10180 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
10181 assert!(registry.get_module("vault").unwrap().is_none());
10182
10183 let ok = handler
10185 .handle_control(
10186 ConnectionId::new(3),
10187 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
10188 )
10189 .unwrap();
10190 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
10191 assert!(registry.get_module("vault").unwrap().is_some());
10192 }
10193
10194 #[test]
10195 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
10196 let registry = Arc::new(Registry::default());
10197 let supervisor = SupervisorHandle::new();
10198 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
10199 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
10200 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
10201
10202 let squat = handler
10203 .handle_control(
10204 ConnectionId::new(1),
10205 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
10206 )
10207 .unwrap();
10208 assert_eq!(squat[0].header.ty, FrameType::Error);
10209 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
10210 assert!(parse_error(&squat[0])["message"]
10211 .as_str()
10212 .unwrap()
10213 .contains("fed:"));
10214
10215 let accepted_peer = handler
10216 .handle_control(
10217 ConnectionId::new(2),
10218 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
10219 )
10220 .unwrap();
10221 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
10222
10223 let accepted_short = handler
10224 .handle_control(
10225 ConnectionId::new(3),
10226 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
10227 )
10228 .unwrap();
10229 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
10230
10231 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
10232 let response = handler
10233 .handle_control(
10234 ConnectionId::new(conn),
10235 hello_frame(module_id, PROTOCOL_VERSION, conn),
10236 )
10237 .unwrap();
10238 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
10239 }
10240 }
10241
10242 #[test]
10243 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
10244 let registry = Arc::new(Registry::default());
10245 let supervisor = SupervisorHandle::new();
10246 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
10247 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
10248 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
10249 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
10250
10251 let owner_nonce = handler
10252 .handle_control(
10253 ConnectionId::new(1),
10254 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
10255 )
10256 .unwrap();
10257 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
10258 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
10259 assert!(registry.get_module("fed:special").unwrap().is_none());
10260
10261 let exact_nonce = handler
10262 .handle_control(
10263 ConnectionId::new(2),
10264 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
10265 )
10266 .unwrap();
10267 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
10268 assert!(registry.get_module("fed:special").unwrap().is_some());
10269 }
10270
10271 #[test]
10272 fn non_reserved_module_ignores_launch_nonce() {
10273 let registry = Arc::new(Registry::default());
10274 let handler = ControlHandler::new(Arc::clone(®istry));
10277 let no_nonce = handler
10278 .handle_control(
10279 ConnectionId::new(1),
10280 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
10281 )
10282 .unwrap();
10283 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
10284 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
10285
10286 let echoed_nonce = handler
10287 .handle_control(
10288 ConnectionId::new(2),
10289 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
10290 )
10291 .unwrap();
10292 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
10293 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
10294 }
10295
10296 #[test]
10297 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
10298 let handler = ControlHandler::default();
10299 let conn = ConnectionId::new(1);
10300 let malformed = Frame::build(
10301 FrameType::Hello,
10302 control_flags(),
10303 0,
10304 0,
10305 3,
10306 b"{not json".to_vec(),
10307 )
10308 .unwrap();
10309
10310 let error = handler.handle_control(conn, malformed).unwrap();
10311 assert_eq!(error[0].header.ty, FrameType::Error);
10312 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
10313
10314 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
10315 let pong = handler.handle_control(conn, ping).unwrap();
10316 assert_eq!(pong[0].header.ty, FrameType::Pong);
10317 assert_eq!(pong[0].header.corr, 4);
10318 }
10319
10320 #[test]
10321 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
10322 let registry = Arc::new(Registry::default());
10323 let handler = ControlHandler::new(Arc::clone(®istry));
10324
10325 handler
10326 .handle_control(
10327 ConnectionId::new(1),
10328 hello_frame("aft", PROTOCOL_VERSION, 1),
10329 )
10330 .unwrap();
10331 let duplicate = handler
10332 .handle_control(
10333 ConnectionId::new(2),
10334 hello_frame("aft", PROTOCOL_VERSION, 2),
10335 )
10336 .unwrap();
10337
10338 assert_eq!(duplicate[0].header.ty, FrameType::Error);
10339 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
10340 let registration = registry.get_module("aft").unwrap().unwrap();
10341 assert_eq!(registration.connection_id, ConnectionId::new(1));
10342 }
10343
10344 #[test]
10345 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
10346 let registry = Arc::new(Registry::default());
10347 let forwarding = Arc::new(ForwardingTable::default());
10348 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
10349 let handler =
10350 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10351 .with_process_liveness(process_liveness);
10352 let (ctx, route_channel, route_epoch) =
10353 bind_liveness_route(®istry, &forwarding, "aft-dead");
10354 let responses = handler
10355 .handle_route_poll(
10356 &ctx,
10357 route_poll_frame(41, PollKind::Liveness, route_channel),
10358 route_channel,
10359 route_epoch,
10360 PollKind::Liveness,
10361 )
10362 .unwrap();
10363
10364 assert_eq!(responses.len(), 1);
10365 assert_eq!(responses[0].header.ty, FrameType::Response);
10366 assert_route_poll_liveness(&responses[0], false);
10367 }
10368
10369 #[test]
10370 fn liveness_poll_without_process_source_uses_bound_route() {
10371 let registry = Arc::new(Registry::default());
10372 let forwarding = Arc::new(ForwardingTable::default());
10373 let handler =
10374 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10375 let (ctx, route_channel, route_epoch) =
10376 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
10377 let responses = handler
10378 .handle_route_poll(
10379 &ctx,
10380 route_poll_frame(42, PollKind::Liveness, route_channel),
10381 route_channel,
10382 route_epoch,
10383 PollKind::Liveness,
10384 )
10385 .unwrap();
10386
10387 assert_route_poll_liveness(&responses[0], true);
10388 }
10389
10390 #[test]
10391 fn liveness_poll_untracked_process_source_uses_bound_route() {
10392 let registry = Arc::new(Registry::default());
10393 let forwarding = Arc::new(ForwardingTable::default());
10394 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
10395 let handler =
10396 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10397 .with_process_liveness(process_liveness);
10398 let (ctx, route_channel, route_epoch) =
10399 bind_liveness_route(®istry, &forwarding, "aft-untracked");
10400 let responses = handler
10401 .handle_route_poll(
10402 &ctx,
10403 route_poll_frame(43, PollKind::Liveness, route_channel),
10404 route_channel,
10405 route_epoch,
10406 PollKind::Liveness,
10407 )
10408 .unwrap();
10409
10410 assert_route_poll_liveness(&responses[0], true);
10411 }
10412
10413 #[tokio::test]
10414 async fn unknown_op_returns_unknown_control_op() {
10415 let handler = ControlHandler::default();
10416 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
10417 let request = Frame::build(
10418 FrameType::Request,
10419 control_flags(),
10420 0,
10421 0,
10422 55,
10423 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
10424 )
10425 .unwrap();
10426
10427 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
10428
10429 assert_eq!(response.len(), 1);
10430 assert_eq!(response[0].header.ty, FrameType::Error);
10431 assert_eq!(response[0].header.corr, 55);
10432 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
10433 }
10434
10435 #[tokio::test]
10436 async fn supervisor_provenance_rejects_unknown_exact_module() {
10437 let handler = ControlHandler::default();
10438 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
10439 let request = Frame::build(
10440 FrameType::Request,
10441 control_flags(),
10442 0,
10443 0,
10444 57,
10445 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
10446 )
10447 .unwrap();
10448
10449 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
10450
10451 assert_eq!(response.len(), 1);
10452 assert_eq!(response[0].header.ty, FrameType::Error);
10453 assert_eq!(response[0].header.corr, 57);
10454 let error = parse_error(&response[0]);
10455 assert_eq!(error["code"], "unknown_module");
10456 assert_eq!(error["message"], "module_id 'missing' is not supervised");
10457 }
10458
10459 #[test]
10460 fn provenance_probe_override_keeps_handler_tests_deterministic() {
10461 let expected = subc_control::RunningImageAgreement::Unavailable {
10462 reason: subc_control::RunningImageUnavailableReason::HashFailed,
10463 };
10464 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
10465 assert_eq!(handler.provenance_probe_override, Some(expected));
10466 }
10467
10468 #[test]
10469 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
10470 let verdict = reload_verdict(
10471 std::path::Path::new("/bin/new"),
10472 Some(std::path::Path::new("/bin/old")),
10473 subc_control::RunningImageAgreement::Unavailable {
10474 reason: subc_control::RunningImageUnavailableReason::HashFailed,
10475 },
10476 );
10477 assert!(matches!(
10478 verdict.path,
10479 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
10480 if configured == std::path::Path::new("/bin/new")
10481 && spawned_from == std::path::Path::new("/bin/old")
10482 ));
10483 }
10484
10485 #[test]
10486 fn reload_verdict_detects_replaced_image_at_same_path() {
10487 let image = subc_control::RunningImageAgreement::Mismatch {
10488 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
10489 digest: "old".into(),
10490 },
10491 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
10492 digest: "new".into(),
10493 },
10494 };
10495 let verdict = reload_verdict(
10496 std::path::Path::new("/bin/same"),
10497 Some(std::path::Path::new("/bin/same")),
10498 image.clone(),
10499 );
10500 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
10501 assert_eq!(verdict.image, image);
10502 }
10503
10504 #[test]
10505 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
10506 let image = subc_control::RunningImageAgreement::Unavailable {
10507 reason: subc_control::RunningImageUnavailableReason::NotRunning,
10508 };
10509 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
10510 assert_eq!(
10511 verdict.path,
10512 subc_control::ReloadPathAgreement::Unavailable {
10513 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
10514 }
10515 );
10516 assert_eq!(verdict.image, image);
10517
10518 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
10519 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
10520 };
10521 let verdict = reload_verdict(
10522 std::path::Path::new("/bin/same"),
10523 Some(std::path::Path::new("/bin/same")),
10524 unconfirmed.clone(),
10525 );
10526 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
10527 assert_eq!(verdict.image, unconfirmed);
10528 }
10529
10530 #[test]
10531 fn reload_verdict_preserves_each_image_unavailability_reason() {
10532 use subc_control::RunningImageUnavailableReason as Reason;
10533
10534 for reason in [
10535 Reason::NotRunning,
10536 Reason::UnsupportedPlatform,
10537 Reason::RunningExecutableUnreadable,
10538 Reason::SpawnedPathUnreadable,
10539 Reason::HashFailed,
10540 Reason::ProcessIdentityUnconfirmed,
10541 Reason::Unknown("future_probe_reason".to_string()),
10542 ] {
10543 let image = subc_control::RunningImageAgreement::Unavailable {
10544 reason: reason.clone(),
10545 };
10546 let verdict = reload_verdict(
10547 std::path::Path::new("/bin/same"),
10548 Some(std::path::Path::new("/bin/same")),
10549 image.clone(),
10550 );
10551 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
10552 assert_eq!(verdict.image, image, "{reason:?}");
10553 }
10554 }
10555
10556 #[tokio::test]
10557 async fn malformed_control_bodies_return_invalid_control_body() {
10558 let handler = ControlHandler::default();
10559 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
10560
10561 for (corr, body) in [
10562 (56, br#"{"route_channel":1}"#.as_slice()),
10563 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
10564 (
10565 58,
10566 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
10567 ),
10568 ] {
10569 let request = Frame::build(
10570 FrameType::Request,
10571 control_flags(),
10572 0,
10573 0,
10574 corr,
10575 body.to_vec(),
10576 )
10577 .unwrap();
10578 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
10579
10580 assert_eq!(response.len(), 1);
10581 assert_eq!(response[0].header.ty, FrameType::Error);
10582 assert_eq!(response[0].header.corr, corr);
10583 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
10584 }
10585 }
10586
10587 #[tokio::test]
10588 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
10589 let registry = Arc::new(Registry::default());
10590 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
10591 let router = Router::with_control_handler(Arc::clone(&control));
10592 let connection = router.begin_connection();
10593 let (ctx, mut rx) = route_ctx(connection.id());
10594
10595 router
10596 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
10597 .await
10598 .unwrap();
10599 let response = rx.recv().await.unwrap();
10600 let ack = parse_ack(&response);
10601 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
10602 let channel = 1;
10603
10604 let goodbye =
10605 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
10606 router.route_for_connection(&ctx, goodbye).await.unwrap();
10607 assert!(rx.try_recv().is_err());
10608 assert!(registry.get_module("aft").unwrap().is_none());
10609
10610 router
10611 .route_for_connection(&ctx, channel_request(channel, 13))
10612 .await
10613 .unwrap();
10614 let error_frame = rx.recv().await.unwrap();
10615 assert_eq!(error_frame.header.ty, FrameType::Error);
10616 assert_eq!(error_frame.header.channel, channel);
10617 }
10618
10619 #[tokio::test]
10620 async fn dropping_router_connection_releases_registration() {
10621 let registry = Arc::new(Registry::default());
10622 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
10623 let router = Router::with_control_handler(control);
10624 let connection = router.begin_connection();
10625 let (ctx, mut rx) = route_ctx(connection.id());
10626
10627 router
10628 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
10629 .await
10630 .unwrap();
10631 let response = rx.recv().await.unwrap();
10632 let ack = parse_ack(&response);
10633 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
10634 assert!(registry.get_module("aft").unwrap().is_some());
10635
10636 drop(connection);
10637
10638 assert!(registry.get_module("aft").unwrap().is_none());
10639 assert_eq!(registry.active_registration_count().unwrap(), 0);
10640 }
10641
10642 fn capability_manifest(
10643 module_id: &str,
10644 provides: &[&str],
10645 must_never_reach: &[&str],
10646 ) -> ModuleManifest {
10647 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
10648 manifest.capabilities = Some(CapabilityDeclarations {
10649 provides: provides
10650 .iter()
10651 .map(|capability| (*capability).to_string())
10652 .collect(),
10653 requires: Vec::new(),
10654 must_never_reach: must_never_reach
10655 .iter()
10656 .map(|capability| (*capability).to_string())
10657 .collect(),
10658 });
10659 manifest
10660 }
10661
10662 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
10663 Frame::build(
10664 FrameType::Hello,
10665 control_flags(),
10666 0,
10667 0,
10668 corr,
10669 serde_json::to_vec(&ModuleHelloBody {
10670 protocol_ver: manifest.protocol_ver,
10671 manifest,
10672 control_ops: None,
10673 launch_nonce: None,
10674 })
10675 .expect("capability test HELLO serializes"),
10676 )
10677 .expect("capability test HELLO frame builds")
10678 }
10679
10680 fn catalog_update_with_capabilities_frame(
10681 corr: u64,
10682 capabilities: CapabilityDeclarations,
10683 ) -> Frame {
10684 Frame::build(
10685 FrameType::Request,
10686 control_flags(),
10687 0,
10688 0,
10689 corr,
10690 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
10691 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
10692 capabilities: Some(capabilities),
10693 ready: None,
10694 })
10695 .expect("capability catalog.update serializes"),
10696 )
10697 .expect("capability catalog.update frame builds")
10698 }
10699
10700 async fn register_capability_manifest(
10701 handler: &ControlHandler,
10702 ctx: &RouteCtx,
10703 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
10704 manifest: ModuleManifest,
10705 corr: u64,
10706 ) {
10707 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
10708 }
10709
10710 async fn open_route_for_capability_test(
10711 handler: &ControlHandler,
10712 target_ctx: &RouteCtx,
10713 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
10714 client_connection_id: u64,
10715 corr: u64,
10716 target_module_id: &str,
10717 consumer_identity: Option<ConsumerIdentity>,
10718 ) -> (
10719 mpsc::Receiver<crate::router::OutboundFrame>,
10720 ModuleControlRequest,
10721 ) {
10722 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
10723 let route_handler = handler.clone();
10724 let target_module_id = target_module_id.to_string();
10725 let route_task = tokio::spawn(async move {
10726 route_handler
10727 .handle_control_frame(
10728 &client_ctx,
10729 route_open_frame_with_admission_facts(
10730 corr,
10731 &target_module_id,
10732 unique_project_root("admission-facts"),
10733 consumer_identity,
10734 None,
10735 ),
10736 )
10737 .await
10738 .expect("capability test route.open succeeds")
10739 });
10740 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
10741 .await
10742 .expect("capability test route.open must reach route.bind")
10743 .expect("target control receiver stays open");
10744 let bind_request: ModuleControlRequest =
10745 serde_json::from_slice(&bind.body).expect("route.bind decodes");
10746 handler
10747 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
10748 .await
10749 .expect("capability test route.bind ACK succeeds");
10750 assert!(route_task.await.expect("route.open task joins").is_empty());
10751 let opened = client_rx
10752 .recv()
10753 .await
10754 .expect("successful route.open publishes a response");
10755 assert!(matches!(
10756 serde_json::from_slice::<ClientControlResponse>(&opened.body),
10757 Ok(ClientControlResponse::RouteOpen { .. })
10758 ));
10759 (client_rx, bind_request)
10760 }
10761
10762 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
10763 assert_eq!(frame.header.ty, FrameType::Push);
10764 assert_eq!(frame.header.channel, 0);
10765 assert_eq!(
10766 serde_json::from_slice::<ClientControlPush>(&frame.body)
10767 .expect("route.closed control push decodes"),
10768 ClientControlPush::RouteClosed {
10769 module_id: target_module_id.to_string(),
10770 reason: RouteCloseReason::CapabilityDenied,
10771 drained: false,
10772 abandoned: 0,
10773 excluded_subscriptions: 0,
10774 terminal: Some(false),
10775 }
10776 );
10777 }
10778
10779 #[tokio::test]
10780 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
10781 let registry = Arc::new(Registry::default());
10782 let forwarding = Arc::new(ForwardingTable::default());
10783 let supervisor = SupervisorHandle::new();
10784 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
10785 let handler =
10786 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10787 .with_supervisor(supervisor);
10788 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
10789 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
10790 register_capability_manifest(
10791 &handler,
10792 &target_ctx,
10793 &mut target_rx,
10794 capability_manifest("target", &["credentials-provider/v1"], &[]),
10795 1,
10796 )
10797 .await;
10798 register_capability_manifest(
10799 &handler,
10800 &opener_ctx,
10801 &mut opener_rx,
10802 capability_manifest("opener", &[], &["credentials-provider/v1"]),
10803 2,
10804 )
10805 .await;
10806
10807 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
10808 let replies = handler
10809 .handle_control_frame(
10810 &client_ctx,
10811 route_open_frame_with_admission_facts(
10812 3,
10813 "target",
10814 unique_project_root("admission-facts"),
10815 Some(ConsumerIdentity {
10816 module_id: "opener".to_string(),
10817 launch_nonce: "opener-nonce".to_string(),
10818 }),
10819 None,
10820 ),
10821 )
10822 .await
10823 .expect("denied route.open returns a typed frame");
10824 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
10825 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
10826 assert!(
10827 target_rx.try_recv().is_err(),
10828 "forbidden route.open must not relay route.bind"
10829 );
10830 }
10831
10832 #[tokio::test]
10833 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
10834 let registry = Arc::new(Registry::default());
10835 let forwarding = Arc::new(ForwardingTable::default());
10836 let supervisor = SupervisorHandle::new();
10837 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
10838 let handler =
10839 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10840 .with_supervisor(supervisor);
10841 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
10842 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
10843 register_capability_manifest(
10844 &handler,
10845 &target_ctx,
10846 &mut target_rx,
10847 capability_manifest("target", &["credentials-provider/v1"], &[]),
10848 1,
10849 )
10850 .await;
10851 register_capability_manifest(
10852 &handler,
10853 &old_opener_ctx,
10854 &mut old_opener_rx,
10855 capability_manifest("opener", &[], &[]),
10856 2,
10857 )
10858 .await;
10859 let (mut client_rx, _) = open_route_for_capability_test(
10860 &handler,
10861 &target_ctx,
10862 &mut target_rx,
10863 712,
10864 3,
10865 "target",
10866 Some(ConsumerIdentity {
10867 module_id: "opener".to_string(),
10868 launch_nonce: "opener-nonce".to_string(),
10869 }),
10870 )
10871 .await;
10872 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
10873
10874 handler
10875 .cleanup_connection(old_opener_ctx.connection_id)
10876 .expect("old opener registration cleans up");
10877 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
10878 register_capability_manifest(
10879 &handler,
10880 &new_opener_ctx,
10881 &mut new_opener_rx,
10882 capability_manifest("opener", &[], &["credentials-provider/v1"]),
10883 4,
10884 )
10885 .await;
10886
10887 assert_capability_denied_push(
10888 client_rx
10889 .try_recv()
10890 .expect("HELLO deny addition must emit route.closed")
10891 .frame,
10892 "target",
10893 );
10894 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
10895 assert!(matches!(
10896 target_rx.try_recv(),
10897 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
10898 ));
10899 }
10900
10901 #[tokio::test]
10902 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
10903 let registry = Arc::new(Registry::default());
10904 let forwarding = Arc::new(ForwardingTable::default());
10905 let supervisor = SupervisorHandle::new();
10906 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
10907 let handler =
10908 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10909 .with_supervisor(supervisor);
10910 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
10911 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
10912 register_capability_manifest(
10913 &handler,
10914 &target_ctx,
10915 &mut target_rx,
10916 capability_manifest("target", &[], &[]),
10917 1,
10918 )
10919 .await;
10920 register_capability_manifest(
10921 &handler,
10922 &opener_ctx,
10923 &mut opener_rx,
10924 capability_manifest("opener", &[], &["credentials-provider/v1"]),
10925 2,
10926 )
10927 .await;
10928 let (mut client_rx, _) = open_route_for_capability_test(
10929 &handler,
10930 &target_ctx,
10931 &mut target_rx,
10932 722,
10933 3,
10934 "target",
10935 Some(ConsumerIdentity {
10936 module_id: "opener".to_string(),
10937 launch_nonce: "opener-nonce".to_string(),
10938 }),
10939 )
10940 .await;
10941 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
10942
10943 let replies = handler
10944 .handle_control_frame(
10945 &target_ctx,
10946 catalog_update_with_capabilities_frame(
10947 4,
10948 CapabilityDeclarations {
10949 provides: vec!["credentials-provider/v1".to_string()],
10950 requires: Vec::new(),
10951 must_never_reach: Vec::new(),
10952 },
10953 ),
10954 )
10955 .await
10956 .expect("claim catalog.update succeeds");
10957 assert!(matches!(
10958 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
10959 Ok(ModuleControlResponseToModule::CatalogUpdate {})
10960 ));
10961 assert_capability_denied_push(
10962 client_rx
10963 .try_recv()
10964 .expect("claim addition must emit route.closed")
10965 .frame,
10966 "target",
10967 );
10968 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
10969 assert!(matches!(
10970 target_rx.try_recv(),
10971 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
10972 ));
10973 }
10974
10975 #[tokio::test]
10976 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
10977 let registry = Arc::new(Registry::default());
10978 let forwarding = Arc::new(ForwardingTable::default());
10979 let supervisor = SupervisorHandle::new();
10980 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
10981 let handler =
10982 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10983 .with_supervisor(supervisor);
10984 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
10985 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
10986 register_capability_manifest(
10987 &handler,
10988 &target_ctx,
10989 &mut target_rx,
10990 capability_manifest("target", &["credentials-provider/v1"], &[]),
10991 1,
10992 )
10993 .await;
10994 register_capability_manifest(
10995 &handler,
10996 &opener_ctx,
10997 &mut opener_rx,
10998 capability_manifest("opener", &[], &[]),
10999 2,
11000 )
11001 .await;
11002 let (mut client_rx, _) = open_route_for_capability_test(
11003 &handler,
11004 &target_ctx,
11005 &mut target_rx,
11006 732,
11007 3,
11008 "target",
11009 Some(ConsumerIdentity {
11010 module_id: "opener".to_string(),
11011 launch_nonce: "opener-nonce".to_string(),
11012 }),
11013 )
11014 .await;
11015 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
11016
11017 handler
11018 .handle_control_frame(
11019 &target_ctx,
11020 catalog_update_with_capabilities_frame(
11021 4,
11022 CapabilityDeclarations {
11023 provides: Vec::new(),
11024 requires: Vec::new(),
11025 must_never_reach: Vec::new(),
11026 },
11027 ),
11028 )
11029 .await
11030 .expect("claim removal catalog.update succeeds");
11031 assert_eq!(
11032 forwarding.active_binding_count().unwrap(),
11033 1,
11034 "removing an attested target claim must leave the route census unchanged"
11035 );
11036 assert!(
11037 client_rx.try_recv().is_err(),
11038 "claim removal must not emit route.closed capability_denied"
11039 );
11040 assert!(
11041 target_rx.try_recv().is_err(),
11042 "claim removal must not send the target a route GOODBYE"
11043 );
11044 }
11045
11046 #[tokio::test]
11049 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
11050 let registry = Arc::new(Registry::default());
11051 let forwarding = Arc::new(ForwardingTable::default());
11052 let supervisor = SupervisorHandle::new();
11053 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11054 let handler =
11055 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11056 .with_supervisor(supervisor);
11057 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
11058 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
11059 register_capability_manifest(
11060 &handler,
11061 &target_ctx,
11062 &mut target_rx,
11063 capability_manifest("target", &["credentials-provider/v1"], &[]),
11064 1,
11065 )
11066 .await;
11067 register_capability_manifest(
11068 &handler,
11069 &opener_ctx,
11070 &mut opener_rx,
11071 capability_manifest("opener", &[], &["credentials-provider/v1"]),
11072 2,
11073 )
11074 .await;
11075
11076 let (_client_rx, bind) = open_route_for_capability_test(
11077 &handler,
11078 &target_ctx,
11079 &mut target_rx,
11080 742,
11081 3,
11082 "target",
11083 None,
11084 )
11085 .await;
11086 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
11087 panic!("direct scope-honesty route must bind");
11088 };
11089 assert_eq!(principal, Some(Principal::Direct));
11090 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
11091 }
11092
11093 #[tokio::test]
11096 async fn must_never_reach_self_route_is_capability_forbidden() {
11097 let registry = Arc::new(Registry::default());
11098 let forwarding = Arc::new(ForwardingTable::default());
11099 let supervisor = SupervisorHandle::new();
11100 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
11101 let handler =
11102 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11103 .with_supervisor(supervisor);
11104 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
11105 register_capability_manifest(
11106 &handler,
11107 &self_ctx,
11108 &mut self_rx,
11109 capability_manifest(
11110 "self-provider",
11111 &["credentials-provider/v1"],
11112 &["credentials-provider/v1"],
11113 ),
11114 1,
11115 )
11116 .await;
11117
11118 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
11119 let replies = handler
11120 .handle_control_frame(
11121 &client_ctx,
11122 route_open_frame_with_admission_facts(
11123 2,
11124 "self-provider",
11125 unique_project_root("admission-facts"),
11126 Some(ConsumerIdentity {
11127 module_id: "self-provider".to_string(),
11128 launch_nonce: "self-nonce".to_string(),
11129 }),
11130 None,
11131 ),
11132 )
11133 .await
11134 .expect("self-route refusal returns a typed frame");
11135 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
11136 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
11137 assert!(
11138 self_rx.try_recv().is_err(),
11139 "self denial must not relay route.bind"
11140 );
11141 }
11142
11143 #[test]
11144 fn unsupported_channel_zero_frame_returns_error() {
11145 let handler = ControlHandler::default();
11146 let request = Frame::build(
11147 FrameType::Request,
11148 control_flags(),
11149 0,
11150 0,
11151 21,
11152 b"opaque".to_vec(),
11153 )
11154 .unwrap();
11155
11156 let response = handler
11157 .handle_control(ConnectionId::new(1), request)
11158 .unwrap();
11159
11160 assert_eq!(response[0].header.ty, FrameType::Error);
11161 assert_eq!(
11162 parse_error(&response[0])["code"],
11163 "unsupported_control_frame"
11164 );
11165 }
11166
11167 mod swap {
11172 use super::*;
11173
11174 const INCUMBENT: ConnectionId = ConnectionId::new(30);
11175 const CANDIDATE: ConnectionId = ConnectionId::new(40);
11176
11177 struct Swap {
11178 registry: Arc<Registry>,
11179 forwarding: Arc<ForwardingTable>,
11180 handler: ControlHandler,
11181 incumbent_ctx: RouteCtx,
11182 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
11183 candidate_ctx: RouteCtx,
11184 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
11185 }
11186
11187 async fn swap_with_incumbent() -> Swap {
11188 let registry = Arc::new(Registry::default());
11189 let forwarding = Arc::new(ForwardingTable::default());
11190 let handler =
11191 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11192 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
11193 hello_via_sink(
11194 &handler,
11195 &incumbent_ctx,
11196 &mut incumbent_rx,
11197 hello_frame("aft", PROTOCOL_VERSION, 7),
11198 )
11199 .await;
11200 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
11201 Swap {
11202 registry,
11203 forwarding,
11204 handler,
11205 incumbent_ctx,
11206 incumbent_rx,
11207 candidate_ctx,
11208 candidate_rx,
11209 }
11210 }
11211
11212 fn register_candidate(swap: &Swap, ready: Option<bool>) {
11213 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
11214 candidate_manifest.ready = ready;
11215 let registration = swap
11216 .registry
11217 .register_candidate_with_control_ops(
11218 candidate_manifest,
11219 PROTOCOL_VERSION,
11220 CANDIDATE,
11221 module_baseline_control_ops(),
11222 )
11223 .unwrap();
11224 swap.forwarding
11225 .register_candidate_module_connection(
11226 CANDIDATE,
11227 "aft".to_string(),
11228 PROTOCOL_VERSION,
11229 manifest_concurrency(®istration.manifest),
11230 swap.candidate_ctx.egress.clone(),
11231 )
11232 .unwrap();
11233 }
11234
11235 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
11236 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
11237 swap.registry.promote_candidate("aft").unwrap().unwrap();
11238 cutover.incumbent.unwrap()
11239 }
11240
11241 fn keyed_total(counters: &Value, key: &str) -> u64 {
11242 counters[key]
11243 .as_object()
11244 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
11245 .unwrap_or(0)
11246 }
11247
11248 #[tokio::test]
11254 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
11255 let mut swap = swap_with_incumbent().await;
11256 let handler = swap.handler.clone();
11257
11258 let cotenant = ConnectionId::new(31);
11260 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
11261 let (cotenant_task, cotenant_bind) = relay_route_open(
11262 &handler,
11263 cotenant,
11264 &cotenant_ctx.egress,
11265 &mut swap.incumbent_rx,
11266 100,
11267 "aft",
11268 "swap-cotenant",
11269 )
11270 .await;
11271 handler
11272 .handle_control_frame(
11273 &swap.incumbent_ctx,
11274 route_bind_ack(cotenant_bind.header.corr),
11275 )
11276 .await
11277 .unwrap();
11278 assert!(cotenant_task.await.unwrap().is_empty());
11279 let (cotenant_channel, cotenant_epoch) =
11280 published_route(&cotenant_rx.recv().await.unwrap());
11281
11282 let caller = ConnectionId::new(32);
11284 let (caller_ctx, mut caller_rx) = route_ctx(caller);
11285 let (caller_task, caller_bind) = relay_route_open(
11286 &handler,
11287 caller,
11288 &caller_ctx.egress,
11289 &mut swap.incumbent_rx,
11290 101,
11291 "aft",
11292 "swap-caller",
11293 )
11294 .await;
11295 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
11296
11297 register_candidate(&swap, None);
11298 cutover(&swap);
11299
11300 let ack = handler
11302 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
11303 .await;
11304 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
11305 if module_loop_error.is_some() {
11306 handler.cleanup_connection(INCUMBENT).unwrap();
11309 }
11310
11311 assert!(
11313 cotenant_rx.try_recv().is_err(),
11314 "the co-tenant route on the incumbent was torn down by one late ack: \
11315 {module_loop_error:?}"
11316 );
11317 assert!(matches!(
11318 swap.forwarding
11319 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
11320 .unwrap(),
11321 DataRoute::Client(DataRouteState::Bound(_))
11322 ));
11323 assert_eq!(module_loop_error, None);
11324 assert!(swap
11325 .registry
11326 .get_module_by_connection(INCUMBENT)
11327 .unwrap()
11328 .is_some());
11329
11330 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
11332 .await
11333 .expect("the incumbent is told to drop the abandoned binding")
11334 .unwrap()
11335 .frame;
11336 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11337 assert_eq!(goodbye.header.channel, abandoned_channel);
11338 assert_eq!(goodbye.header.epoch, abandoned_epoch);
11339 assert!(swap.incumbent_rx.try_recv().is_err());
11340
11341 let response = caller_task.await.unwrap();
11343 assert_eq!(response.len(), 1);
11344 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
11345 assert!(caller_rx.try_recv().is_err());
11346
11347 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
11351 let counters = handler.counters().snapshot();
11352 assert_eq!(
11353 keyed_total(&counters, "route_open_accepted_by_principal"),
11354 1
11355 );
11356 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
11357 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
11358 }
11359
11360 #[tokio::test]
11364 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
11365 let mut swap = swap_with_incumbent().await;
11366 register_candidate(&swap, None);
11367 let incumbent = cutover(&swap);
11368 swap.forwarding
11369 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
11370 .unwrap()
11371 .expect("the incumbent is still registered");
11372
11373 let client = ConnectionId::new(33);
11374 let (client_ctx, mut client_rx) = route_ctx(client);
11375 let route_handler = swap.handler.clone();
11376 let open_ctx = RouteCtx {
11377 connection_id: client,
11378 egress: client_ctx.egress.clone(),
11379 };
11380 let mut route_task = tokio::spawn(async move {
11381 route_handler
11382 .handle_control_frame(
11383 &open_ctx,
11384 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
11385 )
11386 .await
11387 .unwrap()
11388 });
11389 let bind = tokio::select! {
11390 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
11391 response = &mut route_task => {
11392 let response = response.unwrap();
11393 panic!(
11394 "post-cutover route.open was refused instead of relayed to the candidate: {}",
11395 parse_error(&response[0])["code"]
11396 );
11397 }
11398 };
11399 swap.handler
11400 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
11401 .await
11402 .unwrap();
11403 assert!(route_task.await.unwrap().is_empty());
11404 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
11405 match swap
11406 .forwarding
11407 .lookup_data_route(client, channel, epoch)
11408 .unwrap()
11409 {
11410 DataRoute::Client(DataRouteState::Bound(route)) => {
11411 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
11412 }
11413 other => panic!("expected a bound route on the candidate, got {other:?}"),
11414 }
11415 assert!(swap.incumbent_rx.try_recv().is_err());
11416 }
11417
11418 #[tokio::test]
11423 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
11424 let swap = swap_with_incumbent().await;
11425 register_candidate(&swap, Some(false));
11426 let update = Frame::build(
11427 FrameType::Request,
11428 control_flags(),
11429 0,
11430 0,
11431 55,
11432 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
11433 provides: manifest("aft", PROTOCOL_VERSION).provides,
11434 capabilities: None,
11435 ready: Some(true),
11436 })
11437 .unwrap(),
11438 )
11439 .unwrap();
11440
11441 let replies = swap
11442 .handler
11443 .handle_control_frame(&swap.candidate_ctx, update)
11444 .await
11445 .unwrap();
11446
11447 assert_eq!(replies.len(), 1);
11448 assert_eq!(
11449 replies[0].header.ty,
11450 FrameType::Response,
11451 "candidate catalog.update was refused: {:?}",
11452 serde_json::from_slice::<Value>(&replies[0].body).ok()
11453 );
11454 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
11455 assert_eq!(
11456 swap.registry
11457 .get_module("aft")
11458 .unwrap()
11459 .unwrap()
11460 .connection_id,
11461 INCUMBENT
11462 );
11463 }
11464 }
11465
11466 mod swap_admission {
11470 use super::*;
11471
11472 const INCUMBENT_NONCE: &str = "incumbent-nonce";
11473 const CANDIDATE_NONCE: &str = "candidate-nonce";
11474
11475 fn handler_with_incumbent(
11476 module_id: &str,
11477 reserved: bool,
11478 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
11479 let registry = Arc::new(Registry::default());
11480 let supervisor = SupervisorHandle::new();
11481 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
11482 if reserved {
11483 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
11484 }
11485 let handler =
11486 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
11487 let incumbent = handler
11488 .handle_control(
11489 ConnectionId::new(1),
11490 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
11491 )
11492 .unwrap();
11493 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
11494 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
11495 (registry, supervisor, handler)
11496 }
11497
11498 #[test]
11504 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
11505 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
11506
11507 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
11508 let replies = handler
11509 .handle_control(
11510 ConnectionId::new(connection),
11511 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
11512 )
11513 .unwrap();
11514 assert_eq!(replies[0].header.ty, FrameType::Error);
11515 assert_eq!(
11516 parse_error(&replies[0])["code"],
11517 "swap_token_invalid",
11518 "nonce {nonce:?}"
11519 );
11520 }
11521 assert!(registry.get_candidate("aft").unwrap().is_none());
11522 assert_eq!(
11523 registry.get_module("aft").unwrap().unwrap().connection_id,
11524 ConnectionId::new(1)
11525 );
11526
11527 let admitted = handler
11530 .handle_control(
11531 ConnectionId::new(4),
11532 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
11533 )
11534 .unwrap();
11535 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
11536 assert_eq!(
11537 registry
11538 .get_candidate("aft")
11539 .unwrap()
11540 .unwrap()
11541 .connection_id,
11542 ConnectionId::new(4)
11543 );
11544 assert_eq!(
11545 registry.get_module("aft").unwrap().unwrap().connection_id,
11546 ConnectionId::new(1),
11547 "the candidate must not take the active slot"
11548 );
11549 let replayed = handler
11550 .handle_control(
11551 ConnectionId::new(5),
11552 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
11553 )
11554 .unwrap();
11555 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
11556
11557 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
11561 let squatter = handler
11562 .handle_control(
11563 ConnectionId::new(6),
11564 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
11565 )
11566 .unwrap();
11567 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
11568 assert!(
11569 registry.get_module("aft").unwrap().is_none(),
11570 "a squatter took the active slot of an id being swapped"
11571 );
11572 }
11573
11574 #[test]
11579 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
11580 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
11581
11582 let replies = handler
11583 .handle_control(
11584 ConnectionId::new(2),
11585 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
11586 )
11587 .unwrap();
11588
11589 assert_eq!(
11590 replies[0].header.ty,
11591 FrameType::HelloAck,
11592 "reserved candidate refused: {:?}",
11593 serde_json::from_slice::<Value>(&replies[0].body).ok()
11594 );
11595 assert_eq!(
11596 registry
11597 .get_candidate("vault")
11598 .unwrap()
11599 .unwrap()
11600 .connection_id,
11601 ConnectionId::new(2)
11602 );
11603 }
11604
11605 #[test]
11608 fn without_an_open_swap_the_ordinary_gates_decide() {
11609 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
11610 supervisor.close_swap("vault");
11611
11612 let candidate = handler
11613 .handle_control(
11614 ConnectionId::new(2),
11615 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
11616 )
11617 .unwrap();
11618 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
11619 let duplicate = handler
11620 .handle_control(
11621 ConnectionId::new(3),
11622 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
11623 )
11624 .unwrap();
11625 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
11626 assert!(registry.get_candidate("vault").unwrap().is_none());
11627 }
11628 }
11629
11630 mod scopes {
11634 use subc_protocol::scope::{
11635 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
11636 ScopeStatus,
11637 };
11638
11639 use super::*;
11640
11641 const OWNER: &str = "prefrontal-core";
11642
11643 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
11644 ScopeRecord {
11645 scope_ref: scope_ref.to_string(),
11646 scope_epoch,
11647 kind: ScopeKind::Head,
11648 parent: None,
11649 child_owners: Vec::new(),
11650 carriers: Vec::new(),
11651 attributes: Default::default(),
11652 }
11653 }
11654
11655 async fn call(
11656 handler: &ControlHandler,
11657 ctx: &RouteCtx,
11658 request: &ModuleControlRequestFromModule,
11659 ) -> Frame {
11660 let body = serde_json::to_vec(request).unwrap();
11661 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
11662 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
11663 assert_eq!(replies.len(), 1, "{replies:?}");
11664 replies.pop().unwrap()
11665 }
11666
11667 async fn sync(
11668 handler: &ControlHandler,
11669 ctx: &RouteCtx,
11670 generation: u64,
11671 scopes: Vec<ScopeRecord>,
11672 ) -> Result<ModuleControlResponseToModule, String> {
11673 let reply = call(
11674 handler,
11675 ctx,
11676 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
11677 )
11678 .await;
11679 match reply.header.ty {
11680 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
11681 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
11682 }
11683 }
11684
11685 async fn describe(
11686 handler: &ControlHandler,
11687 ctx: &RouteCtx,
11688 owner: &str,
11689 scope_ref: &str,
11690 ) -> ModuleControlResponseToModule {
11691 let reply = call(
11692 handler,
11693 ctx,
11694 &ModuleControlRequestFromModule::ScopeDescribe {
11695 owner: Principal::Reserved {
11696 module_id: owner.to_string(),
11697 },
11698 scope_ref: scope_ref.to_string(),
11699 },
11700 )
11701 .await;
11702 assert_eq!(
11703 reply.header.ty,
11704 FrameType::Response,
11705 "{:?}",
11706 parse_error(&reply)
11707 );
11708 serde_json::from_slice(&reply.body).unwrap()
11709 }
11710
11711 async fn module(
11713 handler: &ControlHandler,
11714 connection: u64,
11715 module_id: &str,
11716 nonce: Option<&str>,
11717 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
11718 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
11719 hello_via_sink(
11720 handler,
11721 &ctx,
11722 &mut rx,
11723 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
11724 )
11725 .await;
11726 (ctx, rx)
11727 }
11728
11729 #[tokio::test]
11732 async fn a_client_connection_cannot_sync_or_describe() {
11733 let handler = ControlHandler::new(Arc::new(Registry::default()));
11734 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
11735 for request in [
11736 ModuleControlRequestFromModule::ScopeSync {
11737 generation: 1,
11738 scopes: vec![head("s", 1)],
11739 },
11740 ModuleControlRequestFromModule::ScopeDescribe {
11741 owner: Principal::Direct,
11742 scope_ref: "s".to_string(),
11743 },
11744 ] {
11745 let reply = call(&handler, &ctx, &request).await;
11746 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
11747 }
11748 assert!(
11749 !handler
11750 .scopes
11751 .read()
11752 .unwrap()
11753 .describe(
11754 &Principal::Reserved {
11755 module_id: OWNER.to_string()
11756 },
11757 "s"
11758 )
11759 .owner_synced
11760 );
11761 }
11762
11763 #[tokio::test]
11766 async fn a_module_without_a_supervised_launch_cannot_sync() {
11767 let handler = ControlHandler::new(Arc::new(Registry::default()));
11768 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
11769 assert_eq!(
11770 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
11771 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
11772 );
11773 }
11774
11775 #[tokio::test]
11776 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
11777 let supervisor = SupervisorHandle::new();
11778 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
11779 let handler = ControlHandler::new(Arc::new(Registry::default()))
11780 .with_supervisor(supervisor.clone());
11781 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
11782 sync(&handler, &incumbent, 1, vec![head("s", 1)])
11783 .await
11784 .expect("the current launch syncs");
11785
11786 supervisor.open_swap(OWNER, "n2".to_string());
11789 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
11790 assert_eq!(
11791 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
11792 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
11793 );
11794 sync(&handler, &incumbent, 2, vec![head("s", 1)])
11795 .await
11796 .expect("the serving owner syncs during the swap");
11797
11798 supervisor.close_swap(OWNER);
11801 assert_eq!(
11802 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
11803 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
11804 );
11805 sync(&handler, &incumbent, 3, vec![head("s", 1)])
11806 .await
11807 .expect("the serving owner syncs after the rollback");
11808 handler.cleanup_connection(candidate.connection_id).unwrap();
11809
11810 supervisor.open_swap(OWNER, "n3".to_string());
11815 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
11816 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
11817 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
11818 .await
11819 .expect("the promoted launch takes authority");
11820 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
11821 panic!("unexpected reply {reply:?}");
11822 };
11823 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
11824 assert_eq!(
11825 sync(&handler, &incumbent, 4, Vec::new()).await,
11826 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
11827 );
11828 }
11829
11830 #[tokio::test]
11833 async fn closing_the_authority_connection_frees_sync_authority() {
11834 let supervisor = SupervisorHandle::new();
11835 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
11836 let handler = ControlHandler::new(Arc::new(Registry::default()))
11837 .with_supervisor(supervisor.clone());
11838 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
11839 sync(&handler, &first, 10, vec![head("s", 1)])
11840 .await
11841 .unwrap();
11842 handler.cleanup_connection(first.connection_id).unwrap();
11843
11844 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
11845 sync(&handler, &second, 1, vec![head("s", 1)])
11846 .await
11847 .expect("the next connection takes the released authority");
11848 }
11849
11850 #[tokio::test]
11851 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
11852 let registry = Arc::new(Registry::default());
11853 let supervisor_handle = SupervisorHandle::new();
11854 let supervisor = Supervisor::new(Arc::clone(®istry), RestartPolicy::default())
11855 .with_handle(supervisor_handle.clone())
11856 .with_daemon_incarnation("incarnation-7".to_string());
11857 supervisor
11860 .supervise_configured(
11861 ModuleSpec {
11862 launch_nonce_env: true,
11863 module_id: OWNER.to_string(),
11864 program: PathBuf::from("/nonexistent/prefrontal-core"),
11865 args: Vec::new(),
11866 env: Vec::new(),
11867 reserved: false,
11868 reserved_prefixes: Vec::new(),
11869 protocol: ModuleProtocol::Subc,
11870 overlap: Default::default(),
11871 },
11872 false,
11873 )
11874 .unwrap();
11875 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
11876 let handler =
11877 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11878 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
11879
11880 let ModuleControlResponseToModule::ScopeDescribe {
11882 status,
11883 daemon_incarnation,
11884 owner_synced,
11885 owner_configured,
11886 scope,
11887 ..
11888 } = describe(&handler, &reader, OWNER, "s").await
11889 else {
11890 panic!("not a describe reply");
11891 };
11892 assert_eq!(status, ScopeStatus::NotLive);
11893 assert_eq!(daemon_incarnation, "incarnation-7");
11894 assert!(!owner_synced);
11895 assert!(owner_configured);
11896 assert!(scope.is_none());
11897
11898 let ModuleControlResponseToModule::ScopeDescribe {
11901 status,
11902 owner_configured,
11903 ..
11904 } = describe(&handler, &reader, "ghost", "s").await
11905 else {
11906 panic!("not a describe reply");
11907 };
11908 assert_eq!(status, ScopeStatus::NotLive);
11909 assert!(!owner_configured);
11910
11911 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
11913 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
11914 let ModuleControlResponseToModule::ScopeDescribe {
11915 status,
11916 scope_epoch,
11917 owner_synced,
11918 scope,
11919 ..
11920 } = describe(&handler, &reader, OWNER, "s").await
11921 else {
11922 panic!("not a describe reply");
11923 };
11924 assert_eq!(status, ScopeStatus::Live);
11925 assert_eq!(scope_epoch, Some(4));
11926 assert!(owner_synced);
11927 let stamp = scope.expect("a live scope carries its stamp");
11928 assert!(
11929 stamp.owner_authorized,
11930 "prefrontal-core is the default authority"
11931 );
11932 assert_eq!(stamp.kind, ScopeKind::Head);
11933 }
11934
11935 #[tokio::test]
11936 async fn scope_authority_owners_decides_owner_authorized() {
11937 let supervisor = SupervisorHandle::new();
11938 supervisor.set_spawn_nonce("broca", "b1".to_string());
11939 let handler = ControlHandler::new(Arc::new(Registry::default()))
11940 .with_supervisor(supervisor)
11941 .with_scope_authority_owners(vec!["broca".to_string()]);
11942 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
11943 let mut gated = head("s", 1);
11944 gated.attributes.agent_id = Some("agent".to_string());
11945 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
11946 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
11947 describe(&handler, &broca, "broca", "s").await
11948 else {
11949 panic!("not a describe reply");
11950 };
11951 assert!(scope.unwrap().owner_authorized);
11952 }
11953
11954 #[tokio::test]
11958 async fn scope_ops_and_the_scopes_capability_are_advertised() {
11959 let handler = ControlHandler::new(Arc::new(Registry::default()));
11960 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
11961 let ack = hello_via_sink(
11962 &handler,
11963 &ctx,
11964 &mut rx,
11965 hello_frame("m", PROTOCOL_VERSION, 1),
11966 )
11967 .await;
11968 let ack = parse_ack(&ack);
11969 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
11970 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
11971 }
11972 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
11973
11974 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
11975 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11976 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11977 let reply = handler
11978 .handle_control_frame(&client, frame)
11979 .await
11980 .unwrap()
11981 .pop()
11982 .unwrap();
11983 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11984 serde_json::from_slice(&reply.body).unwrap()
11985 else {
11986 panic!("not a server.describe reply");
11987 };
11988 assert!(
11989 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
11990 "{capabilities:?}"
11991 );
11992 }
11993
11994 const PLEXUS: &str = "plexus";
11997 const OTHER: &str = "other";
11998 const AFT: &str = "aft";
11999 const BROCA: &str = "broca";
12000 const MAGIC: &str = "magic-context";
12001
12002 fn nonce(module_id: &str) -> String {
12003 format!("nonce-{module_id}")
12004 }
12005
12006 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
12007 let (tx, rx) = mpsc::channel(64);
12008 (
12009 RouteCtx {
12010 connection_id: ConnectionId::new(connection),
12011 egress: FrameSink::new(tx),
12012 },
12013 rx,
12014 )
12015 }
12016
12017 struct Rig {
12021 handler: ControlHandler,
12022 forwarding: Arc<ForwardingTable>,
12023 owner: RouteCtx,
12024 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12025 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
12026 generation: u64,
12027 next_connection: u64,
12028 _supervisor: Supervisor,
12029 }
12030
12031 async fn rig() -> Rig {
12032 let registry = Arc::new(Registry::default());
12033 let forwarding = Arc::new(ForwardingTable::default());
12034 let supervisor_handle = SupervisorHandle::new();
12035 let supervisor = Supervisor::new(Arc::clone(®istry), RestartPolicy::default())
12036 .with_handle(supervisor_handle.clone());
12037 supervisor
12038 .supervise_configured(
12039 ModuleSpec {
12040 launch_nonce_env: true,
12041 module_id: OWNER.to_string(),
12042 program: PathBuf::from("/nonexistent/prefrontal-core"),
12043 args: Vec::new(),
12044 env: Vec::new(),
12045 reserved: false,
12046 reserved_prefixes: Vec::new(),
12047 protocol: ModuleProtocol::Subc,
12048 overlap: Default::default(),
12049 },
12050 false,
12051 )
12052 .unwrap();
12053 for module_id in [OWNER, AFT, BROCA, MAGIC] {
12054 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
12055 }
12056 let handler =
12057 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12058 .with_supervisor(supervisor_handle);
12059 let (owner, mut owner_rx) = wide_ctx(1);
12060 hello_via_sink(
12061 &handler,
12062 &owner,
12063 &mut owner_rx,
12064 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
12065 )
12066 .await;
12067 let mut modules = BTreeMap::new();
12068 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
12069 let (ctx, mut rx) = wide_ctx(connection);
12070 hello_via_sink(
12071 &handler,
12072 &ctx,
12073 &mut rx,
12074 hello_frame(module_id, PROTOCOL_VERSION, connection),
12075 )
12076 .await;
12077 modules.insert(module_id.to_string(), (ctx, rx));
12078 }
12079 Rig {
12080 handler,
12081 forwarding,
12082 owner,
12083 _owner_rx: owner_rx,
12084 modules,
12085 generation: 0,
12086 next_connection: 100,
12087 _supervisor: supervisor,
12088 }
12089 }
12090
12091 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
12092 ScopeCarrier {
12093 principal: Principal::Reserved {
12094 module_id: module_id.to_string(),
12095 },
12096 targets: targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()),
12097 }
12098 }
12099
12100 fn session(scope_epoch: u64) -> ScopeRecord {
12103 let mut record = head("s", scope_epoch);
12104 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
12105 record.attributes.agent_id = Some("agent-1".to_string());
12106 record.attributes.delegates = true;
12107 record
12108 }
12109
12110 impl Rig {
12111 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
12112 self.generation += 1;
12113 sync(&self.handler, &self.owner, self.generation, scopes)
12114 .await
12115 .expect("the owner's sync is accepted");
12116 }
12117
12118 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
12119 ScopeSelector {
12120 owner: Principal::Reserved {
12121 module_id: OWNER.to_string(),
12122 },
12123 scope_ref: scope_ref.to_string(),
12124 scope_epoch,
12125 }
12126 }
12127
12128 fn open_frame(
12129 &mut self,
12130 opener: Option<&str>,
12131 target: &str,
12132 scope: Option<ScopeSelector>,
12133 ) -> (
12134 RouteCtx,
12135 mpsc::Receiver<crate::router::OutboundFrame>,
12136 Frame,
12137 ) {
12138 self.next_connection += 1;
12139 let (ctx, rx) = wide_ctx(self.next_connection);
12140 let root = unique_project_root("scoped-open");
12141 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
12142 target: RouteTarget::ToolProvider {
12143 module_id: target.to_string(),
12144 },
12145 identity: BindIdentity::new(
12146 root.path().to_path_buf(),
12147 "unit".to_string(),
12148 "session".to_string(),
12149 ),
12150 consumer_identity: opener.map(|module_id| ConsumerIdentity {
12151 module_id: module_id.to_string(),
12152 launch_nonce: nonce(module_id),
12153 }),
12154 consumer_capabilities: None,
12155 role_versions: None,
12156 admission_facts: None,
12157 scope,
12158 })
12159 .unwrap();
12160 let frame = Frame::build(
12161 FrameType::Request,
12162 control_flags(),
12163 0,
12164 0,
12165 self.next_connection,
12166 body,
12167 )
12168 .unwrap();
12169 (ctx, rx, frame)
12170 }
12171
12172 async fn refused(
12174 &mut self,
12175 opener: Option<&str>,
12176 target: &str,
12177 scope: Option<ScopeSelector>,
12178 ) -> String {
12179 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
12180 let replies = self
12181 .handler
12182 .handle_control_frame(&ctx, frame)
12183 .await
12184 .unwrap();
12185 assert_eq!(replies.len(), 1, "{replies:?}");
12186 assert_eq!(replies[0].header.ty, FrameType::Error);
12187 let (_, module_rx) = self.modules.get_mut(target).unwrap();
12188 assert!(
12189 module_rx.try_recv().is_err(),
12190 "a refused open relays nothing"
12191 );
12192 parse_error(&replies[0])["code"]
12193 .as_str()
12194 .unwrap()
12195 .to_string()
12196 }
12197
12198 async fn relayed(
12200 &mut self,
12201 opener: Option<&str>,
12202 target: &str,
12203 scope: Option<ScopeSelector>,
12204 ) -> Relayed {
12205 let (ctx, rx, frame) = self.open_frame(opener, target, scope);
12206 let handler = self.handler.clone();
12207 let task_ctx = ctx.clone();
12208 let task = tokio::spawn(async move {
12209 handler
12210 .handle_control_frame(&task_ctx, frame)
12211 .await
12212 .unwrap()
12213 });
12214 let (_, module_rx) = self.modules.get_mut(target).unwrap();
12215 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
12216 .await
12217 .expect("the target receives the relayed route.bind")
12218 .unwrap()
12219 .frame;
12220 Relayed {
12221 target: target.to_string(),
12222 client: ctx,
12223 client_rx: rx,
12224 task,
12225 bind,
12226 }
12227 }
12228
12229 async fn ack(&self, relayed: &Relayed) {
12230 let (module, _) = &self.modules[&relayed.target];
12231 self.handler
12232 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
12233 .await
12234 .unwrap();
12235 }
12236
12237 async fn bound(
12239 &mut self,
12240 opener: Option<&str>,
12241 target: &str,
12242 scope: Option<ScopeSelector>,
12243 ) -> Bound {
12244 let relayed = self.relayed(opener, target, scope).await;
12245 self.ack(&relayed).await;
12246 let Relayed {
12247 target,
12248 client,
12249 mut client_rx,
12250 task,
12251 bind,
12252 } = relayed;
12253 assert!(
12254 task.await.unwrap().is_empty(),
12255 "the open is answered by commit"
12256 );
12257 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
12258 Bound {
12259 target,
12260 client,
12261 client_rx,
12262 channel,
12263 epoch,
12264 bind,
12265 }
12266 }
12267
12268 fn live(&self, route: &Bound) -> bool {
12269 matches!(
12270 self.forwarding
12271 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
12272 .unwrap(),
12273 DataRoute::Client(DataRouteState::Bound(_))
12274 )
12275 }
12276 }
12277
12278 struct Relayed {
12279 target: String,
12280 client: RouteCtx,
12281 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12282 task: tokio::task::JoinHandle<Vec<Frame>>,
12283 bind: Frame,
12284 }
12285
12286 struct Bound {
12287 target: String,
12288 client: RouteCtx,
12289 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12290 channel: u16,
12291 epoch: u32,
12292 bind: Frame,
12293 }
12294
12295 impl Bound {
12296 fn closed_reason(&mut self) -> RouteCloseReason {
12299 let mut reason = None;
12300 let mut goodbye = false;
12301 while let Ok(outbound) = self.client_rx.try_recv() {
12302 let frame = outbound.frame;
12303 match frame.header.ty {
12304 FrameType::Goodbye => {
12305 assert_eq!(
12306 (frame.header.channel, frame.header.epoch),
12307 (self.channel, self.epoch)
12308 );
12309 goodbye = true;
12310 }
12311 FrameType::Push => {
12312 let ClientControlPush::RouteClosed {
12313 reason: r,
12314 module_id,
12315 ..
12316 } = serde_json::from_slice(&frame.body).unwrap()
12317 else {
12318 panic!("unexpected push");
12319 };
12320 assert_eq!(module_id, self.target);
12321 reason = Some(r);
12322 }
12323 other => panic!("unexpected frame {other:?}"),
12324 }
12325 }
12326 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
12327 reason.expect("the client is told why the route closed")
12328 }
12329
12330 fn untouched(&mut self) -> bool {
12331 self.client_rx.try_recv().is_err()
12332 }
12333
12334 fn stamp(&self) -> Option<ScopeStamp> {
12335 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
12336 ModuleControlRequest::RouteBind { scope, .. } => scope,
12337 other => panic!("expected a route.bind, got {other:?}"),
12338 }
12339 }
12340 }
12341
12342 #[tokio::test]
12343 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
12344 ) {
12345 let mut rig = rig().await;
12346 let mut record = session(1);
12347 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
12348 record.child_owners = vec![Principal::Reserved {
12349 module_id: MAGIC.to_string(),
12350 }];
12351 rig.sync(vec![record]).await;
12352 let scope = || Some(rig_selector("s", Some(1)));
12353
12354 rig.bound(Some(OWNER), PLEXUS, scope()).await;
12357 rig.bound(Some(AFT), OTHER, scope()).await;
12358 rig.bound(Some(BROCA), PLEXUS, scope()).await;
12359
12360 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
12364 assert_eq!(
12365 rig.refused(opener, target, scope()).await,
12366 error_codes::SCOPE_NOT_CARRIER,
12367 "{opener:?} -> {target}"
12368 );
12369 }
12370 }
12371
12372 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
12373 ScopeSelector {
12374 owner: Principal::Reserved {
12375 module_id: OWNER.to_string(),
12376 },
12377 scope_ref: scope_ref.to_string(),
12378 scope_epoch,
12379 }
12380 }
12381
12382 #[tokio::test]
12383 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
12384 let mut rig = rig().await;
12385 rig.sync(vec![session(1)]).await;
12386 for opener in [OWNER, AFT] {
12387 assert_eq!(
12388 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
12389 .await,
12390 error_codes::SCOPE_EPOCH_REQUIRED,
12391 "{opener}"
12392 );
12393 }
12394 }
12395
12396 #[tokio::test]
12397 async fn admission_separates_not_synced_not_live_and_ended() {
12398 let mut rig = rig().await;
12399 let code = rig
12401 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12402 .await;
12403 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
12404 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
12405
12406 let ghost = ScopeSelector {
12408 owner: Principal::Reserved {
12409 module_id: "ghost".to_string(),
12410 },
12411 scope_ref: "s".to_string(),
12412 scope_epoch: Some(1),
12413 };
12414 assert_eq!(
12415 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
12416 error_codes::SCOPE_NOT_LIVE
12417 );
12418
12419 rig.sync(vec![session(2)]).await;
12420 assert_eq!(
12421 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
12422 .await,
12423 error_codes::SCOPE_NOT_LIVE
12424 );
12425 for epoch in [1, 3] {
12426 assert_eq!(
12427 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
12428 .await,
12429 error_codes::SCOPE_ENDED,
12430 "epoch {epoch}"
12431 );
12432 }
12433 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
12435 .await;
12436 }
12437
12438 #[tokio::test]
12439 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
12440 let mut rig = rig().await;
12441 rig.sync(vec![session(1)]).await;
12442 let route = rig
12443 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12444 .await;
12445 let stamp = route.stamp().expect("a scoped bind carries the stamp");
12446 assert_eq!(stamp.scope_ref, "s");
12447 assert_eq!(stamp.scope_epoch, 1);
12448 assert_eq!(stamp.kind, ScopeKind::Head);
12449 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
12450 assert!(stamp.attributes.delegates);
12451 assert!(stamp.owner_authorized);
12452 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
12453 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
12454
12455 let (broca, mut broca_rx) = wide_ctx(50);
12458 hello_via_sink(
12459 &rig.handler,
12460 &broca,
12461 &mut broca_rx,
12462 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
12463 )
12464 .await;
12465 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
12466 .await
12467 .unwrap();
12468 let own = ScopeSelector {
12469 owner: Principal::Reserved {
12470 module_id: BROCA.to_string(),
12471 },
12472 scope_ref: "b".to_string(),
12473 scope_epoch: Some(1),
12474 };
12475 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
12476 assert!(!route.stamp().unwrap().owner_authorized);
12477 }
12478
12479 #[tokio::test]
12484 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
12485 let mut rig = rig().await;
12486 rig.sync(vec![session(1)]).await;
12487 let mut cotenant = rig
12488 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
12489 .await;
12490
12491 let mut changed = session(1);
12492 changed.child_owners.push(Principal::Reserved {
12493 module_id: MAGIC.to_string(),
12494 });
12495 let mut ended = None;
12496 for (code, next) in [
12497 (error_codes::SCOPE_CHANGED, vec![changed]),
12498 (error_codes::SCOPE_ENDED, Vec::new()),
12499 ] {
12500 let relayed = rig
12501 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12502 .await;
12503 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
12504 ended = Some(next.is_empty());
12505 rig.sync(next).await;
12506 rig.ack(&relayed).await;
12507 let replies = relayed.task.await.unwrap();
12508 assert_eq!(replies.len(), 1, "{replies:?}");
12509 assert_eq!(parse_error(&replies[0])["code"], code);
12510 assert_eq!(
12511 subc_protocol::error_codes::is_retryable_route_open(code),
12512 code == error_codes::SCOPE_CHANGED
12513 );
12514 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
12515 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
12517 let mut goodbyes = Vec::new();
12520 while let Ok(outbound) = plexus_rx.try_recv() {
12521 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
12522 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
12523 }
12524 assert!(
12525 goodbyes.contains(&(bind_channel, bind_epoch)),
12526 "{goodbyes:?}"
12527 );
12528 assert!(rig
12529 .handler
12530 .registry
12531 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
12532 .unwrap()
12533 .is_some());
12534 }
12535 assert_eq!(ended, Some(true));
12536 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
12539 }
12540
12541 #[tokio::test]
12544 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
12545 struct Case {
12546 name: &'static str,
12547 change: fn(&mut ScopeRecord),
12548 closed: [Option<RouteCloseReason>; 4],
12551 }
12552 use RouteCloseReason::*;
12553 let cases = [
12554 Case {
12555 name: "a carrier entry removed",
12556 change: |r| {
12557 r.carriers.retain(|c| {
12558 c.principal
12559 != Principal::Reserved {
12560 module_id: AFT.to_string(),
12561 }
12562 })
12563 },
12564 closed: [None, Some(ScopeCarrierRemoved), None, None],
12565 },
12566 Case {
12567 name: "a target removed from a carrier",
12568 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
12569 closed: [None, None, None, Some(ScopeCarrierRemoved)],
12570 },
12571 Case {
12572 name: "a bare carrier narrowed to targets",
12573 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
12574 closed: [None, Some(ScopeCarrierRemoved), None, None],
12575 },
12576 Case {
12577 name: "delegates turned off",
12578 change: |r| r.attributes.delegates = false,
12579 closed: [Some(ScopeDelegationChanged); 4],
12580 },
12581 Case {
12582 name: "agent_id changed",
12583 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
12584 closed: [Some(ScopeDelegationChanged); 4],
12585 },
12586 Case {
12587 name: "a carrier added, child owners changed, the record re-sent",
12588 change: |r| {
12589 r.carriers.push(carrier(MAGIC, None));
12590 r.child_owners.push(Principal::Reserved {
12591 module_id: MAGIC.to_string(),
12592 });
12593 },
12594 closed: [None; 4],
12595 },
12596 Case {
12597 name: "a target added",
12598 change: |r| {
12599 r.carriers[1]
12600 .targets
12601 .as_mut()
12602 .unwrap()
12603 .push("third".to_string())
12604 },
12605 closed: [None; 4],
12606 },
12607 Case {
12608 name: "delegates turned on",
12609 change: |r| r.attributes.delegates = true,
12610 closed: [None; 4],
12611 },
12612 ];
12613 for case in cases {
12614 let mut rig = rig().await;
12615 rig.sync(vec![session(1)]).await;
12616 let scope = || Some(rig_selector("s", Some(1)));
12617 let mut routes = [
12618 rig.bound(Some(OWNER), PLEXUS, scope()).await,
12619 rig.bound(Some(AFT), PLEXUS, scope()).await,
12620 rig.bound(Some(BROCA), PLEXUS, scope()).await,
12621 rig.bound(Some(BROCA), OTHER, scope()).await,
12622 ];
12623 let mut record = session(1);
12624 (case.change)(&mut record);
12625 rig.sync(vec![record]).await;
12626 for (index, expected) in case.closed.iter().enumerate() {
12627 let route = &mut routes[index];
12628 match expected {
12629 Some(reason) => {
12630 assert!(!rig.live(route), "{}: route {index} still live", case.name);
12631 assert_eq!(
12632 route.closed_reason(),
12633 *reason,
12634 "{}: route {index}",
12635 case.name
12636 );
12637 }
12638 None => {
12639 assert!(rig.live(route), "{}: route {index} closed", case.name);
12640 assert!(
12641 route.untouched(),
12642 "{}: route {index} was told something",
12643 case.name
12644 );
12645 }
12646 }
12647 }
12648 }
12649 }
12650
12651 #[tokio::test]
12652 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
12653 for next in [Vec::new(), vec![session(2)]] {
12655 let mut rig = rig().await;
12656 rig.sync(vec![session(1)]).await;
12657 let mut route = rig
12658 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12659 .await;
12660 rig.sync(next).await;
12661 assert!(!rig.live(&route));
12662 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
12663 }
12664
12665 let mut rig = rig().await;
12668 let mut child = session(1);
12669 child.scope_ref = "child".to_string();
12670 child.kind = ScopeKind::Worker;
12671 child.parent = Some(ScopeParent {
12672 owner: Principal::Reserved {
12673 module_id: OWNER.to_string(),
12674 },
12675 scope_ref: "s".to_string(),
12676 scope_epoch: 1,
12677 });
12678 rig.sync(vec![session(1), child.clone()]).await;
12679 let mut child_route = rig
12680 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
12681 .await;
12682 assert_eq!(
12683 child_route.stamp().unwrap().parent_state,
12684 Some(ParentState::Linked)
12685 );
12686 rig.sync(vec![child]).await;
12687 assert!(!rig.live(&child_route));
12688 assert_eq!(
12689 child_route.closed_reason(),
12690 RouteCloseReason::ScopeParentEnded
12691 );
12692 }
12693
12694 #[tokio::test]
12695 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
12696 ) {
12697 let mut rig = rig().await;
12698 rig.sync(vec![session(1)]).await;
12699 let mut route = rig
12700 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12701 .await;
12702 let before = rig.forwarding.published_scope_tag(OWNER, "s");
12703 rig.sync(vec![session(1)]).await;
12704 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
12705 assert!(rig.live(&route) && route.untouched());
12706
12707 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
12711 .forwarding
12712 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
12713 .unwrap()
12714 else {
12715 panic!("the route is bound");
12716 };
12717 binding.flow.acquire_tagged(9, false).await.unwrap();
12718 let mut widened = session(1);
12719 widened.carriers.push(carrier(MAGIC, None));
12720 rig.sync(vec![widened]).await;
12721 assert!(rig.live(&route) && route.untouched());
12722 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
12723 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
12724 assert_eq!(binding.flow.in_flight(), 1);
12727 binding
12728 .flow
12729 .acquire_tagged(10, false)
12730 .await
12731 .expect("the flow is still open");
12732 }
12733
12734 #[tokio::test]
12737 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
12738 let mut rig = rig().await;
12739 rig.sync(vec![session(1)]).await;
12740 let mut on_incumbent = rig
12741 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12742 .await;
12743
12744 let (candidate, _candidate_rx) = wide_ctx(9);
12747 let registration = rig
12748 .handler
12749 .registry
12750 .register_candidate_with_control_ops(
12751 manifest(PLEXUS, PROTOCOL_VERSION),
12752 PROTOCOL_VERSION,
12753 candidate.connection_id,
12754 module_baseline_control_ops(),
12755 )
12756 .unwrap();
12757 rig.forwarding
12758 .register_candidate_module_connection(
12759 candidate.connection_id,
12760 PLEXUS.to_string(),
12761 PROTOCOL_VERSION,
12762 manifest_concurrency(®istration.manifest),
12763 candidate.egress.clone(),
12764 )
12765 .unwrap();
12766 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
12767 rig.handler
12768 .registry
12769 .promote_candidate(PLEXUS)
12770 .unwrap()
12771 .unwrap();
12772 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
12773
12774 rig.sync(Vec::new()).await;
12775 assert!(!rig.live(&on_incumbent));
12776 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
12777 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
12778 let goodbye = incumbent_rx
12779 .try_recv()
12780 .expect("the superseded endpoint is told")
12781 .frame;
12782 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
12783 }
12784 }
12785}
12786
12787#[cfg(test)]
12788mod concurrency_default_exposure_tests {
12789 use super::*;
12790
12791 fn hello_body(role_json: &str) -> Vec<u8> {
12792 format!(
12793 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
12794 )
12795 .into_bytes()
12796 }
12797
12798 fn manifest_from(body: &[u8]) -> ModuleManifest {
12799 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
12800 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
12801 .expect("manifest parses")
12802 }
12803
12804 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
12805
12806 #[test]
12807 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
12808 let body = hello_body(&format!(
12809 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
12810 ));
12811 let manifest = manifest_from(&body);
12812 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
12815 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
12816 }
12817
12818 #[test]
12819 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
12820 let body = hello_body(&format!(
12821 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
12822 ));
12823 let manifest = manifest_from(&body);
12824 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
12825 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
12826 }
12827
12828 #[test]
12829 fn non_management_roles_are_never_reported() {
12830 let body = hello_body(
12831 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
12832 );
12833 let manifest = manifest_from(&body);
12834 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
12835 }
12836}