1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 path::{Path, PathBuf},
5 sync::{Arc, Mutex, RwLock},
6 time::{Duration, Instant as StdInstant},
7};
8
9use serde::{Deserialize, Serialize};
10use subc_control::{
11 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
12 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
13 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
14 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
15 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
16 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
17 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
18};
19use subc_protocol::{
20 error_codes,
21 manifest::{
22 validate_hello_capability_grammar, validate_hello_self_signal_declarations,
23 CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
24 ProviderRole,
25 },
26 scope::{
27 ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
28 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_DESCRIBE_OP, SCOPE_SYNC_OP,
29 },
30 session::{
31 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
32 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
33 MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE, ROLE_VERSIONS_FIELD,
34 },
35 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
36 Priority, RouteTarget, PROTOCOL_VERSION,
37};
38use tokio::time::{timeout_at, Instant};
39use tracing::{debug, info, warn};
40
41use crate::{
42 capability_requirements::{
43 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
44 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
45 RuntimeModule,
46 },
47 daemon_config::RestartRequiredSection,
48 forwarding::{
49 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
50 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
51 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
52 },
53 observability::{
54 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
55 },
56 provenance::{
57 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
58 },
59 registry::{ChannelState, ConnectionId, Registry, RegistryError},
60 router::{RouteCtx, RouterError},
61 scopes::{BoundScope, HelloLaunchNonces, ScopeTable},
62 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
63 stderr_tail::{CaptureState, TailEntry},
64 supervise::{
65 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
66 SupervisorHandle, SwapHelloAdmission,
67 },
68 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
69};
70
71pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
77
78const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
79const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
80const CAP_PING_PONG: &str = "ping_pong_v1";
81const CAP_SESSION_ATTACH: &str = "session_attach_v1";
82const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
83
84const SUBC_CONTROL_OPS: &[&str] = &[
85 ops::SERVER_DESCRIBE,
86 ops::CATALOG_LIST,
87 ops::ROUTE_OPEN,
88 ops::ROUTE_POLL,
89 ops::ROUTE_CLOSING,
90 ops::ROUTE_CLOSED,
91 ops::SUPERVISOR_LIST,
92 ops::SUPERVISOR_RESTART,
93 ops::SUPERVISOR_SWAP,
94 ops::SUPERVISOR_RELOAD,
95 ops::SUPERVISOR_RESCAN,
96 ops::SUPERVISOR_RELEASE_RESERVED,
97 ops::SUPERVISOR_SET_ENABLED,
98 ops::SUPERVISOR_HEALTH_PROBE,
99 ops::SUPERVISOR_HEALTH,
100 ops::SUPERVISOR_STDERR_TAIL,
101 ops::SUPERVISOR_TERMINALS,
102 ops::SUPERVISOR_ROUTES,
103 ops::SUPERVISOR_PROVENANCE,
104 ops::SUPERVISOR_SPAWN_SNAPSHOT,
105 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
106];
107
108const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
109 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
110 "supervisor.live_roots",
111 SCOPE_SYNC_OP,
112 SCOPE_DESCRIBE_OP,
113];
114
115const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
119
120const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
121
122pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
130
131pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
146
147pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
165
166const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
167const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
168
169fn reload_verdict(
170 configured: &Path,
171 spawned_from: Option<&Path>,
172 image: subc_control::RunningImageAgreement,
173) -> PendingReloadVerdict {
174 let path = match spawned_from {
175 Some(spawned_from) if configured == spawned_from => ReloadPathAgreement::Match,
176 Some(spawned_from) => ReloadPathAgreement::Mismatch {
177 configured: configured.to_path_buf(),
178 spawned_from: spawned_from.to_path_buf(),
179 },
180 None => ReloadPathAgreement::Unavailable {
181 reason: if matches!(
182 image,
183 subc_control::RunningImageAgreement::Unavailable {
184 reason: subc_control::RunningImageUnavailableReason::NotRunning
185 }
186 ) {
187 ReloadPathUnavailableReason::NotRunning
188 } else {
189 ReloadPathUnavailableReason::SpawnedPathUnavailable
190 },
191 },
192 };
193 PendingReloadVerdict { path, image }
194}
195
196#[derive(Clone)]
197struct DaemonProvenanceFacts {
198 build: DaemonBuildProvenance,
199 pid: Option<u32>,
200 started_at_ms: Option<u64>,
201 start_clock: Option<crate::clock::StartClock>,
202 executable_path: Option<PathBuf>,
203 executable_identity: Option<SpawnedFileIdentity>,
204 process_start_time: Option<u64>,
205 probe: ExecutableIdentityProbe,
206}
207
208impl Default for DaemonProvenanceFacts {
209 fn default() -> Self {
210 Self {
211 build: DaemonBuildProvenance {
212 build_git_sha: None,
213 build_lock_digest: None,
214 },
215 pid: None,
216 started_at_ms: None,
217 start_clock: None,
218 executable_path: None,
219 executable_identity: None,
220 process_start_time: None,
221 probe: ExecutableIdentityProbe::default(),
222 }
223 }
224}
225
226#[derive(Debug, Clone)]
227struct SupervisorRescanContext {
228 supervisor: Supervisor,
229 config_path: PathBuf,
230 configured_port: Option<u16>,
231 storage_config: Option<crate::daemon_config::StorageConfig>,
232 admission_facts_carrier_module_id: Option<String>,
233 admission_facts_targets: Option<Vec<String>>,
234 scope_authority_owners: Vec<String>,
235}
236
237const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
258 "reloading",
259 "supervisor_not_live",
260 "registration_not_active",
261 "no_forwarding_connection",
262 "relay_send_failed",
263];
264
265#[derive(Clone)]
267pub struct ControlHandler {
268 registry: Arc<Registry>,
269 forwarding: Arc<ForwardingTable>,
270 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
271 supervisor: SupervisorHandle,
272 subc_capabilities: Arc<[String]>,
273 route_bind_relay_timeout: Duration,
277 route_bind_relay_timeouts: BTreeMap<String, Duration>,
281 route_bind_breakers: RouteBindBreakers,
284 route_bind_concurrency: RouteBindConcurrency,
287 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
292 route_bind_breaker_threshold: u32,
294 route_bind_breaker_cooldown: Duration,
296 health_probe_timeout: Duration,
297 storage_config: Option<crate::daemon_config::StorageConfig>,
300 machine_id: Option<crate::machine_id::MachineId>,
304 admission_facts_carrier_module_id: Option<String>,
305 admission_facts_targets: Option<Vec<String>>,
306 scopes: Arc<RwLock<ScopeTable>>,
310 scope_authority_owners: Vec<String>,
313 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
316 rescan: Option<SupervisorRescanContext>,
317 connected_clients: ConnectedClients,
318 counters: DaemonCounters,
319 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
320 daemon_provenance: DaemonProvenanceFacts,
321 #[cfg(test)]
322 control_dispatch_delay: Option<Duration>,
323 #[cfg(test)]
324 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
325}
326
327impl fmt::Debug for ControlHandler {
328 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
329 f.debug_struct("ControlHandler")
330 .field("registry", &self.registry)
331 .field("forwarding", &self.forwarding)
332 .field("process_liveness", &self.process_liveness.is_some())
333 .field("supervisor", &self.supervisor)
334 .field("subc_capabilities", &self.subc_capabilities)
335 .finish()
336 }
337}
338
339struct RouteOpenRequest {
340 target: RouteTarget,
341 identity: BindIdentity,
342 consumer_identity: Option<ConsumerIdentity>,
343 consumer_capabilities: Option<Vec<String>>,
344 role_versions: Option<BTreeMap<String, String>>,
345 admission_facts: Option<serde_json::Value>,
346 scope: Option<ScopeSelector>,
347}
348
349struct RouteBindReservationGuard {
350 forwarding: Arc<ForwardingTable>,
351 endpoint: ModuleEndpointId,
352 relay_corr: u64,
353 armed: bool,
354}
355
356struct ModuleControlRpcGuard {
357 forwarding: Arc<ForwardingTable>,
358 endpoint: ModuleEndpointId,
359 corr: u64,
360 armed: bool,
361}
362
363impl ModuleControlRpcGuard {
364 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
365 Self {
366 forwarding,
367 endpoint,
368 corr,
369 armed: true,
370 }
371 }
372
373 fn disarm(&mut self) {
374 self.armed = false;
375 }
376}
377
378impl Drop for ModuleControlRpcGuard {
379 fn drop(&mut self) {
380 if self.armed {
381 let _ = self
382 .forwarding
383 .cancel_module_control_rpc(self.endpoint, self.corr);
384 }
385 }
386}
387
388impl RouteBindReservationGuard {
389 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
390 Self {
391 forwarding,
392 endpoint,
393 relay_corr,
394 armed: true,
395 }
396 }
397
398 fn release_and_disarm(&mut self) {
399 if !self.armed {
400 return;
401 }
402 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
403 self.endpoint,
404 self.relay_corr,
405 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
406 ) {
407 send_goodbye_target_best_effort(
408 &self.forwarding.counters(),
409 &target,
410 "canceled route.bind",
411 );
412 }
413 self.armed = false;
414 }
415
416 fn disarm(&mut self) {
417 self.armed = false;
418 }
419}
420
421impl Drop for RouteBindReservationGuard {
422 fn drop(&mut self) {
423 self.release_and_disarm();
424 }
425}
426
427#[derive(Debug, Clone, Default)]
457pub(crate) struct RouteBindBreakers {
458 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
459}
460
461#[derive(Debug, Clone, Default)]
462pub(crate) struct RouteBindConcurrency {
463 modules: Arc<Mutex<HashMap<String, usize>>>,
464}
465
466struct RouteBindConcurrencyGuard {
467 concurrency: RouteBindConcurrency,
468 module_id: String,
469}
470
471impl RouteBindConcurrency {
472 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
475 let mut modules = self
476 .modules
477 .lock()
478 .expect("route.bind concurrency mutex poisoned");
479 let in_flight = modules.entry(module_id.to_string()).or_default();
480 if *in_flight >= limit {
481 return Err(*in_flight);
482 }
483 *in_flight += 1;
484 Ok(RouteBindConcurrencyGuard {
485 concurrency: self.clone(),
486 module_id: module_id.to_string(),
487 })
488 }
489}
490
491impl Drop for RouteBindConcurrencyGuard {
492 fn drop(&mut self) {
493 let mut modules = self
494 .concurrency
495 .modules
496 .lock()
497 .expect("route.bind concurrency mutex poisoned");
498 let remove = {
499 let in_flight = modules
500 .get_mut(&self.module_id)
501 .expect("admitted route.bind has a concurrency entry");
502 *in_flight -= 1;
503 *in_flight == 0
504 };
505 if remove {
506 modules.remove(&self.module_id);
507 }
508 }
509}
510
511#[derive(Debug, Default)]
512struct ModuleBreakerState {
513 consecutive_timeouts: u32,
515 cooldown_until: Option<Instant>,
518 probe_in_flight: bool,
523}
524
525enum RouteBindAdmission<'a> {
527 Admitted {
528 guard: RouteBindBreakerGuard<'a>,
529 probe: bool,
532 },
533 Refused {
534 consecutive_timeouts: u32,
535 retry_in: Duration,
539 probe_in_flight: bool,
540 },
541}
542
543struct RouteBindBreakerGuard<'a> {
550 breakers: RouteBindBreakers,
551 module_id: &'a str,
552 settled: bool,
553}
554
555impl RouteBindBreakerGuard<'_> {
556 fn record_accepted(&mut self) -> bool {
560 self.settled = true;
561 self.breakers.record_accepted(self.module_id)
562 }
563
564 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
567 self.settled = true;
568 self.breakers
569 .record_timeout(self.module_id, threshold, cooldown)
570 }
571
572 fn record_inconclusive(&mut self) {
581 self.settled = true;
582 self.breakers.record_inconclusive(self.module_id);
583 }
584}
585
586impl Drop for RouteBindBreakerGuard<'_> {
587 fn drop(&mut self) {
588 if !self.settled {
589 self.breakers.record_inconclusive(self.module_id);
590 }
591 }
592}
593
594struct BreakerOpened {
598 consecutive_timeouts: u32,
599 reopened_after_probe: bool,
602}
603
604impl RouteBindBreakers {
605 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
606 self.modules
607 .lock()
608 .expect("route.bind breaker mutex poisoned")
609 }
610
611 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
614 let admitted = |probe| RouteBindAdmission::Admitted {
615 guard: RouteBindBreakerGuard {
616 breakers: self.clone(),
617 module_id,
618 settled: false,
619 },
620 probe,
621 };
622
623 let mut modules = self.lock();
624 let Some(state) = modules.get_mut(module_id) else {
625 return admitted(false);
626 };
627 let Some(cooldown_until) = state.cooldown_until else {
628 return admitted(false);
629 };
630 if state.probe_in_flight {
631 return RouteBindAdmission::Refused {
632 consecutive_timeouts: state.consecutive_timeouts,
633 retry_in: Duration::ZERO,
634 probe_in_flight: true,
635 };
636 }
637 let now = Instant::now();
638 if now < cooldown_until {
639 return RouteBindAdmission::Refused {
640 consecutive_timeouts: state.consecutive_timeouts,
641 retry_in: cooldown_until - now,
642 probe_in_flight: false,
643 };
644 }
645 state.probe_in_flight = true;
646 admitted(true)
647 }
648
649 fn record_accepted(&self, module_id: &str) -> bool {
650 self.lock()
651 .remove(module_id)
652 .is_some_and(|state| state.cooldown_until.is_some())
653 }
654
655 fn record_timeout(
656 &self,
657 module_id: &str,
658 threshold: u32,
659 cooldown: Duration,
660 ) -> Option<BreakerOpened> {
661 let mut modules = self.lock();
662 let state = modules.entry(module_id.to_string()).or_default();
663 let was_open = state.cooldown_until.is_some();
664 let was_probe = state.probe_in_flight;
665 state.probe_in_flight = false;
666 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
667 if state.consecutive_timeouts < threshold {
668 return None;
669 }
670 state.cooldown_until = Some(Instant::now() + cooldown);
671 Some(BreakerOpened {
672 consecutive_timeouts: state.consecutive_timeouts,
673 reopened_after_probe: was_open && was_probe,
674 })
675 }
676
677 fn record_inconclusive(&self, module_id: &str) {
678 if let Some(state) = self.lock().get_mut(module_id) {
679 state.probe_in_flight = false;
680 }
681 }
682
683 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
698 self.lock()
699 .remove(module_id)
700 .map(|state| state.consecutive_timeouts)
701 .filter(|discarded| *discarded > 0)
702 }
703
704 fn open_snapshot(&self) -> Option<serde_json::Value> {
711 let now = Instant::now();
712 let modules = self.lock();
713 let open = modules
714 .iter()
715 .filter_map(|(module_id, state)| {
716 let cooldown_until = state.cooldown_until?;
717 Some((
718 module_id.clone(),
719 serde_json::json!({
720 "consecutive_timeouts": state.consecutive_timeouts,
721 "cooldown_remaining_ms":
722 cooldown_until.saturating_duration_since(now).as_millis() as u64,
723 "probe_in_flight": state.probe_in_flight,
724 }),
725 ))
726 })
727 .collect::<serde_json::Map<String, serde_json::Value>>();
728 (!open.is_empty()).then_some(serde_json::Value::Object(open))
729 }
730}
731
732impl ControlHandler {
733 pub fn new(registry: Arc<Registry>) -> Self {
734 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
735 }
736
737 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
738 let counters = forwarding.counters();
739 let route_bind_breakers = forwarding.route_bind_breakers();
743 let route_bind_concurrency = forwarding.route_bind_concurrency();
744 let route_outages = forwarding.route_outages();
745 Self {
746 registry,
747 forwarding,
748 process_liveness: None,
749 supervisor: SupervisorHandle::new(),
750 subc_capabilities: Arc::from([
751 CAP_MANIFEST_REGISTRATION.to_string(),
752 CAP_CHANNEL_LIFECYCLE.to_string(),
753 CAP_PING_PONG.to_string(),
754 CAP_SESSION_ATTACH.to_string(),
755 CAP_ADMISSION_FACTS_RELAY.to_string(),
756 CAP_SCOPES_V1.to_string(),
757 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
758 ]),
759 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
760 route_bind_relay_timeouts: BTreeMap::new(),
761 route_bind_breakers,
762 route_bind_concurrency,
763 route_outages,
764 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
765 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
766 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
767 storage_config: None,
768 machine_id: None,
769 admission_facts_carrier_module_id: None,
770 admission_facts_targets: None,
771 scopes: Arc::new(RwLock::new(ScopeTable::new(
772 crate::daemon_config::default_scope_authority_owners(),
773 ))),
774 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
775 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
776 rescan: None,
777 connected_clients: ConnectedClients::new(),
778 counters,
779 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
780 daemon_provenance: DaemonProvenanceFacts::default(),
781 #[cfg(test)]
782 control_dispatch_delay: None,
783 #[cfg(test)]
784 provenance_probe_override: None,
785 }
786 }
787
788 pub fn with_storage_config(
791 mut self,
792 storage_config: Option<crate::daemon_config::StorageConfig>,
793 ) -> Self {
794 self.storage_config = storage_config;
795 self
796 }
797
798 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
801 self.machine_id = machine_id;
802 self
803 }
804
805 pub fn with_admission_facts_config(
809 mut self,
810 carrier_module_id: Option<String>,
811 targets: Option<Vec<String>>,
812 ) -> Self {
813 self.admission_facts_carrier_module_id = carrier_module_id;
814 self.admission_facts_targets = targets;
815 self
816 }
817
818 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
822 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
823 self.scope_authority_owners = owners;
824 self
825 }
826
827 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
830 self.route_bind_relay_timeout = timeout;
831 self
832 }
833
834 pub fn with_route_bind_relay_timeouts(
840 mut self,
841 timeouts: impl IntoIterator<Item = (String, Duration)>,
842 ) -> Self {
843 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
844 self
845 }
846
847 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
853 self.route_bind_relay_timeouts
854 .get(module_id)
855 .copied()
856 .unwrap_or(self.route_bind_relay_timeout)
857 }
858
859 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
867 self.route_bind_breaker_threshold = threshold.max(1);
868 self.route_bind_breaker_cooldown = cooldown;
869 self
870 }
871
872 #[cfg(test)]
873 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
874 self.health_probe_timeout = timeout;
875 self
876 }
877
878 #[cfg(test)]
879 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
880 self.control_dispatch_delay = Some(delay);
881 self
882 }
883
884 pub fn with_process_liveness(
885 mut self,
886 process_liveness: Arc<dyn ModuleProcessLiveness>,
887 ) -> Self {
888 self.process_liveness = Some(process_liveness);
889 self
890 }
891
892 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
893 self.supervisor = supervisor;
894 self
895 }
896
897 pub fn with_daemon_provenance(
898 mut self,
899 pid: u32,
900 started_at_ms: u64,
901 executable_path: Option<PathBuf>,
902 build_git_sha: Option<String>,
903 build_lock_digest: Option<String>,
904 ) -> Self {
905 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
906 let process_start_time = process_start_time(pid);
907 self.daemon_provenance = DaemonProvenanceFacts {
908 build: DaemonBuildProvenance {
909 build_git_sha,
910 build_lock_digest,
911 },
912 pid: Some(pid),
913 started_at_ms: Some(started_at_ms),
914 start_clock: None,
915 executable_path,
916 executable_identity,
917 process_start_time,
918 probe: ExecutableIdentityProbe::default(),
919 };
920 self
921 }
922
923 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
924 self.daemon_provenance.start_clock = Some(clock);
925 self
926 }
927
928 #[cfg(test)]
929 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
930 self.provenance_probe_override = Some(result);
931 self
932 }
933
934 pub fn with_capability_config(
938 self,
939 modules: impl IntoIterator<Item = (String, bool)>,
940 reserved_capabilities: BTreeMap<String, String>,
941 ) -> Self {
942 self.capability_evaluator
943 .configure(modules, reserved_capabilities);
944 self
945 }
946
947 pub fn with_supervisor_rescan(
948 mut self,
949 supervisor: Supervisor,
950 config_path: impl Into<PathBuf>,
951 configured_port: Option<u16>,
952 ) -> Self {
953 self.rescan = Some(SupervisorRescanContext {
954 supervisor,
955 config_path: config_path.into(),
956 configured_port,
957 storage_config: self.storage_config.clone(),
958 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
959 admission_facts_targets: self.admission_facts_targets.clone(),
960 scope_authority_owners: self.scope_authority_owners.clone(),
961 });
962 self
963 }
964
965 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
966 self.connected_clients = connected_clients;
967 self
968 }
969
970 pub fn forwarding(&self) -> Arc<ForwardingTable> {
971 Arc::clone(&self.forwarding)
972 }
973
974 pub(crate) fn counters(&self) -> DaemonCounters {
975 self.counters.clone()
976 }
977
978 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
981 tokio::spawn(async move {
982 loop {
983 self.capability_evaluator
984 .wait_for_change_or_deadline()
985 .await;
986 self.refresh_capability_requirements();
987 }
988 });
989 }
990
991 fn runtime_capability_snapshot(
992 &self,
993 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
994 let runtime = self
995 .supervisor
996 .list()
997 .into_iter()
998 .map(|module| {
999 let status = module.status().map_err(|err| {
1000 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1001 })?;
1002 Ok(RuntimeModule {
1003 module_id: status.module_id,
1004 state: status.state,
1005 enabled: status.enabled,
1006 })
1007 })
1008 .collect::<Result<Vec<_>, RouterError>>()?;
1009 let (_, registrations) = self.registry.list_modules().map_err(|err| {
1010 RouterError::backend(
1011 0,
1012 0,
1013 format!("failed to list capability registrations: {err}"),
1014 )
1015 })?;
1016 let registrations = registrations
1017 .into_iter()
1018 .map(|registration| RegisteredModule {
1019 module_id: registration.manifest.module_id,
1020 module_version: registration.manifest.module_version,
1021 capabilities: registration.manifest.capabilities,
1022 })
1023 .collect();
1024 Ok((runtime, registrations))
1025 }
1026
1027 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1034 let cached_registration = RegisteredModule {
1035 module_id: registration.manifest.module_id.clone(),
1036 module_version: registration.manifest.module_version.clone(),
1037 capabilities: registration.manifest.capabilities.clone(),
1038 };
1039 if self.capability_evaluator.record_hello(&cached_registration) {
1040 warn!(
1041 module_id = %cached_registration.module_id,
1042 "capability claims drifted from the cached manifest"
1043 );
1044 }
1045 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1046 self.enforce_capability_denies();
1047 }
1048 self.refresh_capability_requirements();
1049 }
1050
1051 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1055 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1056 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1057 self.supervisor.set_swap_promotion_observer(observer);
1058 }
1059
1060 pub fn refresh_capability_requirements(&self) {
1061 match self.runtime_capability_snapshot() {
1062 Ok((runtime, registrations)) => {
1063 log_requirement_events(
1064 self.capability_evaluator
1065 .evaluate_now(&runtime, ®istrations),
1066 );
1067 }
1068 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1069 }
1070 }
1071
1072 fn enforce_capability_denies(&self) {
1076 let (_, registrations) = match self.registry.list_modules() {
1077 Ok(snapshot) => snapshot,
1078 Err(err) => {
1079 warn!(error = %err, "failed to read registrations for capability deny census");
1080 return;
1081 }
1082 };
1083 let manifests = registrations
1084 .into_iter()
1085 .map(|registration| {
1086 (
1087 registration.manifest.module_id.clone(),
1088 registration.manifest,
1089 )
1090 })
1091 .collect::<BTreeMap<_, _>>();
1092 let census = match self.forwarding.route_census(None) {
1093 Ok(census) => census,
1094 Err(err) => {
1095 warn!(error = %err, "failed to read route census for capability deny enforcement");
1096 return;
1097 }
1098 };
1099
1100 for (target_module_id, routes) in census {
1101 let Some(target_manifest) = manifests.get(&target_module_id) else {
1102 continue;
1103 };
1104 let mut closed_routes = Vec::new();
1105 let mut module_goodbyes = Vec::new();
1106 for route in routes {
1107 let Principal::Reserved {
1108 module_id: opening_module_id,
1109 } = &route.principal
1110 else {
1111 continue;
1112 };
1113 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1114 continue;
1115 };
1116 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1117 continue;
1118 };
1119
1120 match self.forwarding.release_client_route(
1121 route.goodbye_target.connection_id,
1122 route.goodbye_target.channel,
1123 route.goodbye_target.epoch,
1124 ) {
1125 Ok(RouteRelease::Removed(module_goodbye)) => {
1126 warn!(
1127 opening_module_id,
1128 target_module_id,
1129 capability,
1130 "force-closing route because an attested capability deny edge now matches"
1131 );
1132 closed_routes.push(route);
1133 module_goodbyes.push(module_goodbye);
1134 }
1135 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1136 Err(err) => warn!(
1137 opening_module_id,
1138 target_module_id,
1139 capability,
1140 error = %err,
1141 "failed to force-close capability-denied route"
1142 ),
1143 }
1144 }
1145
1146 if closed_routes.is_empty() {
1147 continue;
1148 }
1149 send_route_control_pushes(
1150 &self.forwarding,
1151 closed_routes,
1152 ClientControlPush::RouteClosed {
1153 module_id: target_module_id,
1154 channels: Vec::new(),
1155 reason: RouteCloseReason::CapabilityDenied,
1156 drained: false,
1157 abandoned: 0,
1158 excluded_subscriptions: 0,
1159 terminal: Some(false),
1160 },
1161 );
1162 self.emit_route_goodbyes(module_goodbyes);
1163 }
1164 }
1165
1166 fn not_ready_reason(
1172 &self,
1173 registration: &crate::registry::ModuleRegistration,
1174 ) -> Option<NotReadyReason> {
1175 if !registration.ready {
1176 return Some(NotReadyReason {
1177 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1178 capability: None,
1179 });
1180 }
1181 self.first_unprovided_required_capability(registration)
1182 .map(|capability| NotReadyReason {
1183 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1184 capability: Some(capability),
1185 })
1186 }
1187
1188 fn first_unprovided_required_capability(
1206 &self,
1207 registration: &crate::registry::ModuleRegistration,
1208 ) -> Option<String> {
1209 let required = registration
1210 .manifest
1211 .capabilities
1212 .iter()
1213 .flat_map(|declarations| declarations.requires.iter())
1214 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1215 .map(|requirement| requirement.capability.as_str())
1216 .collect::<BTreeSet<_>>();
1217 if required.is_empty() {
1218 return None;
1219 }
1220 let module_id = registration.manifest.module_id.as_str();
1221 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1222 if required
1223 .iter()
1224 .any(|capability| verdict(capability).is_none())
1225 {
1226 self.refresh_capability_requirements();
1227 }
1228 required
1229 .into_iter()
1230 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1231 .map(str::to_string)
1232 }
1233
1234 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1235 self.capability_evaluator
1236 .statuses()
1237 .into_iter()
1238 .map(capability_requirement_status)
1239 .collect()
1240 }
1241
1242 fn deregister_connection(
1249 &self,
1250 connection_id: ConnectionId,
1251 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1252 self.registry.deregister_connection(connection_id)
1253 }
1254
1255 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1256 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1257 return None;
1258 }
1259 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1260 parse_client_control_request(&frame.body)
1261 else {
1262 return None;
1263 };
1264 Some(target_module_id(&target).to_string())
1265 }
1266
1267 pub(crate) fn route_open_capacity_refusal(
1268 &self,
1269 ctx: &RouteCtx,
1270 frame: &Frame,
1271 target_module_id: &str,
1272 in_flight: usize,
1273 limit: usize,
1274 ) -> Result<Frame, RouterError> {
1275 self.route_open_admission_refusal_frame(
1276 ctx,
1277 frame,
1278 target_module_id,
1279 "open_admission_full",
1280 (in_flight, limit),
1281 format!(
1282 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1283 ),
1284 )
1285 }
1286
1287 fn route_open_target_capacity_refusal(
1288 &self,
1289 ctx: &RouteCtx,
1290 frame: &Frame,
1291 target_module_id: &str,
1292 in_flight: usize,
1293 ) -> Result<Frame, RouterError> {
1294 self.route_open_admission_refusal_frame(
1295 ctx,
1296 frame,
1297 target_module_id,
1298 "target_binds_full",
1299 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1300 format!(
1301 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1302 ),
1303 )
1304 }
1305
1306 fn route_open_admission_refusal_frame(
1314 &self,
1315 ctx: &RouteCtx,
1316 frame: &Frame,
1317 target_module_id: &str,
1318 reason: &'static str,
1319 (in_flight, limit): (usize, usize),
1320 message: impl Into<String>,
1321 ) -> Result<Frame, RouterError> {
1322 let code = error_codes::TARGET_UNAVAILABLE;
1323 self.counters.increment_route_open_refused(code);
1324 info!(
1325 target: "control",
1326 code,
1327 reason,
1328 module_id = ?target_module_id,
1329 connection_id = ctx.connection_id.get(),
1330 in_flight,
1331 limit,
1332 "route.open refused"
1333 );
1334 control_error_frame(frame, code, message.into())
1335 }
1336
1337 #[cfg(test)]
1343 pub fn handle_control(
1344 &self,
1345 connection_id: ConnectionId,
1346 frame: Frame,
1347 ) -> Result<Vec<Frame>, RouterError> {
1348 match frame.header.ty {
1349 FrameType::Ping => Ok(vec![pong(&frame)?]),
1350 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1351 FrameType::Goodbye => self.handle_goodbye(connection_id),
1352 ty => Ok(vec![control_error_frame(
1353 &frame,
1354 "unsupported_control_frame",
1355 format!("unsupported channel-0 frame {ty:?}"),
1356 )?]),
1357 }
1358 }
1359
1360 pub async fn handle_control_frame(
1361 &self,
1362 ctx: &RouteCtx,
1363 frame: Frame,
1364 ) -> Result<Vec<Frame>, RouterError> {
1365 self.handle_control_frame_timed(ctx, frame, None).await
1366 }
1367
1368 pub(crate) async fn handle_control_frame_timed(
1369 &self,
1370 ctx: &RouteCtx,
1371 frame: Frame,
1372 dispatch_started_at: Option<StdInstant>,
1373 ) -> Result<Vec<Frame>, RouterError> {
1374 match frame.header.ty {
1375 FrameType::Ping => Ok(vec![pong(&frame)?]),
1376 FrameType::Hello => {
1377 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1378 }
1379 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1380 FrameType::Cancel => {
1381 if self
1382 .supervisor
1383 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1384 {
1385 Ok(Vec::new())
1386 } else {
1387 Ok(vec![control_error_frame(
1388 &frame,
1389 "unknown_subscription",
1390 "no supervisor spawn subscription has this correlation id",
1391 )?])
1392 }
1393 }
1394 FrameType::Request => {
1395 if self
1396 .forwarding
1397 .module_endpoint_for_connection(ctx.connection_id)
1398 .map_err(RouterError::Forwarding)?
1399 .is_some()
1400 {
1401 if !is_known_module_request_op(&frame.body) {
1402 return Ok(vec![control_error_frame(
1403 &frame,
1404 "unsupported_control_frame",
1405 "module-originated channel-0 REQUEST is not supported",
1406 )?]);
1407 }
1408 let request = match parse_module_control_request_from_module(&frame.body) {
1409 Ok(request) => request,
1410 Err((err, ControlRequestBodyError::UnknownOp)) => {
1411 return Ok(vec![control_error_frame(
1412 &frame,
1413 "unsupported_control_frame",
1414 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1415 )?])
1416 }
1417 Err((err, ControlRequestBodyError::InvalidBody)) => {
1418 return Ok(vec![control_error_frame(
1419 &frame,
1420 "invalid_control_body",
1421 format!("malformed module control body: {err}"),
1422 )?])
1423 }
1424 };
1425 let op = module_control_request_op(&request);
1426 let corr = frame.header.corr;
1427 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1428 let result =
1429 self.handle_module_control_request(ctx.connection_id, frame, request);
1430 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1431 return result;
1432 }
1433
1434 if is_known_module_request_op(&frame.body) {
1435 return Ok(vec![control_error_frame(
1436 &frame,
1437 "not_registered",
1438 "catalog.update requires an active module registration owned by this connection",
1439 )?]);
1440 }
1441
1442 let request = match parse_client_control_request(&frame.body) {
1443 Ok(request) => request,
1444 Err((err, ControlRequestBodyError::UnknownOp)) => {
1445 return Ok(vec![control_error_frame(
1446 &frame,
1447 "unknown_control_op",
1448 format!("unknown client control op: {err}"),
1449 )?])
1450 }
1451 Err((err, ControlRequestBodyError::InvalidBody)) => {
1452 return Ok(vec![control_error_frame(
1453 &frame,
1454 "invalid_control_body",
1455 format!("malformed client control body: {err}"),
1456 )?])
1457 }
1458 };
1459 let op = client_control_request_op(&request);
1460 let corr = frame.header.corr;
1461 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1462 #[cfg(test)]
1463 if let Some(delay) = self.control_dispatch_delay {
1464 tokio::time::sleep(delay).await;
1465 }
1466 let result = self
1467 .handle_client_control_request(ctx, frame, request)
1468 .await;
1469 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1470 result
1471 }
1472 FrameType::Push => {
1473 let Some(endpoint) = self
1474 .forwarding
1475 .module_endpoint_for_connection(ctx.connection_id)
1476 .map_err(RouterError::Forwarding)?
1477 else {
1478 return Ok(vec![control_error_frame(
1479 &frame,
1480 "unsupported_control_frame",
1481 "client-originated channel-0 PUSH is not supported",
1482 )?]);
1483 };
1484 self.handle_status_update(endpoint, frame)
1485 }
1486 FrameType::Response | FrameType::Error
1487 if self
1488 .forwarding
1489 .module_endpoint_for_connection(ctx.connection_id)
1490 .map_err(RouterError::Forwarding)?
1491 .is_some() =>
1492 {
1493 self.handle_module_relay_response(ctx.connection_id, frame)
1494 }
1495 ty => Ok(vec![control_error_frame(
1496 &frame,
1497 "unsupported_control_frame",
1498 format!("unsupported channel-0 frame {ty:?}"),
1499 )?]),
1500 }
1501 }
1502
1503 pub fn cleanup_connection(
1504 &self,
1505 connection_id: ConnectionId,
1506 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1507 let crash_closed = self
1508 .registry
1509 .get_module_by_connection(connection_id)?
1510 .and_then(|registration| {
1511 self.forwarding
1512 .module_endpoint_for_connection(connection_id)
1513 .ok()
1514 .flatten()
1515 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1516 .map(|routes| (registration.manifest.module_id, routes))
1517 });
1518 let crash_closed = crash_closed.map(|(module_id, routes)| {
1519 let terminal = match self.supervisor.get(&module_id) {
1520 None => false,
1521 Some(module) => match module.will_recover_after_connection_loss() {
1522 Ok(will_recover) => !will_recover,
1523 Err(err) => {
1524 warn!(
1525 %module_id,
1526 error = %err,
1527 "failed to read crash recovery verdict; reporting non-terminal conservatively"
1528 );
1529 false
1530 }
1531 },
1532 };
1533 let reason = match self.forwarding.is_daemon_draining() {
1537 Ok(true) => RouteCloseReason::Restart,
1538 Ok(false) => RouteCloseReason::Crash,
1539 Err(err) => {
1540 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
1541 RouteCloseReason::Crash
1542 }
1543 };
1544 (module_id, routes, reason, terminal)
1545 });
1546 let registrations = self.deregister_connection(connection_id);
1547 let cleanup = self.forwarding.cleanup_connection_counted(connection_id);
1548 if let Some((module_id, routes, reason, terminal)) = crash_closed {
1553 let abandoned = cleanup
1554 .as_ref()
1555 .map_or(0, |cleanup| cleanup.abandoned_relays);
1556 send_route_control_pushes(
1557 &self.forwarding,
1558 routes,
1559 ClientControlPush::RouteClosed {
1560 module_id,
1561 channels: Vec::new(),
1562 reason,
1563 drained: false,
1564 abandoned,
1565 excluded_subscriptions: 0,
1566 terminal: Some(terminal),
1567 },
1568 );
1569 }
1570 if let Ok(cleanup) = cleanup {
1571 self.emit_route_goodbyes(cleanup.released);
1572 }
1573 if matches!(®istrations, Ok(r) if !r.is_empty()) {
1577 crate::supervise::notify_registration_release();
1578 self.capability_evaluator.wake_deadline_loop();
1579 self.refresh_capability_requirements();
1580 }
1581 self.supervisor.remove_spawn_subscribers(connection_id);
1582 self.hello_launch_nonces
1585 .lock()
1586 .unwrap_or_else(|poisoned| poisoned.into_inner())
1587 .forget(connection_id);
1588 self.scopes
1589 .write()
1590 .unwrap_or_else(|poisoned| poisoned.into_inner())
1591 .release_connection(connection_id);
1592 registrations
1593 }
1594
1595 pub(crate) fn handle_route_goodbye(
1596 &self,
1597 connection_id: ConnectionId,
1598 route_channel: u16,
1599 route_epoch: u32,
1600 ) -> Result<bool, RouterError> {
1601 debug!(
1602 connection_id = connection_id.get(),
1603 route_channel, route_epoch, "handling route GOODBYE"
1604 );
1605 let RouteRelease::Removed(released_route) = self
1606 .forwarding
1607 .release_client_route(connection_id, route_channel, route_epoch)
1608 .map_err(RouterError::Forwarding)?
1609 else {
1610 return Ok(false);
1611 };
1612 self.emit_route_goodbyes(vec![released_route]);
1613 Ok(true)
1614 }
1615
1616 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
1617 for released in released_routes {
1618 let frame = match Frame::build_with_version(
1619 released.negotiated_ver,
1620 FrameType::Goodbye,
1621 control_flags(),
1622 released.channel,
1623 released.epoch,
1624 0,
1625 Vec::new(),
1626 ) {
1627 Ok(frame) => frame,
1628 Err(err) => {
1629 warn!(
1630 route_channel = released.channel,
1631 error = %err,
1632 "failed to build route GOODBYE frame"
1633 );
1634 continue;
1635 }
1636 };
1637 if !released.close_on_delivery_failure() {
1638 crate::forwarding::send_module_route_goodbye(
1639 &self.counters,
1640 &released.sink,
1641 frame,
1642 released.module_id.as_deref(),
1643 "client route released",
1644 );
1645 continue;
1646 }
1647 if let Err(err) = released.sink.try_send(frame) {
1648 warn!(
1649 target_connection_id = released.connection_id.get(),
1650 route_channel = released.channel,
1651 error = %err,
1652 "route GOODBYE was not delivered to client; closing target connection"
1653 );
1654 if self
1655 .forwarding
1656 .escalate_client_delivery_failure(
1657 released.connection_id,
1658 released.channel,
1659 released.epoch,
1660 CloseReason::new(
1661 "route_goodbye_delivery_failed",
1662 format!(
1663 "failed to enqueue route GOODBYE for channel {}: {err}",
1664 released.channel
1665 ),
1666 ),
1667 crate::forwarding::UndeliveredFrame {
1668 module_id: released.module_id.as_deref(),
1669 sink: &released.sink,
1670 },
1671 )
1672 .unwrap_or(false)
1673 {
1674 self.counters.increment_goodbye_relay_client_failed();
1675 }
1676 }
1677 }
1678 }
1679
1680 fn send_abandoned_route_bind_goodbye(
1692 &self,
1693 module_sink: &crate::FrameSink,
1694 negotiated_ver: u8,
1695 module_channel: u16,
1696 module_epoch: u32,
1697 ) {
1698 let frame = match Frame::build_with_version(
1699 negotiated_ver,
1700 FrameType::Goodbye,
1701 control_flags(),
1702 module_channel,
1703 module_epoch,
1704 0,
1705 Vec::new(),
1706 ) {
1707 Ok(frame) => frame,
1708 Err(err) => {
1709 warn!(
1710 route_channel = module_channel,
1711 error = %err,
1712 "failed to build GOODBYE for abandoned route.bind"
1713 );
1714 return;
1715 }
1716 };
1717 crate::forwarding::send_module_route_goodbye(
1718 &self.counters,
1719 module_sink,
1720 frame,
1721 None,
1722 "abandoned route.bind",
1723 );
1724 }
1725
1726 fn handle_hello(
1727 &self,
1728 connection_id: ConnectionId,
1729 sink: Option<crate::FrameSink>,
1730 frame: Frame,
1731 ) -> Result<Vec<Frame>, RouterError> {
1732 debug!(
1733 connection_id = connection_id.get(),
1734 corr = frame.header.corr,
1735 "handling HELLO"
1736 );
1737 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
1738 Ok(value) => value,
1739 Err(err) => {
1740 return Ok(vec![control_error_frame(
1741 &frame,
1742 "invalid_hello",
1743 format!("malformed HELLO body: {err}"),
1744 )?])
1745 }
1746 };
1747 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
1748 return Ok(vec![control_error_frame(
1749 &frame,
1750 "invalid_capability_grammar",
1751 err.to_string(),
1752 )?]);
1753 }
1754 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
1755 return Ok(vec![control_error_frame(
1756 &frame,
1757 "invalid_manifest",
1758 err.to_string(),
1759 )?]);
1760 }
1761 if let Some(provenance) = hello_value
1762 .get("manifest")
1763 .and_then(|manifest| manifest.get("provenance"))
1764 {
1765 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
1766 return Ok(vec![control_error_frame(
1767 &frame,
1768 "invalid_manifest",
1769 format!("malformed manifest provenance: {err}"),
1770 )?]);
1771 }
1772 }
1773 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
1774 Ok(hello) => hello,
1775 Err(err) => {
1776 return Ok(vec![control_error_frame(
1777 &frame,
1778 "invalid_hello",
1779 format!("malformed HELLO body: {err}"),
1780 )?])
1781 }
1782 };
1783
1784 if hello.protocol_ver != hello.manifest.protocol_ver {
1785 return Ok(vec![control_error_frame(
1786 &frame,
1787 "invalid_manifest",
1788 format!(
1789 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
1790 hello.protocol_ver, hello.manifest.protocol_ver
1791 ),
1792 )?]);
1793 }
1794
1795 if hello.manifest.module_id.trim().is_empty() {
1796 return Ok(vec![control_error_frame(
1797 &frame,
1798 "invalid_manifest",
1799 "manifest module_id must not be empty",
1800 )?]);
1801 }
1802
1803 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
1804 Ok(negotiated_ver) => negotiated_ver,
1805 Err(message) => {
1806 return Ok(vec![control_error_frame(
1807 &frame,
1808 "version_unsupported",
1809 message,
1810 )?])
1811 }
1812 };
1813
1814 let swap_admission = self
1822 .supervisor
1823 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
1824 if swap_admission == SwapHelloAdmission::Refused {
1825 warn!(
1826 module_id = %hello.manifest.module_id,
1827 connection_id = connection_id.get(),
1828 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
1829 );
1830 return Ok(vec![control_error_frame(
1831 &frame,
1832 "swap_token_invalid",
1833 format!(
1834 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
1835 hello.manifest.module_id
1836 ),
1837 )?]);
1838 }
1839 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
1840
1841 if let Some(rejection) = (!swap_candidate)
1849 .then(|| {
1850 self.supervisor.reserved_hello_rejection(
1851 &hello.manifest.module_id,
1852 hello.launch_nonce.as_deref(),
1853 )
1854 })
1855 .flatten()
1856 {
1857 let message = match rejection {
1858 ReservedHelloRejection::Exact { module_id } => format!(
1859 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
1860 ),
1861 ReservedHelloRejection::Prefix {
1862 prefix,
1863 owner_module_id,
1864 } => format!(
1865 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
1866 hello.manifest.module_id
1867 ),
1868 };
1869 return Ok(vec![control_error_frame(
1870 &frame,
1871 "reserved_module",
1872 message,
1873 )?]);
1874 }
1875
1876 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
1877 &hello.manifest.module_id,
1878 hello.manifest.capabilities.as_ref(),
1879 );
1880 if let Some(refusal) = reserved_capability_refusals.first() {
1881 let capability = refusal.capability.clone();
1882 let bound_module = refusal.claimants[0].clone();
1883 log_duplicate_claim_events(reserved_capability_refusals);
1884 return Ok(vec![control_error_frame(
1885 &frame,
1886 "reserved_capability",
1887 format!(
1888 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
1889 capability, bound_module, hello.manifest.module_id
1890 ),
1891 )?]);
1892 }
1893
1894 if self
1897 .forwarding
1898 .connection_has_client_routes(connection_id)
1899 .map_err(RouterError::Forwarding)?
1900 {
1901 return Ok(vec![control_error_frame(
1902 &frame,
1903 "invalid_hello",
1904 "connection has open client routes and cannot also register as a module",
1905 )?]);
1906 }
1907
1908 self.hello_launch_nonces
1913 .lock()
1914 .unwrap_or_else(|poisoned| poisoned.into_inner())
1915 .record(connection_id, hello.launch_nonce.as_deref());
1916 let control_ops = effective_module_control_ops(hello.control_ops);
1917 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
1920 if swap_candidate {
1921 return self.register_swap_candidate(
1922 connection_id,
1923 sink,
1924 &frame,
1925 hello.manifest,
1926 negotiated_ver,
1927 control_ops,
1928 hello_ack,
1929 );
1930 }
1931 let registration = match self.registry.register_with_control_ops(
1932 hello.manifest,
1933 negotiated_ver,
1934 connection_id,
1935 control_ops,
1936 ) {
1937 Ok(registration) => registration,
1938 Err(RegistryError::DuplicateModuleId { module_id }) => {
1939 return Ok(vec![control_error_frame(
1940 &frame,
1941 "duplicate_module_id",
1942 format!(
1943 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
1944 ),
1945 )?])
1946 }
1947 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
1948 return Ok(vec![control_error_frame(
1949 &frame,
1950 "invalid_module_id",
1951 err.to_string(),
1952 )?])
1953 }
1954 Err(err) => {
1955 return Ok(vec![control_error_frame(
1956 &frame,
1957 "registry_error",
1958 err.to_string(),
1959 )?])
1960 }
1961 };
1962
1963 let reply = if let Some(sink) = sink {
1964 let concurrency = manifest_concurrency(®istration.manifest);
1980 if let Err(err) = self.forwarding.register_module_connection_acked(
1981 connection_id,
1982 registration.manifest.module_id.clone(),
1983 negotiated_ver,
1984 concurrency,
1985 sink,
1986 hello_ack,
1987 ) {
1988 if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
1992 crate::supervise::notify_registration_release();
1993 }
1994 return Ok(vec![control_error_frame(
1995 &frame,
1996 forwarding_error_code(&err),
1997 err.to_string(),
1998 )?]);
1999 }
2000 Vec::new()
2001 } else {
2002 vec![hello_ack]
2005 };
2006
2007 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2016 info!(
2017 module_id = %registration.manifest.module_id,
2018 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2019 );
2020 }
2021
2022 self.apply_registration_capabilities(®istration);
2023
2024 info!(
2025 module_id = %registration.manifest.module_id,
2026 module_version = %registration.manifest.module_version,
2027 negotiated_ver,
2028 routable_provider = manifest_provides_routable_role(®istration.manifest),
2029 connection_id = connection_id.get(),
2030 "module registered"
2031 );
2032
2033 Ok(reply)
2034 }
2035
2036 #[allow(clippy::too_many_arguments)]
2046 fn register_swap_candidate(
2047 &self,
2048 connection_id: ConnectionId,
2049 sink: Option<crate::FrameSink>,
2050 frame: &Frame,
2051 manifest: ModuleManifest,
2052 negotiated_ver: u8,
2053 control_ops: Vec<String>,
2054 hello_ack: Frame,
2055 ) -> Result<Vec<Frame>, RouterError> {
2056 let module_id = manifest.module_id.clone();
2057 let registration = match self.registry.register_candidate_with_control_ops(
2058 manifest,
2059 negotiated_ver,
2060 connection_id,
2061 control_ops,
2062 ) {
2063 Ok(registration) => registration,
2064 Err(RegistryError::DuplicateModuleId { module_id }) => {
2065 return Ok(vec![control_error_frame(
2066 frame,
2067 "duplicate_module_id",
2068 format!(
2069 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2070 ),
2071 )?])
2072 }
2073 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2074 return Ok(vec![control_error_frame(
2075 frame,
2076 "invalid_module_id",
2077 err.to_string(),
2078 )?])
2079 }
2080 Err(err) => {
2081 return Ok(vec![control_error_frame(
2082 frame,
2083 "registry_error",
2084 err.to_string(),
2085 )?])
2086 }
2087 };
2088 let reply = if let Some(sink) = sink {
2089 let concurrency = manifest_concurrency(®istration.manifest);
2093 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2094 connection_id,
2095 module_id.clone(),
2096 negotiated_ver,
2097 concurrency,
2098 sink,
2099 hello_ack,
2100 ) {
2101 if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
2102 crate::supervise::notify_registration_release();
2103 }
2104 return Ok(vec![control_error_frame(
2105 frame,
2106 forwarding_error_code(&err),
2107 err.to_string(),
2108 )?]);
2109 }
2110 Vec::new()
2111 } else {
2112 vec![hello_ack]
2113 };
2114 self.supervisor.mark_swap_candidate_admitted(&module_id);
2115 info!(
2116 module_id = %module_id,
2117 module_version = %registration.manifest.module_version,
2118 negotiated_ver,
2119 ready = registration.ready,
2120 connection_id = connection_id.get(),
2121 "swap candidate registered; not routable until cutover"
2122 );
2123 Ok(reply)
2124 }
2125
2126 fn build_hello_ack(
2127 &self,
2128 frame: &Frame,
2129 negotiated_ver: u8,
2130 module_id: &str,
2131 ) -> Result<Frame, RouterError> {
2132 let ack = ModuleHelloAckBody {
2133 negotiated_ver,
2134 subc_ops: module_subc_ops(),
2135 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2136 storage: self
2137 .storage_config
2138 .as_ref()
2139 .map(|cfg| cfg.descriptor_for(module_id)),
2140 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2141 };
2142 let body = serde_json::to_vec(&ack).map_err(|err| {
2143 RouterError::backend(
2144 0,
2145 frame.header.corr,
2146 format!("failed to encode HELLO_ACK: {err}"),
2147 )
2148 })?;
2149
2150 Frame::build_with_version(
2151 negotiated_ver,
2152 FrameType::HelloAck,
2153 control_flags(),
2154 0,
2155 0,
2156 frame.header.corr,
2157 body,
2158 )
2159 .map_err(RouterError::FrameBuild)
2160 }
2161
2162 async fn handle_client_control_request(
2163 &self,
2164 ctx: &RouteCtx,
2165 frame: Frame,
2166 request: ClientControlRequest,
2167 ) -> Result<Vec<Frame>, RouterError> {
2168 match request {
2169 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2170 ClientControlRequest::CatalogList { module_id } => {
2171 self.handle_catalog_list(frame, module_id)
2172 }
2173 ClientControlRequest::RouteOpen {
2174 target,
2175 identity,
2176 consumer_identity,
2177 consumer_capabilities,
2178 role_versions,
2179 admission_facts,
2180 scope,
2181 } => {
2182 self.handle_route_open(
2183 ctx,
2184 frame,
2185 RouteOpenRequest {
2186 target,
2187 identity,
2188 consumer_identity,
2189 consumer_capabilities,
2190 role_versions,
2191 admission_facts,
2192 scope,
2193 },
2194 )
2195 .await
2196 }
2197 ClientControlRequest::RoutePoll {
2198 route_channel,
2199 route_epoch,
2200 kind,
2201 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2202 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2203 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2204 self.handle_supervisor_spawn_snapshot(frame)
2205 }
2206 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2207 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2208 }
2209 ClientControlRequest::SupervisorRestart {
2210 module_id,
2211 drain_timeout_ms,
2212 } => {
2213 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms)
2214 .await
2215 }
2216 ClientControlRequest::SupervisorSwap {
2217 module_id,
2218 ready_timeout_ms,
2219 } => {
2220 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms)
2221 .await
2222 }
2223 ClientControlRequest::SupervisorReload { module_id } => {
2224 self.handle_supervisor_reload(frame, module_id).await
2225 }
2226 ClientControlRequest::SupervisorRescan { preview } => {
2227 self.handle_supervisor_rescan(frame, preview).await
2228 }
2229 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2230 self.handle_supervisor_release_reserved(frame, module_id)
2231 .await
2232 }
2233 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2234 self.handle_supervisor_set_enabled(frame, module_id, enabled)
2235 .await
2236 }
2237 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2238 self.handle_supervisor_health_probe(frame, module_id).await
2239 }
2240 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2241 ClientControlRequest::SupervisorRoutes { module_id } => {
2242 self.handle_supervisor_routes(frame, module_id)
2243 }
2244 ClientControlRequest::SupervisorProvenance { module_id } => {
2245 self.handle_supervisor_provenance(frame, module_id).await
2246 }
2247 ClientControlRequest::SupervisorStderrTail {
2248 module_id,
2249 max_lines,
2250 max_bytes,
2251 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2252 ClientControlRequest::SupervisorTerminals { module_id } => {
2253 self.handle_supervisor_terminals(frame, module_id).await
2254 }
2255 }
2256 }
2257
2258 fn handle_module_control_request(
2259 &self,
2260 connection_id: ConnectionId,
2261 frame: Frame,
2262 request: ModuleControlRequestFromModule,
2263 ) -> Result<Vec<Frame>, RouterError> {
2264 match request {
2265 ModuleControlRequestFromModule::CatalogUpdate {
2266 provides,
2267 capabilities,
2268 ready,
2269 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2270 ModuleControlRequestFromModule::LiveRoots {} => {
2271 let registered = self
2272 .registry
2273 .get_module_by_connection(connection_id)
2274 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2275 let Some(registration) = registered else {
2276 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2277 };
2278 let response = self
2279 .forwarding
2280 .live_roots(®istration.manifest.module_id)
2281 .map_err(RouterError::Forwarding)?;
2282 Ok(vec![control_response_body_frame(
2283 &frame,
2284 &response,
2285 "ModuleControlResponseToModule::LiveRoots",
2286 )?])
2287 }
2288 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
2289 self.handle_scope_sync(connection_id, frame, generation, scopes)
2290 }
2291 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
2292 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
2293 }
2294 }
2295 }
2296
2297 fn handle_scope_sync(
2301 &self,
2302 connection_id: ConnectionId,
2303 frame: Frame,
2304 generation: u64,
2305 scopes: Vec<ScopeRecord>,
2306 ) -> Result<Vec<Frame>, RouterError> {
2307 let Some(registration) = self
2308 .registry
2309 .get_module_by_connection(connection_id)
2310 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2311 else {
2312 return Ok(vec![control_error_frame(
2313 &frame,
2314 "not_registered",
2315 "scope.sync requires an active module registration owned by this connection",
2316 )?]);
2317 };
2318 let owner = registration.manifest.module_id;
2319 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
2320 let is_current_launch = |connection: ConnectionId| {
2321 self.hello_launch_nonces
2322 .lock()
2323 .unwrap_or_else(|poisoned| poisoned.into_inner())
2324 .presented(connection, current_nonce.as_deref())
2325 };
2326 let mut table = self
2331 .scopes
2332 .write()
2333 .unwrap_or_else(|poisoned| poisoned.into_inner());
2334 let outcome = table.sync(&owner, connection_id, is_current_launch, generation, scopes);
2335 let drained = match &outcome {
2336 Ok(applied) => self
2337 .forwarding
2338 .publish_scope_changes(&applied.tag_changes)
2339 .map_err(RouterError::Forwarding)?,
2340 Err(_) => Vec::new(),
2341 };
2342 drop(table);
2343 match outcome {
2344 Ok(applied) => {
2345 let counts = ScopeOutcomeCounts::of(&applied.results);
2346 info!(
2347 owner = %owner,
2348 generation,
2349 records = applied.results.len(),
2350 created = counts.created,
2351 replaced = counts.replaced,
2352 updated = counts.updated,
2353 unchanged = counts.unchanged,
2354 refused = counts.refused,
2355 ended = applied.ended.len(),
2356 tag_changes = applied.tag_changes.len(),
2357 routes_closed = drained.len(),
2358 "scope sync accepted"
2359 );
2360 for refused in applied
2366 .results
2367 .iter()
2368 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
2369 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
2370 {
2371 warn!(
2372 owner = %owner,
2373 generation,
2374 scope_ref = %refused.scope_ref,
2375 scope_epoch = refused.scope_epoch,
2376 code = refused.code.as_deref().unwrap_or(""),
2377 "scope record refused"
2378 );
2379 }
2380 self.close_scope_drained_routes(drained);
2381 let response = ModuleControlResponseToModule::ScopeSync {
2382 generation,
2383 results: applied.results,
2384 ended: applied.ended,
2385 };
2386 Ok(vec![control_response_body_frame(
2387 &frame,
2388 &response,
2389 "ModuleControlResponseToModule::ScopeSync",
2390 )?])
2391 }
2392 Err(refusal) => {
2393 info!(
2394 owner = %owner,
2395 generation,
2396 code = refusal.code,
2397 "scope sync refused"
2398 );
2399 Ok(vec![control_error_frame(
2400 &frame,
2401 refusal.code,
2402 refusal.message,
2403 )?])
2404 }
2405 }
2406 }
2407
2408 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
2414 if drained.is_empty() {
2415 return;
2416 }
2417 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
2418 BTreeMap::new();
2419 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
2420 for route in drained {
2421 warn!(
2422 module_id = %route.module_id,
2423 reason = ?route.reason,
2424 client_connection_id = route.client.connection_id.get(),
2425 route_channel = route.client.channel,
2426 "closing route because its scope changed"
2427 );
2428 pushes
2429 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
2430 .or_insert_with(|| (route.reason, Vec::new()))
2431 .1
2432 .push(EndpointRoute {
2433 goodbye_target: route.client.clone(),
2434 principal: Principal::Unverified,
2435 bound_at: Instant::now(),
2436 draining: false,
2437 drain_reason: None,
2438 });
2439 goodbyes.push(route.module);
2440 goodbyes.push(route.client);
2441 }
2442 for ((module_id, _), (reason, routes)) in pushes {
2443 send_route_control_pushes(
2444 &self.forwarding,
2445 routes,
2446 ClientControlPush::RouteClosed {
2447 module_id,
2448 channels: Vec::new(),
2449 reason,
2450 drained: false,
2451 abandoned: 0,
2452 excluded_subscriptions: 0,
2453 terminal: Some(false),
2454 },
2455 );
2456 }
2457 self.emit_route_goodbyes(goodbyes);
2458 }
2459
2460 fn handle_scope_describe(
2463 &self,
2464 connection_id: ConnectionId,
2465 frame: Frame,
2466 owner: Principal,
2467 scope_ref: String,
2468 ) -> Result<Vec<Frame>, RouterError> {
2469 let registered = self
2470 .registry
2471 .get_module_by_connection(connection_id)
2472 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2473 if registered.is_none() {
2474 return Ok(vec![control_error_frame(
2475 &frame,
2476 "not_registered",
2477 "scope.describe requires an active module registration owned by this connection",
2478 )?]);
2479 }
2480 let description = self
2481 .scopes
2482 .read()
2483 .unwrap_or_else(|poisoned| poisoned.into_inner())
2484 .describe(&owner, &scope_ref);
2485 let owner_configured = match &owner {
2486 Principal::Reserved { module_id } => self.supervisor.get(module_id).is_some(),
2487 _ => false,
2488 };
2489 let response = ModuleControlResponseToModule::ScopeDescribe {
2490 status: description.status,
2491 scope_epoch: description.scope_epoch,
2492 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
2493 owner_synced: description.owner_synced,
2494 owner_configured,
2495 scope: description.stamp,
2496 };
2497 Ok(vec![control_response_body_frame(
2498 &frame,
2499 &response,
2500 "ModuleControlResponseToModule::ScopeDescribe",
2501 )?])
2502 }
2503
2504 fn handle_catalog_update(
2505 &self,
2506 connection_id: ConnectionId,
2507 frame: Frame,
2508 provides: Vec<ProviderRole>,
2509 capabilities: Option<CapabilityDeclarations>,
2510 ready: Option<bool>,
2511 ) -> Result<Vec<Frame>, RouterError> {
2512 self.refresh_capability_requirements();
2513 let Some(registration) = self
2514 .registry
2515 .get_module_by_connection(connection_id)
2516 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2517 else {
2518 return Ok(vec![control_error_frame(
2519 &frame,
2520 "not_registered",
2521 "catalog.update requires an active module registration owned by this connection",
2522 )?]);
2523 };
2524
2525 if let Some(message) =
2526 catalog_update_frozen_field_message(®istration.manifest, &provides)
2527 {
2528 return Ok(vec![control_error_frame(
2529 &frame,
2530 "catalog_update_frozen_field",
2531 message,
2532 )?]);
2533 }
2534
2535 let mut candidate = registration.manifest.clone();
2536 candidate.provides = provides.clone();
2537 candidate.capabilities = capabilities
2538 .clone()
2539 .or_else(|| registration.manifest.capabilities.clone());
2540 if let Err(err) = candidate.validate_capability_grammar() {
2541 return Ok(vec![control_error_frame(
2542 &frame,
2543 "invalid_capability_grammar",
2544 err.to_string(),
2545 )?]);
2546 }
2547
2548 let updated = self
2549 .registry
2550 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
2551 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2552 if updated.is_none() {
2553 return Ok(vec![control_error_frame(
2554 &frame,
2555 "not_registered",
2556 "catalog.update requires an active module registration owned by this connection",
2557 )?]);
2558 }
2559 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
2560 log_duplicate_claim_events(
2561 self.capability_evaluator
2562 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
2563 );
2564 }
2565 if capability_census_trigger(
2566 registration.manifest.capabilities.as_ref(),
2567 updated
2568 .as_ref()
2569 .and_then(|entry| entry.manifest.capabilities.as_ref()),
2570 ) {
2571 self.enforce_capability_denies();
2572 }
2573 self.refresh_capability_requirements();
2574
2575 let response = ModuleControlResponseToModule::CatalogUpdate {};
2576 control_response_body_frame(
2577 &frame,
2578 &response,
2579 "ModuleControlResponseToModule::CatalogUpdate",
2580 )
2581 .map(|frame| vec![frame])
2582 }
2583
2584 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
2585 self.refresh_capability_requirements();
2586 let mut counters = self.counters.snapshot();
2592 if let (Ok((connections_with_routes, max)), Some(obj)) = (
2593 self.forwarding.client_route_concentration(),
2594 counters.as_object_mut(),
2595 ) {
2596 obj.insert(
2597 "client_connections_with_routes".into(),
2598 connections_with_routes.into(),
2599 );
2600 obj.insert("max_routes_on_one_connection".into(), max.into());
2601 }
2602 if let (Some(open_breakers), Some(obj)) = (
2608 self.route_bind_breakers.open_snapshot(),
2609 counters.as_object_mut(),
2610 ) {
2611 obj.insert("route_bind_breakers_open".into(), open_breakers);
2612 }
2613 let response = ClientControlResponse::ServerDescribe {
2614 protocol_ver: PROTOCOL_VERSION,
2615 subc_ops: subc_ops(),
2616 capabilities: self.subc_capabilities.as_ref().to_vec(),
2617 connected_clients: self.connected_clients.count(),
2618 counters: Some(counters),
2619 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
2620 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
2621 capability_requirements: self.capability_requirement_statuses(),
2622 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2623 };
2624 Ok(vec![control_response_body_frame(
2625 &frame,
2626 &response,
2627 "ClientControlResponse::ServerDescribe",
2628 )?])
2629 }
2630
2631 fn handle_catalog_list(
2632 &self,
2633 frame: Frame,
2634 module_id: Option<String>,
2635 ) -> Result<Vec<Frame>, RouterError> {
2636 let (generation, modules) = self.registry.list_modules().map_err(|err| {
2637 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
2638 })?;
2639 let entries = modules
2640 .into_iter()
2641 .filter(|registration| {
2642 module_id
2643 .as_deref()
2644 .map(|wanted| registration.manifest.module_id == wanted)
2645 .unwrap_or(true)
2646 })
2647 .map(|registration| {
2648 let not_ready = self.not_ready_reason(®istration);
2649 let roles = registration.manifest.provides;
2650 CatalogEntry {
2651 module_id: registration.manifest.module_id,
2652 ready: not_ready.is_none(),
2653 not_ready,
2654 module_version: Some(registration.manifest.module_version),
2655 roles,
2656 control_ops: registration.control_ops,
2657 capabilities: registration.manifest.capabilities,
2658 self_signals: registration.manifest.self_signals,
2659 }
2660 })
2661 .collect();
2662 let response = ClientControlResponse::CatalogList {
2663 generation,
2664 modules: entries,
2665 subc_ops: subc_ops(),
2666 };
2667 Ok(vec![control_response_body_frame(
2668 &frame,
2669 &response,
2670 "ClientControlResponse::CatalogList",
2671 )?])
2672 }
2673
2674 fn route_open_principal(
2675 &self,
2676 frame: &Frame,
2677 consumer_identity: Option<ConsumerIdentity>,
2678 ) -> Result<Result<Principal, Frame>, RouterError> {
2679 let Some(consumer_identity) = consumer_identity else {
2680 return Ok(Ok(Principal::Direct));
2681 };
2682
2683 if self.supervisor.spawned_consumer_authorized(
2684 &consumer_identity.module_id,
2685 &consumer_identity.launch_nonce,
2686 ) {
2687 return Ok(Ok(Principal::Reserved {
2688 module_id: consumer_identity.module_id,
2689 }));
2690 }
2691
2692 Ok(Err(control_error_frame(
2693 frame,
2694 "bad_consumer_identity",
2695 format!(
2696 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
2697 consumer_identity.module_id
2698 ),
2699 )?))
2700 }
2701
2702 fn route_open_refusal_frame(
2709 &self,
2710 ctx: &RouteCtx,
2711 frame: &Frame,
2712 module_id: &str,
2713 reason: &'static str,
2714 code: &'static str,
2715 message: impl Into<String>,
2716 ) -> Result<Frame, RouterError> {
2717 self.observe_route_open_refusal(ctx, module_id, reason, code);
2718 control_error_frame(frame, code, message.into())
2719 }
2720
2721 fn route_open_breaker_refusal_frame(
2739 &self,
2740 ctx: &RouteCtx,
2741 frame: &Frame,
2742 module_id: &str,
2743 consecutive_timeouts: u32,
2744 retry_in: Duration,
2745 probe_in_flight: bool,
2746 ) -> Result<Frame, RouterError> {
2747 self.counters
2748 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
2749 debug!(
2750 target: "control",
2751 code = "module_timeout",
2752 module_id = ?module_id,
2753 connection_id = ctx.connection_id.get(),
2754 consecutive_timeouts,
2755 retry_in_ms = retry_in.as_millis() as u64,
2756 probe_in_flight,
2757 "route.open refused by open bind-relay breaker"
2758 );
2759 let detail = if probe_in_flight {
2760 "one probe bind is already in flight; retry once it settles".to_string()
2761 } else {
2762 format!("not relaying for another {retry_in:?}")
2763 };
2764 control_error_frame(
2765 frame,
2766 "module_timeout",
2767 format!(
2768 "module_id '{module_id}' failed {consecutive_timeouts} consecutive route.bind \
2769 relays; {detail}"
2770 ),
2771 )
2772 }
2773
2774 fn observe_route_open_refusal(
2787 &self,
2788 ctx: &RouteCtx,
2789 module_id: &str,
2790 reason: &'static str,
2791 code: &'static str,
2792 ) {
2793 self.counters.increment_route_open_refused(code);
2794 info!(
2795 target: "control",
2796 code,
2797 reason,
2798 module_id = ?module_id,
2799 connection_id = ctx.connection_id.get(),
2800 "route.open refused"
2801 );
2802 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
2803 self.route_outages.record_not_serving(module_id, reason);
2804 }
2805 }
2806
2807 fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
2852 self.route_outages.record_accepted(module_id);
2853 self.counters.increment_route_open_accepted(principal);
2854 info!(
2855 target: "control",
2856 principal,
2857 module_id,
2858 connection_id = ctx.connection_id.get(),
2859 "route.open accepted"
2860 );
2861 }
2862
2863 fn supervised_absent_route_open_refusal_frame(
2864 &self,
2865 ctx: &RouteCtx,
2866 frame: &Frame,
2867 module_id: &str,
2868 code: &'static str,
2869 status: &crate::supervise::ModuleStatus,
2870 ) -> Result<Frame, RouterError> {
2871 self.counters.increment_route_open_refused(code);
2872 info!(
2873 target: "control",
2874 code,
2875 reason = "supervised_not_registered",
2876 module_id = ?module_id,
2877 connection_id = ctx.connection_id.get(),
2878 state = %status.state,
2879 enabled = status.enabled,
2880 live = status.live,
2881 "route.open refused"
2882 );
2883 self.route_outages
2887 .record_not_serving(module_id, "supervised_not_registered");
2888 control_error_frame(
2889 frame,
2890 code,
2891 format!(
2892 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
2893 status.state, status.enabled, status.live
2894 ),
2895 )
2896 }
2897
2898 async fn handle_route_open(
2899 &self,
2900 ctx: &RouteCtx,
2901 frame: Frame,
2902 request: RouteOpenRequest,
2903 ) -> Result<Vec<Frame>, RouterError> {
2904 let RouteOpenRequest {
2905 target,
2906 mut identity,
2907 consumer_identity,
2908 consumer_capabilities,
2909 role_versions,
2910 admission_facts,
2911 scope,
2912 } = request;
2913 let target_module_id = target_module_id(&target).to_string();
2914 debug!(
2915 connection_id = ctx.connection_id.get(),
2916 corr = frame.header.corr,
2917 module_id = %target_module_id,
2918 "handling route.open"
2919 );
2920
2921 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
2927 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
2928 self.observe_route_open_refusal(
2929 ctx,
2930 &target_module_id,
2931 "invalid_role_versions",
2932 error_codes::INVALID_REQUEST,
2933 );
2934 return Ok(vec![control_error_body_frame(
2935 &frame,
2936 ErrorBody {
2937 code: error_codes::INVALID_REQUEST.to_string(),
2938 message: error.to_string(),
2939 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
2940 },
2941 )?]);
2942 }
2943
2944 let Some(registration) = self
2963 .registry
2964 .get_module(&target_module_id)
2965 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2966 else {
2967 if let Some((status, warming)) =
2968 self.supervisor_status(&target_module_id, frame.header.corr)?
2969 {
2970 if status.protocol == ModuleProtocol::None {
2977 return Ok(vec![self.route_open_refusal_frame(
2978 ctx,
2979 &frame,
2980 &target_module_id,
2981 "protocol_none",
2982 error_codes::MODULE_NO_PROTOCOL,
2983 format!(
2984 "module_id '{target_module_id}' is declared protocol: none; \
2985 it speaks no subc wire and serves no routes"
2986 ),
2987 )?]);
2988 }
2989 let code = if warming {
2990 "module_warming"
2991 } else {
2992 "target_unavailable"
2993 };
2994 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
2995 ctx,
2996 &frame,
2997 &target_module_id,
2998 code,
2999 &status,
3000 )?]);
3001 }
3002 if let Some(removed_ago_ms) =
3003 self.supervisor.removal_tombstone_age_ms(&target_module_id)
3004 {
3005 return Ok(vec![self.route_open_refusal_frame(
3006 ctx,
3007 &frame,
3008 &target_module_id,
3009 "removed",
3010 error_codes::MODULE_REMOVED,
3011 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
3012 )?]);
3013 }
3014 return Ok(vec![self.route_open_refusal_frame(
3015 ctx,
3016 &frame,
3017 &target_module_id,
3018 "not_registered",
3019 error_codes::UNKNOWN_MODULE,
3020 format!("module_id '{target_module_id}' is not registered"),
3021 )?]);
3022 };
3023
3024 if !registration.ready {
3028 self.counters
3029 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
3030 info!(
3031 target: "control",
3032 code = error_codes::MODULE_WARMING,
3033 module_id = ?target_module_id,
3034 connection_id = ctx.connection_id.get(),
3035 reason = "declared_not_ready",
3036 "route.open refused"
3037 );
3038 self.route_outages
3041 .record_not_serving(&target_module_id, "declared_not_ready");
3042 return Ok(vec![control_error_body_frame(
3043 &frame,
3044 ErrorBody {
3045 code: error_codes::MODULE_WARMING.to_string(),
3046 message: format!(
3047 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
3048 ),
3049 detail: Some(serde_json::json!({
3050 "reason": "declared_not_ready"
3051 })),
3052 },
3053 )?]);
3054 }
3055
3056 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
3075 self.counters
3076 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
3077 info!(
3078 target: "control",
3079 code = error_codes::MODULE_WARMING,
3080 module_id = ?target_module_id,
3081 connection_id = ctx.connection_id.get(),
3082 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3083 capability = %capability,
3084 "route.open refused"
3085 );
3086 return Ok(vec![control_error_body_frame(
3087 &frame,
3088 ErrorBody {
3089 code: error_codes::MODULE_WARMING.to_string(),
3090 message: format!(
3091 "module_id '{target_module_id}' requires capability '{capability}', \
3092 which no registered module provides; retry"
3093 ),
3094 detail: Some(serde_json::json!({
3095 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3096 "capability": capability,
3097 })),
3098 },
3099 )?]);
3100 }
3101
3102 if !target_has_required_role(&target, ®istration.manifest.provides) {
3103 return Ok(vec![self.route_open_refusal_frame(
3104 ctx,
3105 &frame,
3106 &target_module_id,
3107 "role_not_provided",
3108 "target_unavailable",
3109 format!("module_id '{target_module_id}' does not provide the requested target"),
3110 )?]);
3111 }
3112
3113 if registration.state != ChannelState::Active {
3114 return Ok(vec![self.route_open_refusal_frame(
3115 ctx,
3116 &frame,
3117 &target_module_id,
3118 "registration_not_active",
3119 "target_unavailable",
3120 format!("module_id '{target_module_id}' is not active"),
3121 )?]);
3122 }
3123
3124 if self
3125 .forwarding
3126 .module_is_draining(&target_module_id)
3127 .map_err(RouterError::Forwarding)?
3128 {
3129 return Ok(vec![self.route_open_refusal_frame(
3130 ctx,
3131 &frame,
3132 &target_module_id,
3133 "reloading",
3134 "module_reloading",
3135 format!("module_id '{target_module_id}' is reloading"),
3136 )?]);
3137 }
3138
3139 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
3140 process_liveness.process_live(&target_module_id) == Some(false)
3141 }) {
3142 if process_liveness.process_replacing(&target_module_id) {
3149 return Ok(vec![self.route_open_refusal_frame(
3150 ctx,
3151 &frame,
3152 &target_module_id,
3153 "reloading",
3154 "module_reloading",
3155 format!("module_id '{target_module_id}' is reloading"),
3156 )?]);
3157 }
3158 return Ok(vec![self.route_open_refusal_frame(
3159 ctx,
3160 &frame,
3161 &target_module_id,
3162 "supervisor_not_live",
3163 "target_unavailable",
3164 format!("module_id '{target_module_id}' is not live"),
3165 )?]);
3166 }
3167
3168 if !self
3169 .forwarding
3170 .has_live_module_connection(&target_module_id)
3171 .map_err(RouterError::Forwarding)?
3172 {
3173 return Ok(vec![self.route_open_refusal_frame(
3174 ctx,
3175 &frame,
3176 &target_module_id,
3177 "no_forwarding_connection",
3178 "target_unavailable",
3179 format!("module_id '{target_module_id}' has no live forwarding connection"),
3180 )?]);
3181 }
3182
3183 if let Some(error) =
3184 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
3185 {
3186 self.observe_route_open_refusal(
3187 ctx,
3188 &target_module_id,
3189 "op_not_allowed",
3190 "op_not_allowed",
3191 );
3192 return Ok(vec![error]);
3193 }
3194
3195 let principal = match self.route_open_principal(&frame, consumer_identity)? {
3196 Ok(principal) => principal,
3197 Err(error) => {
3198 self.observe_route_open_refusal(
3199 ctx,
3200 &target_module_id,
3201 "bad_consumer_identity",
3202 "bad_consumer_identity",
3203 );
3204 return Ok(vec![error]);
3205 }
3206 };
3207
3208 if let Principal::Reserved {
3212 module_id: opening_module_id,
3213 } = &principal
3214 {
3215 if let Some(opening_registration) = self
3216 .registry
3217 .get_module(opening_module_id)
3218 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3219 {
3220 if let Some(capability) =
3221 denied_capability(&opening_registration.manifest, ®istration.manifest)
3222 {
3223 warn!(
3224 opening_module_id,
3225 target_module_id,
3226 capability,
3227 "refusing route.open because an attested capability deny edge matches"
3228 );
3229 return Ok(vec![self.route_open_refusal_frame(
3230 ctx,
3231 &frame,
3232 &target_module_id,
3233 "capability_deny_edge",
3234 "capability_forbidden",
3235 format!(
3236 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
3237 ),
3238 )?]);
3239 }
3240 }
3241 }
3242
3243 if admission_facts.is_some() {
3244 let carrier_matches = matches!(
3245 &principal,
3246 Principal::Reserved { module_id }
3247 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
3248 );
3249 if !carrier_matches {
3250 return Ok(vec![self.route_open_refusal_frame(
3251 ctx,
3252 &frame,
3253 &target_module_id,
3254 "admission_facts_carrier_not_permitted",
3255 "admission_facts_not_permitted",
3256 "admission facts may only be carried by the configured reserved module",
3257 )?]);
3258 }
3259
3260 let target_allowed = self
3261 .admission_facts_targets
3262 .as_ref()
3263 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
3264 if !target_allowed {
3265 return Ok(vec![self.route_open_refusal_frame(
3266 ctx,
3267 &frame,
3268 &target_module_id,
3269 "admission_facts_target_not_listed",
3270 "admission_facts_target_not_allowed",
3271 format!(
3272 "admission facts are not permitted for target module_id '{target_module_id}'"
3273 ),
3274 )?]);
3275 }
3276
3277 }
3281
3282 let (bound_scope, scope_stamp) = match scope {
3288 None => (None, None),
3289 Some(selector) => {
3290 let owner_configured = match &selector.owner {
3291 Principal::Reserved { module_id } => self.supervisor.get(module_id).is_some(),
3292 _ => false,
3293 };
3294 let admitted = self
3295 .scopes
3296 .read()
3297 .unwrap_or_else(|poisoned| poisoned.into_inner())
3298 .admit(&principal, &target_module_id, &selector, owner_configured);
3299 match admitted {
3300 Ok(admission) => (
3301 Some(BoundScope {
3302 owner: admission.owner,
3303 scope_ref: admission.stamp.scope_ref.clone(),
3304 tag: admission.tag,
3305 }),
3306 Some(admission.stamp),
3307 ),
3308 Err(refusal) => {
3309 return Ok(vec![self.route_open_refusal_frame(
3310 ctx,
3311 &frame,
3312 &target_module_id,
3313 refusal.code,
3314 refusal.code,
3315 refusal.message,
3316 )?]);
3317 }
3318 }
3319 }
3320 };
3321
3322 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
3344 Ok(project_root) => project_root,
3345 Err(err) => {
3346 return Ok(vec![control_error_frame(
3347 &frame,
3348 "invalid_project_root",
3349 err.to_string(),
3350 )?])
3351 }
3352 };
3353 identity.project_root = project_root.as_path().to_path_buf();
3354
3355 let _concurrency_guard = match self
3366 .route_bind_concurrency
3367 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
3368 {
3369 Ok(guard) => guard,
3370 Err(in_flight) => {
3371 return Ok(vec![self.route_open_target_capacity_refusal(
3372 ctx,
3373 &frame,
3374 &target_module_id,
3375 in_flight,
3376 )?]);
3377 }
3378 };
3379
3380 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
3383 RouteBindAdmission::Admitted { guard, probe } => {
3384 if probe {
3385 info!(
3386 module_id = %target_module_id,
3387 connection_id = ctx.connection_id.get(),
3388 "route.bind breaker half-open: admitting one probe"
3389 );
3390 }
3391 guard
3392 }
3393 RouteBindAdmission::Refused {
3394 consecutive_timeouts,
3395 retry_in,
3396 probe_in_flight,
3397 } => {
3398 return Ok(vec![self.route_open_breaker_refusal_frame(
3399 ctx,
3400 &frame,
3401 &target_module_id,
3402 consecutive_timeouts,
3403 retry_in,
3404 probe_in_flight,
3405 )?]);
3406 }
3407 };
3408
3409 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
3414 let relay_deadline = Instant::now() + route_bind_relay_timeout;
3415 let pending = match self
3416 .forwarding
3417 .begin_route_bind_relay_for(
3418 ctx.connection_id,
3419 ctx.egress.clone(),
3420 response_version(&frame),
3421 frame.header.corr,
3422 &target_module_id,
3423 principal.clone(),
3424 bound_scope,
3425 Some(project_root),
3426 relay_deadline,
3427 )
3428 .await
3429 {
3430 Ok(pending) => pending,
3431 Err(err) => {
3432 return Ok(vec![self.route_open_refusal_frame(
3433 ctx,
3434 &frame,
3435 &target_module_id,
3436 "relay_reservation_failed",
3437 forwarding_error_code(&err),
3438 err.to_string(),
3439 )?])
3440 }
3441 };
3442 let crate::forwarding::PendingRouteBindRelay {
3443 endpoint,
3444 module_sink,
3445 negotiated_ver,
3446 client_channel,
3447 client_epoch,
3448 module_channel,
3449 module_epoch,
3450 corr: relay_corr,
3451 receiver,
3452 } = pending;
3453 let mut reservation =
3454 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
3455
3456 debug!(
3457 connection_id = ctx.connection_id.get(),
3458 client_channel,
3459 client_epoch,
3460 module_channel,
3461 module_epoch,
3462 "reserved route handle pair"
3463 );
3464 let principal_label = match &principal {
3467 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
3468 Principal::Direct => "direct".to_string(),
3469 other => format!("{other:?}"),
3470 };
3471 let relay = ModuleControlRequest::RouteBind {
3472 route_channel: module_channel,
3473 epoch: module_epoch,
3474 target,
3475 identity,
3476 principal: Some(principal),
3477 consumer_capabilities,
3478 role_versions,
3479 admission_facts,
3480 scope: scope_stamp,
3481 };
3482 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
3483 RouterError::backend(
3484 0,
3485 frame.header.corr,
3486 format!("failed to encode route.bind request: {err}"),
3487 )
3488 })?;
3489 let relay_frame = Frame::build_with_version(
3490 negotiated_ver,
3491 FrameType::Request,
3492 control_flags(),
3493 0,
3494 0,
3495 relay_corr,
3496 relay_body,
3497 )
3498 .map_err(RouterError::FrameBuild)?;
3499
3500 if let Err(err) = module_sink.send(relay_frame).await {
3501 reservation.release_and_disarm();
3502 return Ok(vec![self.route_open_refusal_frame(
3503 ctx,
3504 &frame,
3505 &target_module_id,
3506 "relay_send_failed",
3507 "target_unavailable",
3508 err.to_string(),
3509 )?]);
3510 }
3511
3512 if !self
3513 .forwarding
3514 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
3515 .map_err(RouterError::Forwarding)?
3516 {
3517 self.send_abandoned_route_bind_goodbye(
3518 &module_sink,
3519 negotiated_ver,
3520 module_channel,
3521 module_epoch,
3522 );
3523 }
3524
3525 match timeout_at(relay_deadline, receiver).await {
3526 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
3527 reservation.disarm();
3528 if breaker.record_accepted() {
3529 info!(
3530 module_id = %target_module_id,
3531 "route.bind breaker closed: the probe was accepted"
3532 );
3533 }
3534 self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
3535 Ok(Vec::new())
3536 }
3537 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
3538 reservation.release_and_disarm();
3539 breaker.record_inconclusive();
3543 let scope_code = match body.code.as_str() {
3547 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
3548 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
3549 _ => None,
3550 };
3551 if let Some(code) = scope_code {
3552 self.observe_route_open_refusal(
3553 ctx,
3554 &target_module_id,
3555 "scope_changed_before_commit",
3556 code,
3557 );
3558 return Ok(vec![control_error_body_frame(&frame, body)?]);
3559 }
3560 self.counters
3561 .increment_route_open_refused("module_rejected");
3562 info!(
3563 target: "control",
3564 code = "module_rejected",
3565 module_code = ?body.code,
3566 module_id = ?target_module_id,
3567 connection_id = ctx.connection_id.get(),
3568 "route.open refused"
3569 );
3570 Ok(vec![control_error_body_frame(&frame, body)?])
3571 }
3572 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
3573 reservation.release_and_disarm();
3574 breaker.record_inconclusive();
3575 tracing::warn!(
3582 module_id = %target_module_id,
3583 "route.bind relay abandoned: {message}"
3584 );
3585 Ok(vec![self.route_open_refusal_frame(
3586 ctx,
3587 &frame,
3588 &target_module_id,
3589 "relay_abandoned",
3590 "target_unavailable",
3591 message,
3592 )?])
3593 }
3594 Ok(Err(_)) => {
3595 reservation.release_and_disarm();
3596 breaker.record_inconclusive();
3597 Ok(vec![self.route_open_refusal_frame(
3598 ctx,
3599 &frame,
3600 &target_module_id,
3601 "relay_waiter_canceled",
3602 "target_unavailable",
3603 "route.bind relay waiter was canceled before the module responded",
3604 )?])
3605 }
3606 Err(_) => {
3607 reservation.release_and_disarm();
3608 if let Some(opened) = breaker.record_timeout(
3612 self.route_bind_breaker_threshold,
3613 self.route_bind_breaker_cooldown,
3614 ) {
3615 warn!(
3616 module_id = %target_module_id,
3617 consecutive_timeouts = opened.consecutive_timeouts,
3618 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
3619 reopened_after_probe = opened.reopened_after_probe,
3620 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
3621 );
3622 }
3623 tracing::warn!(
3631 module_id = %target_module_id,
3632 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
3633 "route.bind relay timed out: module did not ack within budget"
3634 );
3635 Ok(vec![self.route_open_refusal_frame(
3636 ctx,
3637 &frame,
3638 &target_module_id,
3639 "relay_timed_out",
3640 "module_timeout",
3641 format!(
3642 "module_id '{target_module_id}' did not answer route.bind within {:?}",
3643 route_bind_relay_timeout
3644 ),
3645 )?])
3646 }
3647 }
3648 }
3649
3650 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3651 let response = ClientControlResponse::SupervisorSpawnSnapshot {
3652 snapshot: self.supervisor.spawn_snapshot(),
3653 };
3654 Ok(vec![control_response_body_frame(
3655 &frame,
3656 &response,
3657 "ClientControlResponse::SupervisorSpawnSnapshot",
3658 )?])
3659 }
3660
3661 fn handle_supervisor_spawn_subscribe(
3662 &self,
3663 ctx: &RouteCtx,
3664 frame: Frame,
3665 since: Option<SpawnCursor>,
3666 ) -> Result<Vec<Frame>, RouterError> {
3667 match self.supervisor.subscribe_spawns(
3668 ctx.connection_id,
3669 frame.header.corr,
3670 response_version(&frame),
3671 since,
3672 ctx.egress.clone(),
3673 ) {
3674 Ok(()) => Ok(Vec::new()),
3675 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
3676 Ok(vec![control_error_body_frame(
3677 &frame,
3678 ErrorBody {
3679 code: "spawn_cursor_incarnation_mismatch".to_string(),
3680 message: "spawn cursor belongs to a different daemon incarnation"
3681 .to_string(),
3682 detail: Some(serde_json::json!({
3683 "current_daemon_incarnation": current
3684 })),
3685 },
3686 )?])
3687 }
3688 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
3689 &frame,
3690 ErrorBody {
3691 code: "spawn_cursor_too_old".to_string(),
3692 message: "spawn cursor predates the retained event ring".to_string(),
3693 detail: Some(serde_json::json!({
3694 "oldest_retained_cursor": oldest
3695 })),
3696 },
3697 )?]),
3698 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
3699 0,
3700 frame.header.corr,
3701 format!("failed to open supervisor spawn subscription: {error}"),
3702 )),
3703 }
3704 }
3705
3706 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3707 let generation = self
3708 .registry
3709 .generation()
3710 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3711 let mut modules = Vec::new();
3712 for module in self.supervisor.list() {
3713 let status = module.status_for_control("list").map_err(|err| {
3714 RouterError::backend(
3715 0,
3716 frame.header.corr,
3717 format!("failed to read supervisor status: {err}"),
3718 )
3719 })?;
3720 let (configured, _) = module.configuration().map_err(|err| {
3721 RouterError::backend(
3722 0,
3723 frame.header.corr,
3724 format!("failed to read module configuration: {err}"),
3725 )
3726 })?;
3727 let image = module.running_image_agreement().await;
3729 let resources = Some(module.child_resource_usage());
3732 let pending_reload = Some(reload_verdict(
3733 &configured.program,
3734 status.spawned_from.as_deref(),
3735 image,
3736 ));
3737 modules.push(SupervisorEntry {
3738 launch_nonce_env: Some(!cfg!(unix)),
3741 module_id: status.module_id,
3742 state: status.state.to_string(),
3743 enabled: status.enabled,
3744 live: status.live,
3745 protocol: status.protocol,
3746 health: status.health.status,
3747 pending_reload,
3748 last_probe_ms: status.health.last_probe_ms,
3749 last_exit_code: status.last_exit.as_ref().and_then(|e| e.code),
3750 last_exit_signal: status.last_exit.as_ref().and_then(|e| e.signal),
3751 last_exit_ms: status.last_exit.as_ref().map(|e| e.at_ms),
3752 last_exit_kind: status.last_exit.as_ref().map(|e| e.kind.into()),
3753 restart_count: Some(status.restart_count),
3754 max_restarts: Some(status.max_restarts),
3755 lifetime_restarts: Some(status.lifetime_restarts),
3756 spawn_generation: Some(status.spawn_generation),
3757 restart_window_secs: Some(status.restart_window.as_secs()),
3758 drain_timeout_ms: Some(status.drain_timeout.as_millis() as u64),
3759 restart_backoff_ms: Some(status.restart_backoff.as_millis() as u64),
3760 restart_max_backoff_ms: Some(status.restart_max_backoff.as_millis() as u64),
3761 resources,
3762 });
3763 }
3764 let response = ClientControlResponse::SupervisorList {
3765 generation,
3766 modules,
3767 };
3768 Ok(vec![control_response_body_frame(
3769 &frame,
3770 &response,
3771 "ClientControlResponse::SupervisorList",
3772 )?])
3773 }
3774
3775 fn handle_supervisor_stderr_tail(
3776 &self,
3777 frame: Frame,
3778 module_id: String,
3779 max_lines: Option<u32>,
3780 max_bytes: Option<u32>,
3781 ) -> Result<Vec<Frame>, RouterError> {
3782 let Some(module) = self.supervisor.get(&module_id) else {
3783 return Ok(vec![control_error_frame(
3784 &frame,
3785 "unknown_module",
3786 format!("module_id '{module_id}' is not supervised"),
3787 )?]);
3788 };
3789
3790 let snapshot = module.stderr_tail(
3791 max_lines.map(|value| value as usize),
3792 max_bytes.map(|value| value as usize),
3793 );
3794
3795 let response = ClientControlResponse::SupervisorStderrTail {
3796 module_id,
3797 tail: StderrTail {
3798 capture: match snapshot.capture {
3799 CaptureState::Captured => StderrCaptureState::Captured,
3800 CaptureState::Incomplete { reason } => {
3801 StderrCaptureState::Incomplete { reason }
3802 }
3803 CaptureState::NotCaptured { reason } => {
3804 StderrCaptureState::NotCaptured { reason }
3805 }
3806 },
3807 entries: snapshot
3808 .entries
3809 .into_iter()
3810 .map(|entry| match entry {
3811 TailEntry::Line {
3812 text,
3813 truncated,
3814 at_ms,
3815 } => StderrTailEntry::Line {
3816 text,
3817 truncated,
3818 at_ms,
3819 },
3820 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
3821 })
3822 .collect(),
3823 dropped_lines: snapshot.dropped_lines,
3824 },
3825 };
3826 Ok(vec![control_response_body_frame(
3827 &frame,
3828 &response,
3829 "ClientControlResponse::SupervisorStderrTail",
3830 )?])
3831 }
3832
3833 async fn handle_supervisor_terminals(
3834 &self,
3835 frame: Frame,
3836 module_id: String,
3837 ) -> Result<Vec<Frame>, RouterError> {
3838 let Some(module) = self.supervisor.get(&module_id) else {
3839 return Ok(vec![control_error_frame(
3840 &frame,
3841 "unknown_module",
3842 format!("module_id '{module_id}' is not supervised"),
3843 )?]);
3844 };
3845
3846 let terminals = module
3849 .read_durable_terminal_history()
3850 .await
3851 .map_err(|error| {
3852 RouterError::backend(
3853 0,
3854 frame.header.corr,
3855 format!("failed to read terminal history: {error}"),
3856 )
3857 })?;
3858 let response = ClientControlResponse::SupervisorTerminals {
3859 module_id,
3860 terminals,
3861 };
3862 Ok(vec![control_response_body_frame(
3863 &frame,
3864 &response,
3865 "ClientControlResponse::SupervisorTerminals",
3866 )?])
3867 }
3868
3869 fn handle_supervisor_routes(
3870 &self,
3871 frame: Frame,
3872 module_id: Option<String>,
3873 ) -> Result<Vec<Frame>, RouterError> {
3874 let modules = self
3875 .forwarding
3876 .route_census(module_id.as_deref())
3877 .map_err(RouterError::Forwarding)?
3878 .into_iter()
3879 .map(|(module_id, routes)| SupervisorRouteModule {
3880 module_id,
3881 routes: routes
3882 .into_iter()
3883 .map(|route| SupervisorRoute {
3884 consumer: match route.principal {
3885 Principal::Reserved { module_id } => {
3886 SupervisorRouteConsumer::Reserved { module_id }
3887 }
3888 Principal::Direct | Principal::Unverified => {
3889 SupervisorRouteConsumer::Direct {
3890 connection_id: route.goodbye_target.connection_id.get(),
3891 }
3892 }
3893 },
3894 age_ms: Instant::now()
3895 .saturating_duration_since(route.bound_at)
3896 .as_millis()
3897 .try_into()
3898 .unwrap_or(u64::MAX),
3899 draining: route.draining,
3900 drain_reason: route.drain_reason,
3901 })
3902 .collect(),
3903 })
3904 .collect();
3905 let response = ClientControlResponse::SupervisorRoutes { modules };
3906 Ok(vec![control_response_body_frame(
3907 &frame,
3908 &response,
3909 "ClientControlResponse::SupervisorRoutes",
3910 )?])
3911 }
3912
3913 async fn handle_supervisor_provenance(
3914 &self,
3915 frame: Frame,
3916 module_id: Option<String>,
3917 ) -> Result<Vec<Frame>, RouterError> {
3918 let mut selected = if let Some(module_id) = module_id {
3919 let Some(module) = self.supervisor.get(&module_id) else {
3920 return Ok(vec![control_error_frame(
3921 &frame,
3922 "unknown_module",
3923 format!("module_id '{module_id}' is not supervised"),
3924 )?]);
3925 };
3926 vec![module]
3927 } else {
3928 self.supervisor.list()
3929 };
3930
3931 let mut modules = Vec::with_capacity(selected.len());
3932 for module in selected.drain(..) {
3933 let status = module.status().map_err(|err| {
3934 RouterError::backend(
3935 0,
3936 frame.header.corr,
3937 format!("failed to read supervisor status: {err}"),
3938 )
3939 })?;
3940 let module_declared = self
3941 .registry
3942 .get_module(&status.module_id)
3943 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3944 .and_then(|registration| registration.manifest.provenance)
3945 .map(|build| ModuleDeclaredProvenance::Reported { build })
3946 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
3947 #[cfg(test)]
3948 let running_image = match &self.provenance_probe_override {
3949 Some(result) => result.clone(),
3950 None => module.running_image_agreement().await,
3951 };
3952 #[cfg(not(test))]
3953 let running_image = module.running_image_agreement().await;
3954 modules.push(SupervisorModuleProvenance {
3955 module_id: status.module_id,
3956 module_declared,
3957 daemon_observed: SupervisorObservedProcess {
3958 pid: status.pid,
3959 spawned_at_ms: status.spawned_at_ms,
3960 spawned_from: status.spawned_from,
3961 running_image,
3962 },
3963 });
3964 }
3965 let daemon = SupervisorDaemonProvenance {
3966 daemon_build: self.daemon_provenance.build.clone(),
3967 daemon_observed: DaemonObservedProcess {
3968 pid: self.daemon_provenance.pid,
3969 started_at_ms: self
3970 .daemon_provenance
3971 .start_clock
3972 .map(|clock| clock.started_at_ms())
3973 .or(self.daemon_provenance.started_at_ms),
3974 running_image: self
3975 .daemon_provenance
3976 .probe
3977 .observe(
3978 self.daemon_provenance.pid,
3979 self.daemon_provenance.executable_path.as_deref(),
3980 self.daemon_provenance.executable_identity,
3981 self.daemon_provenance.process_start_time,
3982 )
3983 .await,
3984 },
3985 };
3986 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
3987 Ok(vec![control_response_body_frame(
3988 &frame,
3989 &response,
3990 "ClientControlResponse::SupervisorProvenance",
3991 )?])
3992 }
3993
3994 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3995 self.refresh_capability_requirements();
3996 let generation = self
3997 .registry
3998 .generation()
3999 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4000 let modules = self
4001 .supervisor
4002 .list()
4003 .into_iter()
4004 .map(|module| {
4005 let status = module.status_for_control("health").map_err(|err| {
4006 RouterError::backend(
4007 0,
4008 frame.header.corr,
4009 format!("failed to read supervisor health: {err}"),
4010 )
4011 })?;
4012 let module_id = status.module_id;
4013 let capability_detail = self
4014 .capability_evaluator
4015 .required_problem_detail(&module_id);
4016 Ok(SupervisorHealthEntry {
4017 module_id,
4018 status: status.health.status,
4019 detail: append_capability_problem_detail(
4020 status.health.detail,
4021 capability_detail,
4022 ),
4023 metrics: status.health.metrics,
4024 consecutive_failures: status.health.consecutive_failures,
4025 late_answer_count: status.health.late_answer_count,
4026 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
4027 last_action: status.health.last_action,
4028 last_action_ms: status.health.last_action_ms,
4029 last_probe_ms: status.health.last_probe_ms,
4030 })
4031 })
4032 .collect::<Result<Vec<_>, RouterError>>()?;
4033 let response = ClientControlResponse::SupervisorHealth {
4034 generation,
4035 modules,
4036 };
4037 Ok(vec![control_response_body_frame(
4038 &frame,
4039 &response,
4040 "ClientControlResponse::SupervisorHealth",
4041 )?])
4042 }
4043
4044 async fn handle_supervisor_restart(
4045 &self,
4046 frame: Frame,
4047 module_id: String,
4048 drain_timeout_ms: Option<u64>,
4049 ) -> Result<Vec<Frame>, RouterError> {
4050 let operation_lock = self.supervisor.operation_lock();
4051 let _operation_guard = operation_lock.lock().await;
4052 let Some(module) = self.supervisor.get(&module_id) else {
4053 return Ok(vec![control_error_frame(
4054 &frame,
4055 "unknown_module",
4056 format!("module_id '{module_id}' is not supervised"),
4057 )?]);
4058 };
4059
4060 self.route_outages.mark_operator_action(&module_id);
4061 if let Err(err) = module.restart(drain_timeout_ms).await {
4062 self.route_outages
4063 .operator_action_ended_unrefused(&module_id);
4064 let (code, message) = match err {
4065 crate::supervise::SuperviseError::Disabled { .. } => {
4066 ("module_disabled", err.to_string())
4067 }
4068 crate::supervise::SuperviseError::SwapInProgress { .. } => {
4069 ("swap_in_progress", err.to_string())
4070 }
4071 _ => (
4072 "target_unavailable",
4073 format!("failed to restart module_id '{module_id}': {err}"),
4074 ),
4075 };
4076 return Ok(vec![control_error_frame(&frame, code, message)?]);
4077 }
4078
4079 let response = ClientControlResponse::SupervisorAck {
4080 module_id,
4081 applied: true,
4082 };
4083 Ok(vec![control_response_body_frame(
4084 &frame,
4085 &response,
4086 "ClientControlResponse::SupervisorAck",
4087 )?])
4088 }
4089
4090 async fn handle_supervisor_swap(
4094 &self,
4095 frame: Frame,
4096 module_id: String,
4097 ready_timeout_ms: Option<u64>,
4098 ) -> Result<Vec<Frame>, RouterError> {
4099 let module = {
4106 let operation_lock = self.supervisor.operation_lock();
4107 let _operation_guard = operation_lock.lock().await;
4108 self.supervisor.get(&module_id)
4109 };
4110 let Some(module) = module else {
4111 return Ok(vec![control_error_frame(
4112 &frame,
4113 "unknown_module",
4114 format!("module_id '{module_id}' is not supervised"),
4115 )?]);
4116 };
4117
4118 self.route_outages.mark_operator_action(&module_id);
4119 if let Err(err) = module
4120 .swap(ready_timeout_ms.map(Duration::from_millis))
4121 .await
4122 {
4123 self.route_outages
4124 .operator_action_ended_unrefused(&module_id);
4125 use crate::supervise::SuperviseError;
4126 let message = err.to_string();
4127 let error = match err {
4128 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
4129 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
4130 code: "swap_refused".to_string(),
4131 message,
4132 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
4133 },
4134 SuperviseError::SwapFailed {
4135 arm,
4136 candidate_exit,
4137 ..
4138 } => ErrorBody {
4139 code: "swap_failed".to_string(),
4140 message,
4141 detail: Some(serde_json::json!({
4142 "arm": arm.as_str(),
4143 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
4144 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
4145 })),
4146 },
4147 _ => ErrorBody::new(
4148 "target_unavailable",
4149 format!("failed to swap module_id '{module_id}': {message}"),
4150 ),
4151 };
4152 return Ok(vec![control_error_body_frame(&frame, error)?]);
4153 }
4154 self.route_outages
4158 .operator_action_ended_unrefused(&module_id);
4159
4160 let response = ClientControlResponse::SupervisorAck {
4161 module_id,
4162 applied: true,
4163 };
4164 Ok(vec![control_response_body_frame(
4165 &frame,
4166 &response,
4167 "ClientControlResponse::SupervisorAck",
4168 )?])
4169 }
4170
4171 async fn handle_supervisor_reload(
4172 &self,
4173 frame: Frame,
4174 module_id: String,
4175 ) -> Result<Vec<Frame>, RouterError> {
4176 let operation_lock = self.supervisor.operation_lock();
4177 let _operation_guard = operation_lock.lock().await;
4178 let Some(module) = self.supervisor.get(&module_id) else {
4179 return Ok(vec![control_error_frame(
4180 &frame,
4181 "unknown_module",
4182 format!("module_id '{module_id}' is not supervised"),
4183 )?]);
4184 };
4185
4186 self.route_outages.mark_operator_action(&module_id);
4187 if let Err(err) = module.reload().await {
4188 self.route_outages
4189 .operator_action_ended_unrefused(&module_id);
4190 let (code, message) = match err {
4191 crate::supervise::SuperviseError::Disabled { .. } => {
4192 ("module_disabled", err.to_string())
4193 }
4194 crate::supervise::SuperviseError::SwapInProgress { .. } => {
4195 ("swap_in_progress", err.to_string())
4196 }
4197 _ => (
4198 "reload_failed",
4199 format!("failed to reload module_id '{module_id}': {err}"),
4200 ),
4201 };
4202 return Ok(vec![control_error_frame(&frame, code, message)?]);
4203 }
4204
4205 let response = ClientControlResponse::SupervisorAck {
4206 module_id,
4207 applied: true,
4208 };
4209 Ok(vec![control_response_body_frame(
4210 &frame,
4211 &response,
4212 "ClientControlResponse::SupervisorAck",
4213 )?])
4214 }
4215
4216 async fn handle_supervisor_rescan(
4217 &self,
4218 frame: Frame,
4219 preview: bool,
4220 ) -> Result<Vec<Frame>, RouterError> {
4221 let Some(context) = self.rescan.clone() else {
4222 return Ok(vec![control_error_frame(
4223 &frame,
4224 "rescan_unavailable",
4225 "the daemon was not started with a reloadable config path".to_string(),
4226 )?]);
4227 };
4228
4229 let operation_lock = self.supervisor.operation_lock();
4230 let _operation_guard = operation_lock.lock().await;
4231 let loaded = match crate::daemon_config::load(&context.config_path) {
4232 Ok(config) => config,
4233 Err(err) => {
4234 return Ok(vec![control_error_frame(
4235 &frame,
4236 "invalid_daemon_config",
4237 format!("supervisor rescan rejected daemon config: {err}"),
4238 )?])
4239 }
4240 };
4241 let Some(config) = loaded else {
4249 return Ok(vec![control_error_frame(
4250 &frame,
4251 "invalid_daemon_config",
4252 format!(
4253 "daemon config not found at {}; refusing to rescan (an absent config would \
4254 retire every supervised module)",
4255 context.config_path.display()
4256 ),
4257 )?]);
4258 };
4259 let (
4260 configured_port,
4261 storage_config,
4262 admission_facts_carrier_module_id,
4263 admission_facts_targets,
4264 scope_authority_owners,
4265 modules,
4266 reserved_capabilities,
4267 ) = (
4268 config.port,
4269 config.storage,
4270 config.admission_facts_carrier_module_id,
4271 config.admission_facts_targets,
4272 config.scope_authority_owners,
4273 config.modules,
4274 config.reserved_capabilities,
4275 );
4276
4277 let mut restart_required = Vec::new();
4286 for section in RestartRequiredSection::ALL {
4287 let changed = match section {
4288 RestartRequiredSection::Port => configured_port != context.configured_port,
4289 RestartRequiredSection::Storage => storage_config != context.storage_config,
4290 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
4291 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
4292 }
4293 RestartRequiredSection::AdmissionFactsTargets => {
4294 admission_facts_targets != context.admission_facts_targets
4295 }
4296 RestartRequiredSection::ScopeAuthorityOwners => {
4297 scope_authority_owners != context.scope_authority_owners
4298 }
4299 };
4300 if changed {
4301 restart_required.push(section.label().to_string());
4302 }
4303 }
4304 if !restart_required.is_empty() {
4305 warn!(
4306 config_path = %context.config_path.display(),
4307 sections = %restart_required.join(", "),
4308 "daemon config changed outside the modules section; restart the daemon to apply those changes"
4309 );
4310 }
4311
4312 for configured in &modules {
4313 if let Err(err) = validate_spec(&configured.module_spec()) {
4314 return Ok(vec![control_error_frame(
4315 &frame,
4316 "invalid_daemon_config",
4317 format!("supervisor rescan rejected daemon config: {err}"),
4318 )?]);
4319 }
4320 }
4321
4322 let configured_capabilities = modules
4323 .iter()
4324 .map(|module| (module.module_id.clone(), module.enabled))
4325 .collect::<Vec<_>>();
4326 let preview_capability_warnings = if preview {
4327 let (_, registrations) = self.runtime_capability_snapshot()?;
4328 let current_modules = self
4329 .supervisor
4330 .list()
4331 .into_iter()
4332 .map(|module| module.module_id().to_string())
4333 .collect::<BTreeSet<_>>();
4334 let resulting_modules = configured_capabilities.clone();
4335 let removed = current_modules
4336 .into_iter()
4337 .filter(|module_id| {
4338 !resulting_modules
4339 .iter()
4340 .any(|(configured_id, _)| configured_id == module_id)
4341 })
4342 .collect::<Vec<_>>();
4343 self.capability_evaluator.preview_removal_warnings(
4344 resulting_modules,
4345 &removed,
4346 ®istrations,
4347 )
4348 } else {
4349 Vec::new()
4350 };
4351 let result = match self
4352 .reconcile_supervised_modules(&context.supervisor, modules, preview)
4353 .await
4354 {
4355 Ok(result) => result,
4356 Err(message) => {
4357 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
4358 }
4359 };
4360 if !preview {
4361 self.capability_evaluator
4362 .configure(configured_capabilities, reserved_capabilities);
4363 self.capability_evaluator.wake_deadline_loop();
4364 self.refresh_capability_requirements();
4365 }
4366 let mut result = result;
4367 result.restart_required = restart_required;
4368 result.capability_warnings = preview_capability_warnings;
4369 let response = ClientControlResponse::SupervisorRescan { result };
4370 Ok(vec![control_response_body_frame(
4371 &frame,
4372 &response,
4373 "ClientControlResponse::SupervisorRescan",
4374 )?])
4375 }
4376
4377 async fn handle_supervisor_release_reserved(
4378 &self,
4379 frame: Frame,
4380 module_id: String,
4381 ) -> Result<Vec<Frame>, RouterError> {
4382 let Some(context) = self.rescan.clone() else {
4383 return Ok(vec![control_error_frame(
4384 &frame,
4385 "release_unavailable",
4386 "reserved-id release requires a daemon started with a reloadable config path",
4387 )?]);
4388 };
4389 let operation_lock = self.supervisor.operation_lock();
4390 let _operation_guard = operation_lock.lock().await;
4391 let loaded = match crate::daemon_config::load(&context.config_path) {
4392 Ok(Some(config)) => config,
4393 Ok(None) => {
4394 return Ok(vec![control_error_frame(
4395 &frame,
4396 "invalid_daemon_config",
4397 format!(
4398 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
4399 context.config_path.display()
4400 ),
4401 )?])
4402 }
4403 Err(err) => {
4404 return Ok(vec![control_error_frame(
4405 &frame,
4406 "invalid_daemon_config",
4407 format!("unable to verify reserved-id release against daemon config: {err}"),
4408 )?])
4409 }
4410 };
4411 if loaded
4412 .modules
4413 .iter()
4414 .any(|configured| configured.module_id == module_id)
4415 {
4416 return Ok(vec![control_error_frame(
4417 &frame,
4418 "reserved_module_configured",
4419 format!(
4420 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
4421 ),
4422 )?]);
4423 }
4424 if !self.supervisor.release_retained_reserved_gate(&module_id) {
4425 return Ok(vec![control_error_frame(
4426 &frame,
4427 "reserved_gate_not_retained",
4428 format!(
4429 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
4430 ),
4431 )?]);
4432 }
4433
4434 let response = ClientControlResponse::SupervisorAck {
4435 module_id,
4436 applied: true,
4437 };
4438 Ok(vec![control_response_body_frame(
4439 &frame,
4440 &response,
4441 "ClientControlResponse::SupervisorAck",
4442 )?])
4443 }
4444
4445 async fn reconcile_supervised_modules(
4454 &self,
4455 supervisor: &Supervisor,
4456 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
4457 preview: bool,
4458 ) -> Result<SupervisorRescanResult, String> {
4459 let mut current = BTreeMap::new();
4460 for module in self.supervisor.list() {
4461 let (spec, health) = module.configuration().map_err(|err| {
4462 format!(
4463 "failed to read configuration for module_id '{}': {err}",
4464 module.module_id()
4465 )
4466 })?;
4467 let enabled = module
4468 .status()
4469 .map_err(|err| {
4470 format!(
4471 "failed to read status for module_id '{}': {err}",
4472 module.module_id()
4473 )
4474 })?
4475 .enabled;
4476 current.insert(
4477 module.module_id().to_string(),
4478 (module, spec, health, enabled),
4479 );
4480 }
4481 let configured = configured_modules
4482 .into_iter()
4483 .map(|module| (module.module_id.clone(), module))
4484 .collect::<BTreeMap<_, _>>();
4485
4486 let added = configured
4487 .keys()
4488 .filter(|module_id| !current.contains_key(*module_id))
4489 .cloned()
4490 .collect::<Vec<_>>();
4491 let removed = current
4492 .keys()
4493 .filter(|module_id| !configured.contains_key(*module_id))
4494 .cloned()
4495 .collect::<Vec<_>>();
4496 let mut changed_pending_reload = Vec::new();
4497 let mut configuration_changes = BTreeSet::new();
4498 let mut enabled_changes = BTreeSet::new();
4499 let mut unchanged = 0_u32;
4500
4501 for (module_id, configured_module) in &configured {
4502 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
4503 else {
4504 continue;
4505 };
4506 let configuration_changed = *current_spec != configured_module.module_spec()
4507 || *current_health != configured_module.health;
4508 let enabled_changed = *current_enabled != configured_module.enabled;
4509 if configuration_changed {
4510 configuration_changes.insert(module_id.clone());
4511 changed_pending_reload.push(module_id.clone());
4512 }
4513 if enabled_changed {
4514 enabled_changes.insert(module_id.clone());
4515 }
4516 if !configuration_changed && !enabled_changed {
4517 unchanged = unchanged.saturating_add(1);
4518 }
4519 }
4520
4521 if preview {
4527 return Ok(SupervisorRescanResult {
4528 added,
4529 removed,
4530 changed_pending_reload,
4531 enabled_changes: enabled_changes.iter().cloned().collect(),
4532 unchanged,
4533 preview: true,
4534 restart_required: Vec::new(),
4538 capability_warnings: Vec::new(),
4539 });
4540 }
4541
4542 for module_id in &removed {
4543 let module = ¤t
4544 .get(module_id)
4545 .expect("removed module came from current supervisor state")
4546 .0;
4547 module.retire().await.map_err(|err| {
4548 format!("failed to retire module_id '{module_id}' during rescan: {err}")
4549 })?;
4550 self.supervisor.record_rescan_removal(module_id);
4584 self.supervisor.retire(module_id);
4585 self.route_outages.forget(module_id);
4586 }
4587
4588 for module_id in configured.keys() {
4589 let Some((module, _, _, _)) = current.get(module_id) else {
4590 continue;
4591 };
4592 let configured_module = configured
4593 .get(module_id)
4594 .expect("configured module id came from configured map");
4595 if configuration_changes.contains(module_id) {
4596 module
4597 .update_configuration(
4598 configured_module.module_spec(),
4599 configured_module.health,
4600 configured_module.drain_timeout_ms,
4601 )
4602 .await
4603 .map_err(|err| {
4604 format!(
4605 "failed to update module_id '{module_id}' configuration during rescan: {err}"
4606 )
4607 })?;
4608 }
4609 if enabled_changes.contains(module_id) {
4610 self.route_outages.mark_operator_action(module_id);
4613 module
4614 .set_enabled(configured_module.enabled)
4615 .await
4616 .map_err(|err| {
4617 self.route_outages.operator_action_ended_unrefused(module_id);
4618 format!(
4619 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
4620 configured_module.enabled
4621 )
4622 })?;
4623 }
4624 }
4625
4626 for module_id in &added {
4627 let configured_module = configured
4628 .get(module_id)
4629 .expect("added module id came from configured map");
4630 supervisor
4631 .supervise_configured_with_health(
4632 configured_module.module_spec(),
4633 configured_module.enabled,
4634 configured_module.health,
4635 configured_module.drain_timeout_ms,
4636 configured_module.restart,
4637 )
4638 .map_err(|err| {
4639 format!("failed to add module_id '{module_id}' during rescan: {err}")
4640 })?;
4641 }
4642
4643 Ok(SupervisorRescanResult {
4644 added,
4645 removed,
4646 changed_pending_reload,
4647 enabled_changes: enabled_changes.iter().cloned().collect(),
4648 unchanged,
4649 preview: false,
4650 restart_required: Vec::new(),
4653 capability_warnings: Vec::new(),
4654 })
4655 }
4656
4657 async fn handle_supervisor_set_enabled(
4658 &self,
4659 frame: Frame,
4660 module_id: String,
4661 enabled: bool,
4662 ) -> Result<Vec<Frame>, RouterError> {
4663 let operation_lock = self.supervisor.operation_lock();
4664 let _operation_guard = operation_lock.lock().await;
4665 let Some(module) = self.supervisor.get(&module_id) else {
4666 return Ok(vec![control_error_frame(
4667 &frame,
4668 "unknown_module",
4669 format!("module_id '{module_id}' is not supervised"),
4670 )?]);
4671 };
4672
4673 self.route_outages.mark_operator_action(&module_id);
4676 let applied = match module.set_enabled(enabled).await {
4677 Ok(applied) => applied,
4678 Err(err) => {
4679 self.route_outages
4680 .operator_action_ended_unrefused(&module_id);
4681 return Ok(vec![control_error_frame(
4682 &frame,
4683 "target_unavailable",
4684 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
4685 )?]);
4686 }
4687 };
4688 if !applied {
4689 self.route_outages
4692 .operator_action_ended_unrefused(&module_id);
4693 }
4694
4695 self.capability_evaluator.wake_deadline_loop();
4696 self.refresh_capability_requirements();
4697 let response = ClientControlResponse::SupervisorAck { module_id, applied };
4698 Ok(vec![control_response_body_frame(
4699 &frame,
4700 &response,
4701 "ClientControlResponse::SupervisorAck",
4702 )?])
4703 }
4704
4705 async fn handle_supervisor_health_probe(
4706 &self,
4707 frame: Frame,
4708 module_id: String,
4709 ) -> Result<Vec<Frame>, RouterError> {
4710 self.refresh_capability_requirements();
4711 let Some(registration) = self
4712 .registry
4713 .get_module(&module_id)
4714 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4715 else {
4716 return Ok(vec![control_error_frame(
4717 &frame,
4718 "unknown_module",
4719 format!("module_id '{module_id}' is not registered"),
4720 )?]);
4721 };
4722
4723 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
4736 {
4737 return Ok(vec![control_error_frame(
4738 &frame,
4739 "health_not_advertised",
4740 format!("module_id '{module_id}' did not advertise health.check"),
4741 )?]);
4742 }
4743
4744 let deadline = Instant::now() + self.health_probe_timeout;
4745 let pending = match self.forwarding.begin_module_control_rpc_for(
4746 &module_id,
4747 MODULE_CONTROL_OP_HEALTH_CHECK,
4748 deadline,
4749 ) {
4750 Ok(pending) => pending,
4751 Err(err) => {
4752 return Ok(vec![control_error_frame(
4753 &frame,
4754 forwarding_error_code(&err),
4755 err.to_string(),
4756 )?])
4757 }
4758 };
4759
4760 let PendingModuleControlRpc {
4761 endpoint,
4762 module_sink,
4763 negotiated_ver,
4764 corr: probe_corr,
4765 receiver,
4766 } = pending;
4767 let mut guard =
4768 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
4769 let probe_body =
4770 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
4771 RouterError::backend(
4772 0,
4773 frame.header.corr,
4774 format!("failed to encode health.check request: {err}"),
4775 )
4776 })?;
4777 let probe_frame = Frame::build_with_version(
4778 negotiated_ver,
4779 FrameType::Request,
4780 control_flags(),
4781 0,
4782 0,
4783 probe_corr,
4784 probe_body,
4785 )
4786 .map_err(RouterError::FrameBuild)?;
4787
4788 if let Err(err) = module_sink.send(probe_frame).await {
4789 return Ok(vec![control_error_frame(
4790 &frame,
4791 "target_unavailable",
4792 err.to_string(),
4793 )?]);
4794 }
4795
4796 match timeout_at(deadline, receiver).await {
4797 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
4798 guard.disarm();
4799 let Some(report) = response.health_report() else {
4800 return Ok(vec![control_error_frame(
4801 &frame,
4802 "invalid_control_body",
4803 "health.check RPC returned a non-health response",
4804 )?]);
4805 };
4806 let HealthReport {
4811 status,
4812 detail,
4813 metrics,
4814 } = report;
4815 let capability_detail = self
4816 .capability_evaluator
4817 .required_problem_detail(&module_id);
4818 let response = ClientControlResponse::SupervisorHealthProbe {
4819 module_id,
4820 status,
4821 detail: append_capability_problem_detail(detail, capability_detail),
4822 metrics,
4823 };
4824 Ok(vec![control_response_body_frame(
4825 &frame,
4826 &response,
4827 "ClientControlResponse::SupervisorHealthProbe",
4828 )?])
4829 }
4830 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
4831 guard.disarm();
4832 Ok(vec![control_error_body_frame(&frame, body)?])
4833 }
4834 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
4835 guard.disarm();
4836 Ok(vec![control_error_frame(
4837 &frame,
4838 "target_unavailable",
4839 message,
4840 )?])
4841 }
4842 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
4843 guard.disarm();
4844 Ok(vec![control_error_frame(
4845 &frame,
4846 "invalid_control_body",
4847 message,
4848 )?])
4849 }
4850 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
4851 guard.disarm();
4852 Ok(vec![control_error_frame(
4853 &frame,
4854 "invalid_control_body",
4855 format!("expected module-control op '{expected}', got '{actual}'"),
4856 )?])
4857 }
4858 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
4859 guard.disarm();
4860 Ok(vec![control_error_frame(
4861 &frame,
4862 "module_timeout",
4863 format!(
4864 "module_id '{module_id}' answered health.check after {:?}",
4865 self.health_probe_timeout
4866 ),
4867 )?])
4868 }
4869 Ok(Err(_)) => Ok(vec![control_error_frame(
4870 &frame,
4871 "target_unavailable",
4872 "health.check waiter was canceled before the module responded",
4873 )?]),
4874 Err(_) => Ok(vec![control_error_frame(
4875 &frame,
4876 "module_timeout",
4877 format!(
4878 "module_id '{module_id}' did not answer health.check within {:?}",
4879 self.health_probe_timeout
4880 ),
4881 )?]),
4882 }
4883 }
4884
4885 fn supervisor_status(
4886 &self,
4887 module_id: &str,
4888 corr: u64,
4889 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
4890 self.supervisor
4891 .get(module_id)
4892 .map(|module| {
4893 let warming = module.is_warming_for_control("status").map_err(|err| {
4894 RouterError::backend(
4895 0,
4896 corr,
4897 format!(
4898 "failed to read supervisor warming state for module_id '{module_id}': {err}"
4899 ),
4900 )
4901 })?;
4902 module.status_for_control("status").map_err(|err| {
4903 RouterError::backend(
4904 0,
4905 corr,
4906 format!(
4907 "failed to read supervisor status for module_id '{module_id}': {err}"
4908 ),
4909 )
4910 }).map(|status| (status, warming))
4911 })
4912 .transpose()
4913 }
4914
4915 fn guard_module_control_op(
4916 &self,
4917 frame: &Frame,
4918 module_id: &str,
4919 op: &str,
4920 ) -> Result<Option<Frame>, RouterError> {
4921 if self.module_grants_op(module_id, op, frame.header.corr)? {
4922 return Ok(None);
4923 }
4924
4925 Ok(Some(control_error_frame(
4926 frame,
4927 "op_not_allowed",
4928 format!("module_id '{module_id}' did not grant control op '{op}'"),
4929 )?))
4930 }
4931
4932 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
4933 let Some(registration) = self
4934 .registry
4935 .get_module(module_id)
4936 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
4937 else {
4938 return Ok(false);
4939 };
4940 Ok(module_registration_grants_op(®istration.control_ops, op))
4941 }
4942
4943 fn handle_status_update(
4944 &self,
4945 endpoint: ModuleEndpointId,
4946 frame: Frame,
4947 ) -> Result<Vec<Frame>, RouterError> {
4948 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
4949 Ok(update) => update,
4950 Err(err) => {
4951 if is_known_module_push_op(&frame.body) {
4956 return Ok(vec![control_error_frame(
4957 &frame,
4958 "invalid_control_body",
4959 format!("malformed module control push body: {err}"),
4960 )?]);
4961 }
4962 return Ok(Vec::new());
4963 }
4964 };
4965
4966 match update {
4967 ModuleControlPush::RouteStatus {
4968 route_channel,
4969 route_epoch,
4970 status,
4971 } => {
4972 self.forwarding
4973 .cache_status(endpoint, route_channel, route_epoch, status)
4974 .map_err(RouterError::Forwarding)?;
4975 }
4976 }
4977 Ok(Vec::new())
4978 }
4979
4980 fn handle_route_poll(
4981 &self,
4982 ctx: &RouteCtx,
4983 frame: Frame,
4984 route_channel: u16,
4985 route_epoch: u32,
4986 kind: PollKind,
4987 ) -> Result<Vec<Frame>, RouterError> {
4988 let snapshot = self
4989 .forwarding
4990 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
4991 .map_err(RouterError::Forwarding)?;
4992 let response = match (kind, snapshot) {
4993 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
4994 ClientControlResponse::RoutePoll {
4995 route_channel,
4996 route_epoch,
4997 status,
4998 live: None,
4999 }
5000 }
5001 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
5002 route_channel,
5003 route_epoch,
5004 status: None,
5005 live: None,
5006 },
5007 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
5008 let live = self
5034 .process_liveness
5035 .as_ref()
5036 .and_then(|source| source.process_live(&module_id))
5037 .unwrap_or(true);
5038 ClientControlResponse::RoutePoll {
5039 route_channel,
5040 route_epoch,
5041 status: None,
5042 live: Some(live),
5043 }
5044 }
5045 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
5046 route_channel,
5047 route_epoch,
5048 status: None,
5049 live: Some(false),
5050 },
5051 };
5052
5053 Ok(vec![control_response_body_frame(
5054 &frame,
5055 &response,
5056 "ClientControlResponse::RoutePoll",
5057 )?])
5058 }
5059
5060 pub(crate) fn observe_module_control_completion(
5061 &self,
5062 completion: ModuleControlRpcCompletion,
5063 ) -> bool {
5064 match completion {
5065 ModuleControlRpcCompletion::Unknown => false,
5066 ModuleControlRpcCompletion::Settled => true,
5067 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
5068 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
5069 info!(
5070 module_id = %module_id,
5071 latency_ms,
5072 "late health.check answer proves the module is alive"
5073 );
5074 match self
5075 .supervisor
5076 .record_late_health_answer(&module_id, latency_ms)
5077 {
5078 Ok(true) => {}
5079 Ok(false) => debug!(
5080 module_id = %module_id,
5081 latency_ms,
5082 "late health.check answer has no active supervisor snapshot"
5083 ),
5084 Err(err) => warn!(
5085 module_id = %module_id,
5086 latency_ms,
5087 error = %err,
5088 "failed to record late health.check answer"
5089 ),
5090 }
5091 true
5092 }
5093 }
5094 }
5095
5096 fn refuse_to_end_module_connection_for_a_client(
5117 &self,
5118 module_connection_id: ConnectionId,
5119 corr: u64,
5120 err: ForwardingError,
5121 ) -> Result<(), RouterError> {
5122 if let ForwardingError::ConnectionClosing { connection_id } = err {
5123 if connection_id != module_connection_id {
5124 warn!(
5125 module_connection_id = module_connection_id.get(),
5126 client_connection_id = connection_id.get(),
5127 corr,
5128 "dropping a route.bind response for a closing client; the module connection keeps serving"
5129 );
5130 return Ok(());
5131 }
5132 }
5133 Err(RouterError::Forwarding(err))
5134 }
5135
5136 fn handle_module_relay_response(
5137 &self,
5138 connection_id: ConnectionId,
5139 frame: Frame,
5140 ) -> Result<Vec<Frame>, RouterError> {
5141 let mut secondary_error = None;
5142 let outcome = match frame.header.ty {
5143 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
5144 Ok(probe) if probe.op == "route.bind" => {
5145 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
5146 Ok(ModuleControlResponse::RouteBindAck {}) => {
5147 RouteBindRelayOutcome::Accepted
5148 }
5149 Ok(other) => {
5150 let message =
5151 format!("route.bind response carried unexpected body: {other:?}");
5152 secondary_error = Some(control_error_frame(
5153 &frame,
5154 "invalid_control_body",
5155 message.clone(),
5156 )?);
5157 RouteBindRelayOutcome::ModuleGone(message)
5158 }
5159 Err(err) => {
5160 let message = format!("malformed route.bind response body: {err}");
5161 secondary_error = Some(control_error_frame(
5162 &frame,
5163 "invalid_control_body",
5164 message.clone(),
5165 )?);
5166 RouteBindRelayOutcome::ModuleGone(message)
5167 }
5168 }
5169 }
5170 Ok(probe) => {
5171 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
5172 {
5173 Ok(response) => ModuleControlRpcOutcome::Response(response),
5174 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
5175 "malformed {} response body: {err}",
5176 probe.op
5177 )),
5178 };
5179 let completion = self
5180 .forwarding
5181 .complete_module_control_rpc(
5182 connection_id,
5183 frame.header.corr,
5184 Some(&probe.op),
5185 outcome,
5186 )
5187 .map_err(RouterError::Forwarding)?;
5188 if !self.observe_module_control_completion(completion) {
5189 debug!(
5190 connection_id = connection_id.get(),
5191 corr = frame.header.corr,
5192 op = %probe.op,
5193 "dropping late or unknown module-control RPC response"
5194 );
5195 }
5196 return Ok(Vec::new());
5197 }
5198 Err(err) => {
5199 if let Some(expected_op) = self
5200 .forwarding
5201 .pending_module_control_op(connection_id, frame.header.corr)
5202 .map_err(RouterError::Forwarding)?
5203 {
5204 let completion = self
5205 .forwarding
5206 .complete_module_control_rpc(
5207 connection_id,
5208 frame.header.corr,
5209 None,
5210 ModuleControlRpcOutcome::MalformedResponse(format!(
5211 "malformed {expected_op} response body: {err}"
5212 )),
5213 )
5214 .map_err(RouterError::Forwarding)?;
5215 if !self.observe_module_control_completion(completion) {
5216 debug!(
5217 connection_id = connection_id.get(),
5218 corr = frame.header.corr,
5219 "dropping late malformed module-control RPC response"
5220 );
5221 }
5222 return Ok(Vec::new());
5223 }
5224 let message = format!("malformed route.bind response body: {err}");
5225 secondary_error = Some(control_error_frame(
5226 &frame,
5227 "invalid_control_body",
5228 message.clone(),
5229 )?);
5230 RouteBindRelayOutcome::ModuleGone(message)
5231 }
5232 },
5233 FrameType::Error => {
5234 if self
5235 .forwarding
5236 .pending_module_control_op(connection_id, frame.header.corr)
5237 .map_err(RouterError::Forwarding)?
5238 .is_some()
5239 {
5240 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
5241 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
5242 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
5243 "malformed module-control ERROR body: {err}"
5244 )),
5245 };
5246 let completion = self
5247 .forwarding
5248 .complete_module_control_rpc(
5249 connection_id,
5250 frame.header.corr,
5251 None,
5252 outcome,
5253 )
5254 .map_err(RouterError::Forwarding)?;
5255 if !self.observe_module_control_completion(completion) {
5256 debug!(
5257 connection_id = connection_id.get(),
5258 corr = frame.header.corr,
5259 "dropping late or unknown module-control RPC error"
5260 );
5261 }
5262 return Ok(Vec::new());
5263 }
5264 match serde_json::from_slice::<ErrorBody>(&frame.body) {
5265 Ok(body) => RouteBindRelayOutcome::Rejected(body),
5266 Err(err) => {
5267 let message = format!("malformed route.bind ERROR body: {err}");
5268 secondary_error = Some(control_error_frame(
5269 &frame,
5270 "invalid_control_body",
5271 message.clone(),
5272 )?);
5273 RouteBindRelayOutcome::ModuleGone(message)
5274 }
5275 }
5276 }
5277 ty => {
5278 return Ok(vec![control_error_frame(
5279 &frame,
5280 "unsupported_control_frame",
5281 format!("unsupported module channel-0 frame {ty:?}"),
5282 )?])
5283 }
5284 };
5285
5286 let settled =
5287 self.forwarding
5288 .complete_pending_relay(connection_id, frame.header.corr, outcome);
5289 let completion = match settled {
5290 Ok(completion) => completion,
5291 Err(err) => {
5292 self.refuse_to_end_module_connection_for_a_client(
5293 connection_id,
5294 frame.header.corr,
5295 err,
5296 )?;
5297 return Ok(secondary_error.into_iter().collect());
5298 }
5299 };
5300 if let Some(target) = completion.abandoned.as_ref() {
5301 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
5302 }
5303 if !completion.settled {
5304 debug!(
5305 connection_id = connection_id.get(),
5306 corr = frame.header.corr,
5307 frame_type = ?frame.header.ty,
5308 "dropping late or unknown route.bind relay response"
5309 );
5310 }
5311 Ok(secondary_error.into_iter().collect())
5312 }
5313
5314 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
5315 debug!(connection_id = connection_id.get(), "handling GOODBYE");
5316 let registrations = self
5317 .deregister_connection(connection_id)
5318 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
5319 let released_routes = self
5320 .forwarding
5321 .cleanup_connection(connection_id)
5322 .map_err(RouterError::Forwarding)?;
5323 self.emit_route_goodbyes(released_routes);
5324 if !registrations.is_empty() {
5326 crate::supervise::notify_registration_release();
5327 }
5328 Ok(Vec::new())
5329 }
5330}
5331
5332impl Default for ControlHandler {
5333 fn default() -> Self {
5334 Self::new(Arc::new(Registry::default()))
5335 }
5336}
5337
5338impl crate::supervise::SwapPromotionObserver for ControlHandler {
5339 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
5340 self.apply_registration_capabilities(registration);
5341 }
5342}
5343
5344fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
5345 CapabilityRequirementStatus {
5346 consumer: status.consumer,
5347 capability: status.capability,
5348 need: match status.need {
5349 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
5350 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
5351 },
5352 verdict: status.verdict.as_str().to_string(),
5353 episode_seq: status.episode_seq,
5354 config_satisfiable: status.config_satisfiable,
5355 runtime_available: status.runtime_available,
5356 detail: status.detail,
5357 }
5358}
5359
5360fn append_capability_problem_detail(
5361 detail: Option<String>,
5362 capability_detail: Option<String>,
5363) -> Option<String> {
5364 match (detail, capability_detail) {
5365 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
5366 (Some(detail), None) => Some(detail),
5367 (None, Some(capability_detail)) => Some(capability_detail),
5368 (None, None) => None,
5369 }
5370}
5371
5372fn subc_ops() -> Vec<String> {
5373 SUBC_CONTROL_OPS
5374 .iter()
5375 .map(|op| (*op).to_string())
5376 .collect()
5377}
5378
5379fn module_subc_ops() -> Vec<String> {
5380 SUBC_CONTROL_OPS
5381 .iter()
5382 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
5383 .map(|op| (*op).to_string())
5384 .collect()
5385}
5386
5387#[cfg(test)]
5388fn module_baseline_control_ops() -> Vec<String> {
5389 MODULE_BASELINE_CONTROL_OPS
5390 .iter()
5391 .map(|op| (*op).to_string())
5392 .collect()
5393}
5394
5395fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
5396 let mut seen = HashSet::new();
5397 let mut effective = Vec::new();
5398 for op in MODULE_BASELINE_CONTROL_OPS {
5399 if seen.insert((*op).to_string()) {
5400 effective.push((*op).to_string());
5401 }
5402 }
5403 for op in declared.unwrap_or_default() {
5404 if seen.insert(op.clone()) {
5405 effective.push(op);
5406 }
5407 }
5408 effective
5409}
5410
5411fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
5412 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
5413}
5414
5415fn target_module_id(target: &RouteTarget) -> &str {
5416 match target {
5417 RouteTarget::ToolProvider { module_id }
5418 | RouteTarget::ManagementSurface { module_id }
5419 | RouteTarget::InternalService { module_id, .. } => module_id,
5420 }
5421}
5422
5423fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
5424 roles.iter().any(|role| match (target, role) {
5425 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
5426 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
5427 (
5428 RouteTarget::InternalService { service_id, .. },
5429 ProviderRole::InternalService {
5430 service_id: provided,
5431 ..
5432 },
5433 ) => service_id == provided,
5434 _ => false,
5435 })
5436}
5437
5438fn is_routable_role(role: &ProviderRole) -> bool {
5439 matches!(
5440 role,
5441 ProviderRole::ToolProvider { .. }
5442 | ProviderRole::ManagementSurface { .. }
5443 | ProviderRole::InternalService { .. }
5444 )
5445}
5446
5447#[derive(Debug, Clone, Copy, PartialEq, Eq)]
5448enum ControlRequestBodyError {
5449 UnknownOp,
5450 InvalidBody,
5451}
5452
5453#[derive(Debug, Deserialize)]
5454struct ControlOpProbe {
5455 op: String,
5456}
5457
5458const MODULE_PUSH_OPS: &[&str] = &["route.status"];
5461
5462fn is_known_module_push_op(body: &[u8]) -> bool {
5463 serde_json::from_slice::<ControlOpProbe>(body)
5464 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
5465 .unwrap_or(false)
5466}
5467
5468fn is_known_module_request_op(body: &[u8]) -> bool {
5469 serde_json::from_slice::<ControlOpProbe>(body)
5470 .map(|probe| is_module_to_subc_op(&probe.op))
5471 .unwrap_or(false)
5472}
5473
5474fn is_module_to_subc_op(op: &str) -> bool {
5475 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
5476}
5477
5478fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
5479 debug!(
5480 op = %op,
5481 connection_id = connection_id.get(),
5482 corr,
5483 "control dispatch"
5484 );
5485}
5486
5487fn log_slow_control_dispatch(
5488 dispatch_started_at: Option<StdInstant>,
5489 op: &'static str,
5490 connection_id: ConnectionId,
5491 corr: u64,
5492) {
5493 let Some(dispatch_started_at) = dispatch_started_at else {
5494 return;
5495 };
5496 let elapsed = dispatch_started_at.elapsed();
5497 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
5498 warn!(
5499 op = %op,
5500 connection_id = connection_id.get(),
5501 corr,
5502 elapsed_ms = elapsed.as_millis() as u64,
5503 "slow control dispatch"
5504 );
5505 }
5506}
5507
5508fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
5509 match request {
5510 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
5511 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
5512 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
5513 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
5514 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
5515 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
5516 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
5517 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
5518 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
5519 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
5520 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
5521 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
5522 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
5523 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
5524 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
5525 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
5526 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
5527 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
5528 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
5529 }
5530}
5531
5532fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
5533 match request {
5534 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
5535 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
5536 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
5537 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
5538 }
5539}
5540
5541fn parse_client_control_request(
5542 body: &[u8],
5543) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
5544 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
5545 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5546 Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
5547 ControlRequestBodyError::InvalidBody
5548 }
5549 Ok(_) => ControlRequestBodyError::UnknownOp,
5550 Err(_) => ControlRequestBodyError::InvalidBody,
5551 };
5552 (err, classification)
5553 })
5554}
5555
5556fn parse_module_control_request_from_module(
5557 body: &[u8],
5558) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
5559 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
5560 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5561 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
5562 Ok(_) => ControlRequestBodyError::UnknownOp,
5563 Err(_) => ControlRequestBodyError::InvalidBody,
5564 };
5565 (err, classification)
5566 })
5567}
5568
5569#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
5570enum ProviderRoleKind {
5571 ToolProvider,
5572 PipelineStage,
5573 ManagementSurface,
5574 InternalService,
5575}
5576
5577fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
5578 match role {
5579 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
5580 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
5581 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
5582 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
5583 }
5584}
5585
5586fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
5587 roles.iter().map(provider_role_kind).collect()
5588}
5589
5590const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
5593
5594#[derive(Debug, Default, PartialEq, Eq)]
5596struct ScopeOutcomeCounts {
5597 created: usize,
5598 replaced: usize,
5599 updated: usize,
5600 unchanged: usize,
5601 refused: usize,
5602}
5603
5604impl ScopeOutcomeCounts {
5605 fn of(results: &[ScopeRecordResult]) -> Self {
5606 let mut counts = Self::default();
5607 for result in results {
5608 let slot = match result.outcome {
5609 ScopeRecordOutcome::Created => &mut counts.created,
5610 ScopeRecordOutcome::Replaced => &mut counts.replaced,
5611 ScopeRecordOutcome::Updated => &mut counts.updated,
5612 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
5613 ScopeRecordOutcome::Refused => &mut counts.refused,
5614 };
5615 *slot += 1;
5616 }
5617 counts
5618 }
5619}
5620
5621#[cfg(test)]
5622mod scope_outcome_count_tests {
5623 use super::*;
5624
5625 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
5626 ScopeRecordResult {
5627 scope_ref: "r".to_string(),
5628 scope_epoch: 1,
5629 outcome,
5630 code: None,
5631 message: None,
5632 version: None,
5633 parent_state: None,
5634 }
5635 }
5636
5637 #[test]
5640 fn every_outcome_is_counted_in_its_own_field() {
5641 let results = [
5642 result(ScopeRecordOutcome::Created),
5643 result(ScopeRecordOutcome::Created),
5644 result(ScopeRecordOutcome::Replaced),
5645 result(ScopeRecordOutcome::Updated),
5646 result(ScopeRecordOutcome::Unchanged),
5647 result(ScopeRecordOutcome::Refused),
5648 result(ScopeRecordOutcome::Refused),
5649 result(ScopeRecordOutcome::Refused),
5650 ];
5651 assert_eq!(
5652 ScopeOutcomeCounts::of(&results),
5653 ScopeOutcomeCounts {
5654 created: 2,
5655 replaced: 1,
5656 updated: 1,
5657 unchanged: 1,
5658 refused: 3,
5659 }
5660 );
5661 }
5662}
5663
5664fn capability_census_trigger(
5668 old: Option<&CapabilityDeclarations>,
5669 new: Option<&CapabilityDeclarations>,
5670) -> bool {
5671 let old_provides = old
5672 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
5673 .unwrap_or_default();
5674 let old_denies = old
5675 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
5676 .unwrap_or_default();
5677 let new = new.cloned().unwrap_or(CapabilityDeclarations {
5678 provides: Vec::new(),
5679 requires: Vec::new(),
5680 must_never_reach: Vec::new(),
5681 });
5682
5683 new.provides
5684 .iter()
5685 .any(|capability| !old_provides.contains(capability))
5686 || new
5687 .must_never_reach
5688 .iter()
5689 .any(|capability| !old_denies.contains(capability))
5690}
5691
5692fn denied_capability<'a>(
5695 opening_manifest: &'a ModuleManifest,
5696 target_manifest: &ModuleManifest,
5697) -> Option<&'a str> {
5698 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
5699 let target_capabilities = target_manifest.capabilities.as_ref()?;
5700 opening_capabilities
5701 .must_never_reach
5702 .iter()
5703 .find(|denied| {
5704 target_capabilities
5705 .provides
5706 .iter()
5707 .any(|provided| provided == *denied)
5708 })
5709 .map(String::as_str)
5710}
5711
5712fn catalog_update_frozen_field_message(
5713 registered: &ModuleManifest,
5714 provides: &[ProviderRole],
5715) -> Option<String> {
5716 let old_has_provides = !registered.provides.is_empty();
5717 let new_has_provides = !provides.is_empty();
5718 if old_has_provides != new_has_provides {
5719 return Some(format!(
5720 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
5721 registered.module_id
5722 ));
5723 }
5724
5725 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
5726 return Some(format!(
5727 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
5728 registered.module_id
5729 ));
5730 }
5731
5732 let registered_concurrency = manifest_concurrency(registered);
5733 let mut candidate = registered.clone();
5734 candidate.provides = provides.to_vec();
5735 let candidate_concurrency = manifest_concurrency(&candidate);
5736 if candidate_concurrency != registered_concurrency {
5737 return Some(format!(
5738 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
5739 registered.module_id, registered_concurrency, candidate_concurrency
5740 ));
5741 }
5742
5743 None
5746}
5747
5748fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
5749 manifest.provides.iter().any(is_routable_role)
5750}
5751
5752fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
5758 manifest
5759 .provides
5760 .iter()
5761 .find_map(|provider| match provider {
5762 ProviderRole::ToolProvider { concurrency, .. }
5763 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
5764 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
5765 })
5766 .unwrap_or(Concurrency::ModuleManaged)
5767}
5768
5769fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
5776 let has_management_surface = manifest
5777 .provides
5778 .iter()
5779 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
5780 if !has_management_surface {
5781 return false;
5782 }
5783 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
5784 return false;
5785 };
5786 let Some(provides) = raw
5787 .get("manifest")
5788 .and_then(|manifest| manifest.get("provides"))
5789 .and_then(serde_json::Value::as_array)
5790 else {
5791 return false;
5792 };
5793 provides.iter().any(|role| {
5798 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
5799 && role.get("concurrency").is_none()
5800 })
5801}
5802
5803fn negotiate_version(peer_version: u8) -> Result<u8, String> {
5804 if peer_version != PROTOCOL_VERSION {
5805 return Err(format!(
5806 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
5807 ));
5808 }
5809 Ok(PROTOCOL_VERSION)
5810}
5811
5812fn pong(frame: &Frame) -> Result<Frame, RouterError> {
5813 Frame::build_with_version(
5814 response_version(frame),
5815 FrameType::Pong,
5816 frame.header.flags,
5817 0,
5818 0,
5819 frame.header.corr,
5820 Vec::new(),
5821 )
5822 .map_err(RouterError::FrameBuild)
5823}
5824
5825fn control_error_frame(
5826 frame: &Frame,
5827 code: &'static str,
5828 message: impl Into<String>,
5829) -> Result<Frame, RouterError> {
5830 control_error_body_frame(
5831 frame,
5832 ErrorBody {
5833 code: code.to_string(),
5834 message: message.into(),
5835 detail: None,
5836 },
5837 )
5838}
5839
5840fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
5841 let body = serde_json::to_vec(&error).map_err(|err| {
5842 RouterError::backend(
5843 0,
5844 frame.header.corr,
5845 format!("failed to encode control ERROR: {err}"),
5846 )
5847 })?;
5848
5849 Frame::build_with_version(
5850 response_version(frame),
5851 FrameType::Error,
5852 control_flags(),
5853 0,
5854 0,
5855 frame.header.corr,
5856 body,
5857 )
5858 .map_err(RouterError::FrameBuild)
5859}
5860
5861fn control_response_body_frame<T: Serialize>(
5862 frame: &Frame,
5863 reply: &T,
5864 label: &'static str,
5865) -> Result<Frame, RouterError> {
5866 let body = serde_json::to_vec(reply).map_err(|err| {
5867 RouterError::backend(
5868 0,
5869 frame.header.corr,
5870 format!("failed to encode {label}: {err}"),
5871 )
5872 })?;
5873
5874 Frame::build_with_version(
5875 response_version(frame),
5876 FrameType::Response,
5877 control_flags(),
5878 0,
5879 0,
5880 frame.header.corr,
5881 body,
5882 )
5883 .map_err(RouterError::FrameBuild)
5884}
5885
5886fn forwarding_error_code(err: &ForwardingError) -> &'static str {
5916 match err {
5917 ForwardingError::NoModuleConnection => "target_unavailable",
5918 ForwardingError::ModuleReloading { .. } => "module_reloading",
5919 ForwardingError::ClientRouteChannelExhausted { .. }
5920 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
5921 ForwardingError::StaleModuleEndpoint
5922 | ForwardingError::UnknownReservation { .. }
5923 | ForwardingError::ConnectionClosing { .. }
5924 | ForwardingError::ClientEgressClosed { .. }
5925 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
5926 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
5929 ForwardingError::RelayCorrelationExhausted
5930 | ForwardingError::RouteOpenBuild(_)
5931 | ForwardingError::Poisoned => "forwarding_error",
5932 }
5933}
5934
5935fn response_version(frame: &Frame) -> u8 {
5936 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
5937 frame.header.ver
5938 } else {
5939 PROTOCOL_VERSION
5940 }
5941}
5942
5943fn control_flags() -> Flags {
5944 Flags::new(false, Priority::Passive, false)
5945}
5946
5947fn send_goodbye_target_best_effort(
5952 counters: &DaemonCounters,
5953 target: &GoodbyeTarget,
5954 context: &'static str,
5955) {
5956 let Ok(frame) = Frame::build_with_version(
5957 target.negotiated_ver,
5958 FrameType::Goodbye,
5959 control_flags(),
5960 target.channel,
5961 target.epoch,
5962 0,
5963 Vec::new(),
5964 ) else {
5965 return;
5966 };
5967 crate::forwarding::send_module_route_goodbye(
5968 counters,
5969 &target.sink,
5970 frame,
5971 target.module_id.as_deref(),
5972 context,
5973 );
5974}
5975
5976pub(crate) fn send_route_control_pushes(
5977 forwarding: &ForwardingTable,
5978 routes: Vec<EndpointRoute>,
5979 push: ClientControlPush,
5980) {
5981 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
5982 for route in routes {
5983 let target = route.goodbye_target;
5984 if let Some((existing, channels)) = targets
5985 .iter_mut()
5986 .find(|(existing, _)| existing.connection_id == target.connection_id)
5987 {
5988 debug_assert_eq!(
5989 existing.negotiated_ver, target.negotiated_ver,
5990 "one connection cannot negotiate multiple frame versions"
5991 );
5992 if !channels.contains(&target.channel) {
5993 channels.push(target.channel);
5994 }
5995 continue;
5996 }
5997 let channel = target.channel;
5998 targets.push((target, vec![channel]));
5999 }
6000 for (target, mut channels) in targets {
6001 channels.sort_unstable();
6002 let mut push = push.clone();
6003 match &mut push {
6004 ClientControlPush::RouteClosing {
6005 channels: covered, ..
6006 }
6007 | ClientControlPush::RouteClosed {
6008 channels: covered, ..
6009 } => *covered = channels,
6010 }
6011 let body = match serde_json::to_vec(&push) {
6012 Ok(body) => body,
6013 Err(err) => {
6014 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
6015 continue;
6016 }
6017 };
6018 let frame = match Frame::build_with_version(
6019 target.negotiated_ver,
6020 FrameType::Push,
6021 control_flags(),
6022 0,
6023 0,
6024 0,
6025 body.clone(),
6026 ) {
6027 Ok(frame) => frame,
6028 Err(err) => {
6029 warn!(
6030 route_channel = target.channel,
6031 error = %err,
6032 "failed to build route lifecycle control PUSH frame"
6033 );
6034 continue;
6035 }
6036 };
6037 if let Err(err) = target.sink.try_send(frame) {
6038 if target.close_on_delivery_failure() {
6039 warn!(
6040 target_connection_id = target.connection_id.get(),
6041 route_channel = target.channel,
6042 error = %err,
6043 "route lifecycle control PUSH was not delivered to client; closing target connection"
6044 );
6045 let _ = forwarding.escalate_client_delivery_failure(
6046 target.connection_id,
6047 target.channel,
6048 target.epoch,
6049 CloseReason::new(
6050 "route_lifecycle_push_delivery_failed",
6051 format!(
6052 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
6053 target.channel
6054 ),
6055 ),
6056 crate::forwarding::UndeliveredFrame {
6057 module_id: target.module_id.as_deref(),
6058 sink: &target.sink,
6059 },
6060 );
6061 }
6062 }
6063 }
6064}
6065
6066#[cfg(test)]
6067mod tests {
6068 use std::{
6069 collections::BTreeMap,
6070 fmt,
6071 path::PathBuf,
6072 sync::{Arc, Mutex},
6073 time::Duration,
6074 };
6075 use subc_test_support::TestTempDir;
6076
6077 use serde_json::{json, Value};
6078 use subc_protocol::{
6079 manifest::{
6080 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
6081 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
6082 },
6083 session::HealthStatus,
6084 FrameType,
6085 };
6086
6087 use super::*;
6088 use crate::{
6089 forwarding::{DataRoute, DataRouteState},
6090 registry::ChannelState,
6091 router::FrameSink,
6092 stderr_tail::DEFAULT_MAX_LINE_BYTES,
6093 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
6094 RouteCtx, Router,
6095 };
6096 use tokio::{
6097 sync::mpsc,
6098 time::{sleep, Instant},
6099 };
6100 use tracing::{
6101 field::{Field, Visit},
6102 Event, Subscriber,
6103 };
6104 use tracing_subscriber::{layer::Context, prelude::*, Layer};
6105
6106 fn fake_aft_stub_path() -> PathBuf {
6125 let mut path = std::env::current_exe().expect("current_exe available in tests");
6126 path.pop(); path.pop(); path.push(if cfg!(windows) {
6129 "fake-aft-stub.exe"
6130 } else {
6131 "fake-aft-stub"
6132 });
6133 assert!(
6134 path.exists(),
6135 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
6136 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
6137 path.display()
6138 );
6139 path
6140 }
6141
6142 fn client_retries(code: &str) -> bool {
6147 subc_protocol::error_codes::is_retryable_route_open(code)
6148 }
6149
6150 #[test]
6161 fn retryability_of_forwarding_codes_matches_the_failure() {
6162 let transient = [
6165 ForwardingError::NoModuleConnection,
6166 ForwardingError::ModuleReloading {
6167 module_id: "m".into(),
6168 },
6169 ForwardingError::StaleModuleEndpoint,
6170 ForwardingError::UnknownReservation {
6171 client_channel: 1,
6172 module_channel: 1,
6173 },
6174 ForwardingError::ConnectionClosing {
6175 connection_id: ConnectionId::new(1),
6176 },
6177 ForwardingError::ClientEgressClosed {
6178 connection_id: ConnectionId::new(1),
6179 },
6180 ForwardingError::ModuleEgressUnavailable {
6181 connection_id: ConnectionId::new(1),
6182 },
6183 ];
6184 for err in transient {
6185 let code = forwarding_error_code(&err);
6186 assert!(
6187 client_retries(code),
6188 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
6189 );
6190 }
6191
6192 let permanent = [
6197 ForwardingError::ClientRouteChannelExhausted {
6198 connection_id: ConnectionId::new(1),
6199 },
6200 ForwardingError::ModuleRouteChannelExhausted {
6201 endpoint: ModuleEndpointId {
6202 connection_id: ConnectionId::new(1),
6203 generation: 1,
6204 },
6205 },
6206 ForwardingError::RelayCorrelationExhausted,
6207 ForwardingError::RouteOpenBuild("x".into()),
6208 ForwardingError::Poisoned,
6209 ];
6210 for err in permanent {
6211 let code = forwarding_error_code(&err);
6212 assert!(
6213 !client_retries(code),
6214 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
6215 );
6216 }
6217 }
6218
6219 #[tokio::test]
6247 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
6248 let handler = ControlHandler::default();
6249 let frame =
6250 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
6251
6252 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
6256 assert_eq!(
6257 stamped,
6258 Principal::Direct,
6259 "a caller that proved nothing must not be stamped as a supervised module"
6260 );
6261
6262 let forged = handler
6267 .route_open_principal(
6268 &frame,
6269 Some(ConsumerIdentity {
6270 module_id: "aft".to_string(),
6271 launch_nonce: "not-a-real-nonce".to_string(),
6272 }),
6273 )
6274 .unwrap();
6275 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
6276 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
6277 }
6278
6279 #[test]
6296 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
6297 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
6298 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
6299 let ClientControlRequest::RouteOpen {
6300 consumer_identity, ..
6301 } = parsed
6302 else {
6303 panic!("route.open body must parse as RouteOpen");
6304 };
6305 assert_eq!(
6306 consumer_identity,
6307 Some(ConsumerIdentity {
6308 module_id: "aft".to_string(),
6309 launch_nonce: "n".to_string(),
6310 }),
6311 "the wire field name must reach the value route_open_principal reads"
6312 );
6313 }
6314
6315 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
6316 ModuleManifest::builder(module_id, "0.1.0")
6317 .protocol_ver(protocol_ver)
6318 .provides(vec![ProviderRole::ToolProvider {
6319 tools: vec![Tool {
6320 name: "read".to_string(),
6321 description: None,
6322 execution_mode: ExecutionMode::Pure,
6323 schema: json!({"type": "object"}),
6324 }],
6325 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
6326 concurrency: Concurrency::ModuleManaged,
6327 emits_push: true,
6328 sub_supervises: true,
6329 }])
6330 .build()
6331 }
6332
6333 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
6334 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
6335 }
6336
6337 fn hello_frame_with_control_ops(
6338 module_id: &str,
6339 protocol_ver: u8,
6340 corr: u64,
6341 control_ops: Option<Vec<String>>,
6342 ) -> Frame {
6343 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
6344 }
6345
6346 fn hello_frame_with_nonce(
6347 module_id: &str,
6348 protocol_ver: u8,
6349 corr: u64,
6350 launch_nonce: Option<&str>,
6351 ) -> Frame {
6352 hello_frame_full(
6353 module_id,
6354 protocol_ver,
6355 corr,
6356 None,
6357 launch_nonce.map(ToOwned::to_owned),
6358 )
6359 }
6360
6361 fn hello_frame_full(
6362 module_id: &str,
6363 protocol_ver: u8,
6364 corr: u64,
6365 control_ops: Option<Vec<String>>,
6366 launch_nonce: Option<String>,
6367 ) -> Frame {
6368 let body = serde_json::to_vec(&ModuleHelloBody {
6369 manifest: manifest(module_id, protocol_ver),
6370 protocol_ver,
6371 control_ops,
6372 launch_nonce,
6373 })
6374 .unwrap();
6375 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
6376 }
6377
6378 fn non_routable_hello_frame_with_control_ops(
6379 module_id: &str,
6380 corr: u64,
6381 control_ops: Option<Vec<String>>,
6382 ) -> Frame {
6383 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
6384 manifest.provides.clear();
6385 let body = serde_json::to_vec(&ModuleHelloBody {
6386 manifest,
6387 protocol_ver: PROTOCOL_VERSION,
6388 control_ops,
6389 launch_nonce: None,
6390 })
6391 .unwrap();
6392 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
6393 }
6394
6395 fn capability_grammar_hello_frame(
6396 capabilities: Value,
6397 runtime_computed: Option<Value>,
6398 corr: u64,
6399 ) -> Frame {
6400 let mut body = serde_json::to_value(ModuleHelloBody {
6401 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
6402 protocol_ver: PROTOCOL_VERSION,
6403 control_ops: None,
6404 launch_nonce: None,
6405 })
6406 .expect("HELLO body serializes");
6407 body["manifest"]["capabilities"] = capabilities;
6408 if let Some(runtime_computed) = runtime_computed {
6409 body["runtime_computed"] = runtime_computed;
6410 }
6411 Frame::build(
6412 FrameType::Hello,
6413 control_flags(),
6414 0,
6415 0,
6416 corr,
6417 serde_json::to_vec(&body).expect("HELLO body reserializes"),
6418 )
6419 .expect("HELLO frame builds")
6420 }
6421
6422 fn channel_request(channel: u16, corr: u64) -> Frame {
6423 Frame::build(
6424 FrameType::Request,
6425 Flags::new(true, Priority::Interactive, false),
6426 channel,
6427 0,
6428 corr,
6429 b"opaque".to_vec(),
6430 )
6431 .unwrap()
6432 }
6433
6434 fn route_ctx(
6435 connection_id: ConnectionId,
6436 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
6437 let (tx, rx) = mpsc::channel(8);
6438 (
6439 RouteCtx {
6440 connection_id,
6441 egress: FrameSink::new(tx),
6442 },
6443 rx,
6444 )
6445 }
6446
6447 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
6448 serde_json::from_slice(&frame.body).unwrap()
6449 }
6450
6451 async fn hello_via_sink(
6456 handler: &ControlHandler,
6457 ctx: &RouteCtx,
6458 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
6459 hello: Frame,
6460 ) -> Frame {
6461 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
6462 assert!(
6463 replies.is_empty(),
6464 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
6465 );
6466 let ack = rx
6467 .try_recv()
6468 .expect("HELLO_ACK is queued on the module sink")
6469 .frame;
6470 assert_eq!(ack.header.ty, FrameType::HelloAck);
6471 ack
6472 }
6473
6474 fn parse_error(frame: &Frame) -> Value {
6475 serde_json::from_slice(&frame.body).unwrap()
6476 }
6477
6478 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
6479 serde_json::from_slice(&frame.body).unwrap()
6480 }
6481
6482 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
6483 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
6484 route_channel,
6485 route_epoch: 0,
6486 kind,
6487 })
6488 .unwrap();
6489 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6490 }
6491
6492 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
6493 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
6494 module_id: module_id.to_string(),
6495 })
6496 .unwrap();
6497 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6498 }
6499
6500 fn route_open_frame(corr: u64, module_id: &str, project_root: TestTempDir) -> Frame {
6501 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
6502 }
6503
6504 fn route_open_frame_with_consumer_capabilities(
6505 corr: u64,
6506 module_id: &str,
6507 project_root: TestTempDir,
6508 consumer_capabilities: Option<Vec<String>>,
6509 ) -> Frame {
6510 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6511 target: RouteTarget::ToolProvider {
6512 module_id: module_id.to_string(),
6513 },
6514 identity: BindIdentity::new(
6515 project_root.path().to_path_buf(),
6516 "unit".to_string(),
6517 "session".to_string(),
6518 ),
6519 consumer_identity: None,
6520 consumer_capabilities,
6521 role_versions: None,
6522 admission_facts: None,
6523 scope: None,
6524 })
6525 .unwrap();
6526 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6527 }
6528
6529 fn route_open_frame_with_role_versions(
6530 corr: u64,
6531 module_id: &str,
6532 project_root: TestTempDir,
6533 role_versions: Option<BTreeMap<String, String>>,
6534 ) -> Frame {
6535 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6536 target: RouteTarget::ToolProvider {
6537 module_id: module_id.to_string(),
6538 },
6539 identity: BindIdentity::new(
6540 project_root.path().to_path_buf(),
6541 "unit".to_string(),
6542 format!("session-{corr}"),
6543 ),
6544 consumer_identity: None,
6545 consumer_capabilities: None,
6546 role_versions,
6547 admission_facts: None,
6548 scope: None,
6549 })
6550 .unwrap();
6551 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6552 }
6553
6554 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
6555 entries
6556 .iter()
6557 .map(|(role, version)| (role.to_string(), version.to_string()))
6558 .collect()
6559 }
6560
6561 fn route_open_frame_with_admission_facts(
6562 corr: u64,
6563 module_id: &str,
6564 project_root: TestTempDir,
6565 consumer_identity: Option<subc_control::ConsumerIdentity>,
6566 facts: Option<Value>,
6567 ) -> Frame {
6568 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6569 target: RouteTarget::ToolProvider {
6570 module_id: module_id.to_string(),
6571 },
6572 identity: BindIdentity::new(
6573 project_root.path().to_path_buf(),
6574 "unit".to_string(),
6575 format!("session-{corr}"),
6576 ),
6577 consumer_identity,
6578 consumer_capabilities: None,
6579 role_versions: None,
6580 admission_facts: facts,
6581 scope: None,
6582 })
6583 .unwrap();
6584 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6585 }
6586
6587 #[derive(Clone, Default)]
6588 struct EventCapture {
6589 events: Arc<Mutex<Vec<CapturedEvent>>>,
6590 }
6591
6592 #[derive(Clone, Debug)]
6593 struct CapturedEvent {
6594 target: String,
6595 level: tracing::Level,
6596 fields: BTreeMap<String, String>,
6597 }
6598
6599 impl EventCapture {
6600 fn events(&self) -> Vec<CapturedEvent> {
6601 self.events.lock().unwrap().clone()
6602 }
6603 }
6604
6605 impl<S> Layer<S> for EventCapture
6606 where
6607 S: Subscriber,
6608 {
6609 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
6610 let mut visitor = EventFieldVisitor::default();
6611 event.record(&mut visitor);
6612 self.events.lock().unwrap().push(CapturedEvent {
6613 target: event.metadata().target().to_string(),
6614 level: *event.metadata().level(),
6615 fields: visitor.fields,
6616 });
6617 }
6618 }
6619
6620 #[derive(Default)]
6621 struct EventFieldVisitor {
6622 fields: BTreeMap<String, String>,
6623 }
6624
6625 impl Visit for EventFieldVisitor {
6626 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
6627 self.fields
6628 .insert(field.name().to_string(), format!("{value:?}"));
6629 }
6630 }
6631
6632 fn health_response(corr: u64, status: HealthStatus) -> Frame {
6633 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
6634 status,
6635 detail: Some("warming".to_string()),
6636 metrics: Some(json!({"queue_depth": 3})),
6637 })
6638 .unwrap();
6639 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
6640 }
6641
6642 fn route_bind_ack(corr: u64) -> Frame {
6643 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
6644 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
6645 }
6646
6647 fn unique_project_root(label: &str) -> TestTempDir {
6648 TestTempDir::new(label)
6649 }
6650
6651 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
6652 match parse_route_poll(frame) {
6653 ClientControlResponse::RoutePoll {
6654 status: None,
6655 live: Some(live),
6656 ..
6657 } => assert_eq!(live, expected_live),
6658 other => panic!("unexpected route.poll response: {other:?}"),
6659 }
6660 }
6661
6662 fn bind_liveness_route(
6663 registry: &Registry,
6664 forwarding: &ForwardingTable,
6665 module_id: &str,
6666 ) -> (RouteCtx, u16, u32) {
6667 let module_connection = ConnectionId::new(101);
6668 let client_connection = ConnectionId::new(202);
6669 let registration = registry
6670 .register_with_control_ops(
6671 manifest(module_id, PROTOCOL_VERSION),
6672 PROTOCOL_VERSION,
6673 module_connection,
6674 module_baseline_control_ops(),
6675 )
6676 .unwrap();
6677 let (module_tx, _module_rx) = mpsc::channel(8);
6678 let endpoint = forwarding
6679 .register_module_connection(
6680 module_connection,
6681 module_id.to_string(),
6682 PROTOCOL_VERSION,
6683 manifest_concurrency(®istration.manifest),
6684 FrameSink::new(module_tx),
6685 )
6686 .unwrap();
6687 let (client_ctx, _client_rx) = route_ctx(client_connection);
6688 let pending = forwarding
6689 .begin_route_bind_relay_for_test(
6690 client_connection,
6691 client_ctx.egress.clone(),
6692 1,
6693 module_id,
6694 )
6695 .unwrap();
6696 assert_eq!(pending.endpoint, endpoint);
6697 let route_channel = pending.client_channel;
6698 let route_epoch = pending.client_epoch;
6699 forwarding
6700 .complete_pending_relay(
6701 module_connection,
6702 pending.corr,
6703 RouteBindRelayOutcome::Accepted,
6704 )
6705 .unwrap();
6706 (client_ctx, route_channel, route_epoch)
6707 }
6708
6709 struct FakeProcessLiveness {
6710 live: Option<bool>,
6711 }
6712
6713 impl ModuleProcessLiveness for FakeProcessLiveness {
6714 fn process_live(&self, _module_id: &str) -> Option<bool> {
6715 self.live
6716 }
6717 }
6718
6719 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6720 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
6721 {
6722 let registry = Arc::new(Registry::default());
6723 let supervisor_handle = SupervisorHandle::new();
6724 let supervisor = Supervisor::new(
6725 Arc::clone(®istry),
6726 RestartPolicy::new(1, Duration::from_millis(10)),
6727 )
6728 .with_handle(supervisor_handle.clone());
6729 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
6730 let module = supervisor
6731 .spawn(ModuleSpec {
6732 module_id: "stderr-tail-wire".to_string(),
6733 program: fake_aft_stub_path(),
6734 args: Vec::new(),
6735 env: vec![
6736 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
6737 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
6738 ],
6739 reserved: false,
6740 reserved_prefixes: Vec::new(),
6741 protocol: ModuleProtocol::Subc,
6742 overlap: Default::default(),
6743 })
6744 .unwrap();
6745
6746 let deadline = Instant::now() + Duration::from_secs(5);
6747 loop {
6748 let tail = module.stderr_tail(None, None);
6749 if tail
6750 .entries
6751 .iter()
6752 .any(|entry| matches!(entry, TailEntry::ProcessStart))
6753 && tail.entries.iter().any(|entry| {
6754 matches!(
6755 entry,
6756 TailEntry::Line {
6757 truncated: true,
6758 ..
6759 }
6760 )
6761 })
6762 {
6763 break;
6764 }
6765 assert!(
6766 Instant::now() < deadline,
6767 "module did not produce a truncated line and restart boundary: {tail:?}"
6768 );
6769 sleep(Duration::from_millis(10)).await;
6770 }
6771
6772 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
6773 let request = ClientControlRequest::SupervisorStderrTail {
6774 module_id: "stderr-tail-wire".to_string(),
6775 max_lines: None,
6776 max_bytes: None,
6777 };
6778 let frame = Frame::build(
6779 FrameType::Request,
6780 control_flags(),
6781 0,
6782 0,
6783 1,
6784 serde_json::to_vec(&request).unwrap(),
6785 )
6786 .unwrap();
6787 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6788 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
6789 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
6790 serde_json::from_slice(&responses[0].body).unwrap()
6791 else {
6792 panic!("expected supervisor.stderr_tail response");
6793 };
6794
6795 assert!(
6796 tail.entries
6797 .iter()
6798 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
6799 "the control response lost the restart boundary"
6800 );
6801 let Some(StderrTailEntry::Line {
6802 text,
6803 truncated,
6804 at_ms,
6805 }) = tail.entries.iter().find(|entry| {
6806 matches!(
6807 entry,
6808 StderrTailEntry::Line {
6809 truncated: true,
6810 ..
6811 }
6812 )
6813 })
6814 else {
6815 panic!("the control response lost the truncated line");
6816 };
6817 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
6818 assert!(*truncated);
6819 assert!(
6820 at_ms.is_some(),
6821 "the control response lost the line's capture time"
6822 );
6823 }
6824
6825 #[tokio::test(flavor = "current_thread")]
6830 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
6831 let dir = TestTempDir::new("terminals-off-worker");
6832 let journal_path = dir.join("terminals.jsonl");
6833 let registry = Arc::new(Registry::default());
6834 let supervisor_handle = SupervisorHandle::new();
6835 let supervisor =
6836 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
6837 .with_handle(supervisor_handle.clone())
6838 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
6839 let module = supervisor
6840 .spawn(ModuleSpec {
6841 module_id: "terminal-off-worker".to_string(),
6842 program: fake_aft_stub_path(),
6843 args: Vec::new(),
6844 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
6845 reserved: false,
6846 reserved_prefixes: Vec::new(),
6847 protocol: ModuleProtocol::Subc,
6848 overlap: Default::default(),
6849 })
6850 .unwrap();
6851 let deadline = Instant::now() + Duration::from_secs(5);
6852 while module.terminal_history().entries.len() != 2 {
6853 assert!(Instant::now() < deadline, "module did not record two exits");
6854 sleep(Duration::from_millis(10)).await;
6855 }
6856
6857 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
6858 let handler =
6859 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
6860 let frame = Frame::build(
6861 FrameType::Request,
6862 control_flags(),
6863 0,
6864 0,
6865 1,
6866 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
6867 module_id: "terminal-off-worker".to_string(),
6868 })
6869 .unwrap(),
6870 )
6871 .unwrap();
6872 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6873 let spawned_at = std::time::Instant::now();
6874 let read = tokio::spawn({
6875 let handler = Arc::clone(&handler);
6876 async move { handler.handle_control_frame(&ctx, frame).await }
6877 });
6878 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
6881 .await
6882 .unwrap()
6883 .expect("the history read reached its pause");
6884 let elapsed = spawned_at.elapsed();
6885 assert!(
6886 elapsed < Duration::from_secs(2) && !read.is_finished(),
6887 "this task could not run while the history read was paused \
6888 (resumed after {elapsed:?}, read finished: {})",
6889 read.is_finished()
6890 );
6891
6892 drop(release);
6893 let responses = read.await.unwrap().unwrap();
6894 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
6895 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
6896 panic!("expected supervisor.terminals response");
6897 };
6898 assert_eq!(terminals.entries.len(), 2);
6899 assert_eq!(terminals.journal_skipped_lines, 0);
6900 assert_eq!(terminals.journal_read_errors, 0);
6901 }
6902
6903 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6904 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
6905 let registry = Arc::new(Registry::default());
6906 let supervisor_handle = SupervisorHandle::new();
6907 let supervisor =
6908 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
6909 .with_handle(supervisor_handle.clone());
6910 let module = supervisor
6911 .spawn(ModuleSpec {
6912 module_id: "terminal-golden".to_string(),
6913 program: fake_aft_stub_path(),
6914 args: Vec::new(),
6915 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
6916 reserved: false,
6917 reserved_prefixes: Vec::new(),
6918 protocol: ModuleProtocol::Subc,
6919 overlap: Default::default(),
6920 })
6921 .unwrap();
6922
6923 let deadline = Instant::now() + Duration::from_secs(5);
6924 while module.terminal_history().entries.len() != 2 {
6925 assert!(
6926 Instant::now() < deadline,
6927 "module did not retain two terminal exits: {:?}",
6928 module.terminal_history()
6929 );
6930 sleep(Duration::from_millis(10)).await;
6931 }
6932
6933 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
6934 let request = ClientControlRequest::SupervisorTerminals {
6935 module_id: "terminal-golden".to_string(),
6936 };
6937 let frame = Frame::build(
6938 FrameType::Request,
6939 control_flags(),
6940 0,
6941 0,
6942 1,
6943 serde_json::to_vec(&request).unwrap(),
6944 )
6945 .unwrap();
6946 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6947 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
6948 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
6949 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
6950 panic!("expected supervisor.terminals response");
6951 };
6952 assert_eq!(terminals.entries.len(), 2);
6953 assert_eq!(terminals.dropped, 0);
6954
6955 let mut rendered = serde_json::to_value(response).unwrap();
6956 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
6959 for (index, entry) in rendered["entries"]
6960 .as_array_mut()
6961 .expect("terminal response entries array")
6962 .iter_mut()
6963 .enumerate()
6964 {
6965 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
6966 }
6967
6968 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
6969 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
6970 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
6971 if std::env::var_os("UPDATE_GOLDEN").is_some() {
6972 std::fs::write(&golden_path, &serialized).unwrap();
6973 }
6974 let expected: Value =
6975 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
6976 assert_eq!(rendered, expected);
6977 }
6978
6979 #[test]
6980 fn hello_registers_manifest_and_returns_ack() {
6981 let registry = Arc::new(Registry::default());
6982 let handler = ControlHandler::new(Arc::clone(®istry));
6983 let conn = ConnectionId::new(1);
6984
6985 let responses = handler
6986 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
6987 .unwrap();
6988
6989 assert_eq!(responses.len(), 1);
6990 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
6991 assert_eq!(responses[0].header.channel, 0);
6992 assert_eq!(responses[0].header.corr, 7);
6993 let ack = parse_ack(&responses[0]);
6994 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
6995 assert!(ack
6996 .subc_capabilities
6997 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
6998 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
6999 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
7000 assert!(ack
7001 .subc_ops
7002 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
7003 assert!(ack
7004 .subc_ops
7005 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
7006
7007 let registration = registry.get_module("aft").unwrap().unwrap();
7008 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
7009 assert_eq!(registration.state, ChannelState::Active);
7010 assert_eq!(registration.connection_id, conn);
7011 assert_eq!(registration.control_ops, module_baseline_control_ops());
7012 }
7013
7014 #[test]
7015 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
7016 let invalid_identifiers = [
7017 ("case_change", "credentials-Provider/v1"),
7018 ("leading_zero", "credentials-provider/v01"),
7019 ("trailing_hyphen", "credentials-provider-/v1"),
7020 ("consecutive_hyphens", "credentials--provider/v1"),
7021 ("uppercase", "Credentials-provider/v1"),
7022 ("missing_v", "credentials-provider/1"),
7023 ("whitespace", "credentials provider/v1"),
7024 ("zero_version", "credentials-provider/v0"),
7025 ("out_of_range_version", "credentials-provider/v4294967296"),
7026 (
7027 "overlength_name",
7028 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
7029 ),
7030 ];
7031 let mut cases = invalid_identifiers
7032 .into_iter()
7033 .map(|(name, identifier)| {
7034 (
7035 format!("identifier_{name}"),
7036 "capabilities.provides[0]".to_string(),
7037 identifier.to_string(),
7038 json!({ "provides": [identifier] }),
7039 None,
7040 )
7041 })
7042 .collect::<Vec<_>>();
7043 cases.extend([
7044 (
7045 "unknown_need".to_string(),
7046 "capabilities.requires[0].need".to_string(),
7047 "deferred".to_string(),
7048 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
7049 None,
7050 ),
7051 (
7052 "duplicate_provides".to_string(),
7053 "capabilities.provides[1]".to_string(),
7054 "credentials-provider/v1".to_string(),
7055 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
7056 None,
7057 ),
7058 (
7059 "duplicate_must_never_reach".to_string(),
7060 "capabilities.must_never_reach[1]".to_string(),
7061 "credentials-provider/v1".to_string(),
7062 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
7063 None,
7064 ),
7065 (
7066 "duplicate_requires_same_need".to_string(),
7067 "capabilities.requires[1]".to_string(),
7068 "credentials-provider/v1".to_string(),
7069 json!({ "requires": [
7070 { "capability": "credentials-provider/v1", "need": "required" },
7071 { "capability": "credentials-provider/v1", "need": "required" }
7072 ] }),
7073 None,
7074 ),
7075 (
7076 "duplicate_requires_conflicting_need".to_string(),
7077 "capabilities.requires[1]".to_string(),
7078 "credentials-provider/v1".to_string(),
7079 json!({ "requires": [
7080 { "capability": "credentials-provider/v1", "need": "required" },
7081 { "capability": "credentials-provider/v1", "need": "optional" }
7082 ] }),
7083 None,
7084 ),
7085 (
7086 "capabilities_root_pointer".to_string(),
7087 "runtime_computed[0]".to_string(),
7088 "/capabilities".to_string(),
7089 json!({}),
7090 Some(json!(["/capabilities"])),
7091 ),
7092 (
7093 "capabilities_descendant_pointer".to_string(),
7094 "runtime_computed[0]".to_string(),
7095 "/capabilities/provides".to_string(),
7096 json!({}),
7097 Some(json!(["/capabilities/provides"])),
7098 ),
7099 (
7100 "malformed_pointer_without_leading_slash".to_string(),
7101 "runtime_computed[0]".to_string(),
7102 "capabilities".to_string(),
7103 json!({}),
7104 Some(json!(["capabilities"])),
7105 ),
7106 (
7107 "malformed_pointer_escape".to_string(),
7108 "runtime_computed[0]".to_string(),
7109 "/roles/~2/tools".to_string(),
7110 json!({}),
7111 Some(json!(["/roles/~2/tools"])),
7112 ),
7113 (
7114 "unknown_capabilities_field".to_string(),
7115 "capabilities.future".to_string(),
7116 "<array>".to_string(),
7117 json!({ "future": [] }),
7118 None,
7119 ),
7120 ]);
7121
7122 for (index, (name, field, value, capabilities, runtime_computed)) in
7123 cases.into_iter().enumerate()
7124 {
7125 let registry = Arc::new(Registry::default());
7126 let handler = ControlHandler::new(Arc::clone(®istry));
7127 let response = handler
7128 .handle_control(
7129 ConnectionId::new((index + 1) as u64),
7130 capability_grammar_hello_frame(
7131 capabilities,
7132 runtime_computed,
7133 index as u64 + 1,
7134 ),
7135 )
7136 .expect("invalid HELLO returns a refusal");
7137
7138 assert_eq!(response.len(), 1, "{name} must emit one refusal");
7139 let error = parse_error(&response[0]);
7140 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
7141 let message = error["message"]
7142 .as_str()
7143 .expect("error message is a string");
7144 assert!(
7145 message.contains(&field),
7146 "{name}: field missing from {message}"
7147 );
7148 assert!(
7149 message.contains(&value),
7150 "{name}: value missing from {message}"
7151 );
7152 assert_eq!(
7153 registry
7154 .active_registration_count()
7155 .expect("registry reads"),
7156 0,
7157 "{name}: refused HELLO must not create a catalog entry"
7158 );
7159 }
7160 }
7161
7162 #[test]
7163 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
7164 let registry = Arc::new(Registry::default());
7165 let handler = ControlHandler::new(Arc::clone(®istry));
7166 let capabilities = json!({
7167 "provides": ["credentials-provider/v1"],
7168 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
7169 "must_never_reach": ["federation-transport/v1"]
7170 });
7171 let response = handler
7172 .handle_control(
7173 ConnectionId::new(99),
7174 capability_grammar_hello_frame(
7175 capabilities.clone(),
7176 Some(json!(["/roles/0/tools"])),
7177 99,
7178 ),
7179 )
7180 .expect("valid HELLO registers");
7181 assert_eq!(response[0].header.ty, FrameType::HelloAck);
7182
7183 let request = Frame::build(
7184 FrameType::Request,
7185 control_flags(),
7186 0,
7187 0,
7188 100,
7189 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7190 .expect("catalog request serializes"),
7191 )
7192 .expect("catalog request frame builds");
7193 let response = handler
7194 .handle_catalog_list(request, None)
7195 .expect("catalog list succeeds");
7196 let ClientControlResponse::CatalogList { modules, .. } =
7197 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
7198 else {
7199 panic!("catalog request must return catalog.list");
7200 };
7201 assert_eq!(modules.len(), 1);
7202 assert_eq!(
7203 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
7204 capabilities
7205 );
7206 }
7207
7208 #[test]
7209 fn catalog_list_mirrors_management_operation_description() {
7210 let registry = Arc::new(Registry::default());
7211 let handler = ControlHandler::new(Arc::clone(®istry));
7212 let description = "List managed records and return their identifiers and metadata.";
7213 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
7214 manifest.provides = vec![ProviderRole::ManagementSurface {
7215 operations: vec![ManagementOperation {
7216 name: "records.list".to_string(),
7217 kind: ManagementOperationKind::Query,
7218 description: Some(description.to_string()),
7219 }],
7220 config_schema: json!({"type": "object"}),
7221 observability: vec![ObservabilitySurface {
7222 name: "records.stats".to_string(),
7223 kind: ObservabilityKind::Snapshot,
7224 }],
7225 identity_scope: vec![IdentityScope::Project],
7226 concurrency: Concurrency::ModuleManaged,
7227 }];
7228 registry
7229 .register_with_control_ops(
7230 manifest,
7231 PROTOCOL_VERSION,
7232 ConnectionId::new(99),
7233 Vec::new(),
7234 )
7235 .expect("described management manifest registers");
7236
7237 let request = Frame::build(
7238 FrameType::Request,
7239 control_flags(),
7240 0,
7241 0,
7242 100,
7243 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7244 .expect("catalog request serializes"),
7245 )
7246 .expect("catalog request frame builds");
7247 let response = handler
7248 .handle_catalog_list(request, None)
7249 .expect("catalog list succeeds");
7250 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
7251 assert_eq!(
7252 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
7253 "catalog.list must preserve the declared operation description verbatim"
7254 );
7255 }
7256
7257 #[test]
7258 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
7259 let registry = Arc::new(Registry::default());
7260 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
7261 [("vault".to_string(), true), ("squatter".to_string(), true)],
7262 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
7263 );
7264 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
7265 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
7266 provides: vec!["credentials-provider/v1".to_string()],
7267 requires: Vec::new(),
7268 must_never_reach: Vec::new(),
7269 });
7270 let frame = Frame::build(
7271 FrameType::Hello,
7272 control_flags(),
7273 0,
7274 0,
7275 77,
7276 serde_json::to_vec(&ModuleHelloBody {
7277 manifest: squatter,
7278 protocol_ver: PROTOCOL_VERSION,
7279 control_ops: None,
7280 launch_nonce: None,
7281 })
7282 .expect("HELLO serializes"),
7283 )
7284 .expect("HELLO frame builds");
7285 let response = handler
7286 .handle_control(ConnectionId::new(77), frame)
7287 .expect("reserved claim receives a typed refusal");
7288 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
7289 assert_eq!(
7290 registry
7291 .active_registration_count()
7292 .expect("registry reads"),
7293 0,
7294 "a reserved capability refusal must not leave a catalog entry"
7295 );
7296 }
7297
7298 #[test]
7299 fn server_describe_surfaces_required_capability_verdict_fields() {
7300 let registry = Arc::new(Registry::default());
7301 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
7302 [
7303 ("consumer".to_string(), true),
7304 ("provider".to_string(), false),
7305 ],
7306 BTreeMap::new(),
7307 );
7308 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
7309 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
7310 provides: Vec::new(),
7311 requires: vec![subc_protocol::manifest::CapabilityRequirement {
7312 capability: "credentials-provider/v1".to_string(),
7313 need: subc_protocol::manifest::CapabilityNeed::Required,
7314 }],
7315 must_never_reach: Vec::new(),
7316 });
7317 let hello = Frame::build(
7318 FrameType::Hello,
7319 control_flags(),
7320 0,
7321 0,
7322 78,
7323 serde_json::to_vec(&ModuleHelloBody {
7324 manifest: consumer,
7325 protocol_ver: PROTOCOL_VERSION,
7326 control_ops: None,
7327 launch_nonce: None,
7328 })
7329 .expect("HELLO serializes"),
7330 )
7331 .expect("HELLO frame builds");
7332 handler
7333 .handle_control(ConnectionId::new(78), hello)
7334 .expect("consumer registers");
7335 let describe = Frame::build(
7336 FrameType::Request,
7337 control_flags(),
7338 0,
7339 0,
7340 79,
7341 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
7342 .expect("request serializes"),
7343 )
7344 .expect("describe frame builds");
7345 let response = handler
7346 .handle_server_describe(describe)
7347 .expect("server.describe succeeds");
7348 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
7349 let requirement = &rendered["capability_requirements"][0];
7350 assert_eq!(requirement["consumer"], "consumer");
7351 assert_eq!(requirement["verdict"], "never_provided");
7352 assert_eq!(requirement["episode_seq"], 1);
7353 assert_eq!(requirement["config_satisfiable"], false);
7354 assert_eq!(requirement["runtime_available"], false);
7355 assert!(requirement["detail"]
7356 .as_str()
7357 .expect("detail string")
7358 .contains("credentials-provider/v1"));
7359 }
7360
7361 #[test]
7362 fn catalog_list_omits_capabilities_for_legacy_manifest() {
7363 let registry = Arc::new(Registry::default());
7364 let handler = ControlHandler::new(Arc::clone(®istry));
7365 let hello = handler
7366 .handle_control(
7367 ConnectionId::new(101),
7368 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
7369 )
7370 .expect("legacy HELLO registers");
7371 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
7372
7373 let request = Frame::build(
7374 FrameType::Request,
7375 control_flags(),
7376 0,
7377 0,
7378 102,
7379 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7380 .expect("catalog request serializes"),
7381 )
7382 .expect("catalog request frame builds");
7383 let response = handler
7384 .handle_catalog_list(request, None)
7385 .expect("catalog list succeeds");
7386 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
7387 assert!(
7388 body["modules"][0].get("capabilities").is_none(),
7389 "legacy manifest must retain an absent capabilities field on catalog.list"
7390 );
7391 }
7392
7393 #[test]
7394 fn hello_ack_omits_storage_when_no_storage_config() {
7395 let registry = Arc::new(Registry::default());
7396 let handler = ControlHandler::new(Arc::clone(®istry));
7397 let responses = handler
7398 .handle_control(
7399 ConnectionId::new(1),
7400 hello_frame("aft", PROTOCOL_VERSION, 7),
7401 )
7402 .unwrap();
7403 let ack = parse_ack(&responses[0]);
7404 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
7405 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
7406 }
7407
7408 #[tokio::test]
7409 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
7410 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
7411 let registry = Arc::new(Registry::default());
7412 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
7413 let responses = handler
7414 .handle_control(
7415 ConnectionId::new(1),
7416 hello_frame("aft", PROTOCOL_VERSION, 7),
7417 )
7418 .unwrap();
7419 let ack = parse_ack(&responses[0]);
7420 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
7421
7422 let described = handler
7423 .handle_control_frame(
7424 &route_ctx(ConnectionId::new(2)).0,
7425 Frame::build(
7426 FrameType::Request,
7427 control_flags(),
7428 0,
7429 0,
7430 9,
7431 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
7432 )
7433 .unwrap(),
7434 )
7435 .await
7436 .unwrap();
7437 let ClientControlResponse::ServerDescribe { machine_id, .. } =
7438 serde_json::from_slice(&described[0].body).unwrap()
7439 else {
7440 panic!("server.describe answered with another shape");
7441 };
7442 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
7443 }
7444
7445 #[test]
7446 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
7447 let registry = Arc::new(Registry::default());
7450 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
7451 crate::daemon_config::StorageConfig::Sqlite {
7452 data_home: std::path::PathBuf::from("/data"),
7453 },
7454 ));
7455
7456 let responses = handler
7457 .handle_control(
7458 ConnectionId::new(1),
7459 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
7460 )
7461 .unwrap();
7462 let ack = parse_ack(&responses[0]);
7463 assert_eq!(
7464 ack.storage,
7465 Some(serde_json::json!({
7466 "module_id": "alfonso-routing",
7467 "storage_namespace": "default",
7468 "isolation": { "kind": "module" },
7469 "backend": {
7470 "backend": "sqlite",
7471 "path": "/data/cortexkit/alfonso-routing/store.db"
7472 }
7473 })),
7474 "the delivered descriptor is the module's own sqlite store path"
7475 );
7476 }
7477
7478 #[test]
7479 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
7480 let registry = Arc::new(Registry::default());
7481 let handler = ControlHandler::new(Arc::clone(®istry));
7482 let conn = ConnectionId::new(1);
7483 let responses = handler
7484 .handle_control(
7485 conn,
7486 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
7487 )
7488 .unwrap();
7489 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
7490 let registration = registry.get_module("aft").unwrap().unwrap();
7491 assert_eq!(registration.control_ops, module_baseline_control_ops());
7492
7493 let frame =
7494 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
7495 assert!(handler
7496 .guard_module_control_op(&frame, "aft", "route.bind")
7497 .unwrap()
7498 .is_none());
7499 let error = handler
7500 .guard_module_control_op(&frame, "aft", "test.synthetic")
7501 .unwrap()
7502 .expect("synthetic ungranted op should be rejected");
7503 assert_eq!(error.header.ty, FrameType::Error);
7504 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
7505 }
7506
7507 #[test]
7508 fn hello_control_ops_some_adds_optional_grants() {
7509 let registry = Arc::new(Registry::default());
7510 let handler = ControlHandler::new(Arc::clone(®istry));
7511 handler
7512 .handle_control(
7513 ConnectionId::new(1),
7514 hello_frame_with_control_ops(
7515 "aft",
7516 PROTOCOL_VERSION,
7517 7,
7518 Some(vec![
7519 "future.synthetic".to_string(),
7520 "route.bind".to_string(),
7521 ]),
7522 ),
7523 )
7524 .unwrap();
7525 let registration = registry.get_module("aft").unwrap().unwrap();
7526 assert_eq!(
7527 registration.control_ops,
7528 vec![
7529 "route.bind".to_string(),
7530 "route.status".to_string(),
7531 "future.synthetic".to_string(),
7532 ]
7533 );
7534 let frame =
7535 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
7536 assert!(handler
7537 .guard_module_control_op(&frame, "aft", "future.synthetic")
7538 .unwrap()
7539 .is_none());
7540 }
7541
7542 #[tokio::test]
7543 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
7544 let registry = Arc::new(Registry::default());
7545 let forwarding = Arc::new(ForwardingTable::default());
7546 let handler =
7547 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
7548 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
7549 hello_via_sink(
7550 &handler,
7551 &module_ctx,
7552 &mut module_rx,
7553 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
7554 )
7555 .await;
7556
7557 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
7558 let responses = handler
7559 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
7560 .await
7561 .unwrap();
7562 assert_eq!(responses.len(), 1);
7563 assert_eq!(responses[0].header.ty, FrameType::Error);
7564 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
7565 assert!(module_rx.try_recv().is_err());
7566 }
7567
7568 #[tokio::test]
7569 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
7570 let registry = Arc::new(Registry::default());
7571 let forwarding = Arc::new(ForwardingTable::default());
7572 let handler =
7573 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
7574 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
7575 hello_via_sink(
7576 &handler,
7577 &module_ctx,
7578 &mut module_rx,
7579 hello_frame_with_control_ops(
7580 "aft",
7581 PROTOCOL_VERSION,
7582 7,
7583 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
7584 ),
7585 )
7586 .await;
7587
7588 let project_root = unique_project_root("demux");
7589 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
7590 let route_handler = handler.clone();
7591 let route_task = tokio::spawn(async move {
7592 route_handler
7593 .handle_control_frame(
7594 &route_client_ctx,
7595 route_open_frame(100, "aft", project_root),
7596 )
7597 .await
7598 .unwrap()
7599 });
7600 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7601 .await
7602 .unwrap()
7603 .unwrap();
7604 assert!(matches!(
7605 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
7606 ModuleControlRequest::RouteBind { .. }
7607 ));
7608
7609 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
7610 let health_handler = handler.clone();
7611 let health_task = tokio::spawn(async move {
7612 health_handler
7613 .handle_control_frame(
7614 &health_client_ctx,
7615 supervisor_health_probe_frame(101, "aft"),
7616 )
7617 .await
7618 .unwrap()
7619 });
7620 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7621 .await
7622 .unwrap()
7623 .unwrap();
7624 assert_eq!(
7625 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
7626 ModuleControlRequest::HealthCheck {}
7627 );
7628
7629 handler
7630 .handle_control_frame(
7631 &module_ctx,
7632 health_response(health_frame.header.corr, HealthStatus::Degraded),
7633 )
7634 .await
7635 .unwrap();
7636 let health_response = health_task.await.unwrap();
7637 assert_eq!(health_response.len(), 1);
7638 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
7639 ClientControlResponse::SupervisorHealthProbe {
7640 module_id,
7641 status,
7642 detail,
7643 metrics,
7644 } => {
7645 assert_eq!(module_id, "aft");
7646 assert_eq!(status, HealthStatus::Degraded);
7647 assert_eq!(detail.as_deref(), Some("warming"));
7648 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
7649 }
7650 other => panic!("unexpected health response: {other:?}"),
7651 }
7652
7653 handler
7654 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
7655 .await
7656 .unwrap();
7657 let route_response = route_task.await.unwrap();
7658 assert!(route_response.is_empty());
7659 let published = route_client_rx.recv().await.unwrap();
7660 assert!(matches!(
7661 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
7662 ClientControlResponse::RouteOpen { .. }
7663 ));
7664 }
7665
7666 async fn relay_route_open(
7671 handler: &ControlHandler,
7672 client_connection: ConnectionId,
7673 client_egress: &FrameSink,
7674 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
7675 corr: u64,
7676 module_id: &str,
7677 project_root_label: &str,
7678 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
7679 let ctx = RouteCtx {
7680 connection_id: client_connection,
7681 egress: client_egress.clone(),
7682 };
7683 let handler = handler.clone();
7684 let project_root = unique_project_root(project_root_label);
7685 let module_id = module_id.to_string();
7686 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
7687 let task = tokio::spawn(async move {
7688 let _guard = tracing::dispatcher::set_default(&dispatch);
7689 handler
7690 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
7691 .await
7692 .unwrap()
7693 });
7694 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
7695 .await
7696 .expect("module receives the relayed route.bind")
7697 .expect("module egress is open");
7698 (task, bind.frame)
7699 }
7700
7701 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
7702 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
7703 ModuleControlRequest::RouteBind {
7704 route_channel,
7705 epoch,
7706 ..
7707 } => (route_channel, epoch),
7708 other => panic!("expected a route.bind request, got {other:?}"),
7709 }
7710 }
7711
7712 fn published_route(frame: &Frame) -> (u16, u32) {
7713 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
7714 ClientControlResponse::RouteOpen {
7715 route_channel,
7716 route_epoch,
7717 } => (route_channel, route_epoch),
7718 other => panic!("expected a route.open response, got {other:?}"),
7719 }
7720 }
7721
7722 #[tokio::test]
7742 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
7743 let registry = Arc::new(Registry::default());
7744 let forwarding = Arc::new(ForwardingTable::default());
7745 let handler =
7746 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
7747
7748 let module_connection = ConnectionId::new(30);
7749 let (module_ctx, mut module_rx) = route_ctx(module_connection);
7750 hello_via_sink(
7751 &handler,
7752 &module_ctx,
7753 &mut module_rx,
7754 hello_frame("aft", PROTOCOL_VERSION, 7),
7755 )
7756 .await;
7757
7758 let dying_client = ConnectionId::new(31);
7759 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
7760
7761 let (first_task, first_bind) = relay_route_open(
7764 &handler,
7765 dying_client,
7766 &dying_ctx.egress,
7767 &mut module_rx,
7768 100,
7769 "aft",
7770 "closing-first",
7771 )
7772 .await;
7773 handler
7774 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
7775 .await
7776 .unwrap();
7777 assert!(first_task.await.unwrap().is_empty());
7778 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
7779
7780 let (second_task, second_bind) = relay_route_open(
7782 &handler,
7783 dying_client,
7784 &dying_ctx.egress,
7785 &mut module_rx,
7786 101,
7787 "aft",
7788 "closing-second",
7789 )
7790 .await;
7791 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
7792
7793 assert!(forwarding
7796 .escalate_client_delivery_failure(
7797 dying_client,
7798 first_channel,
7799 first_epoch,
7800 CloseReason::new(
7801 "module_to_client_delivery_failed",
7802 "client egress refused a module frame",
7803 ),
7804 crate::forwarding::UndeliveredFrame {
7805 module_id: None,
7806 sink: &dying_ctx.egress,
7807 },
7808 )
7809 .unwrap());
7810 assert!(!dying_ctx.egress.is_closed());
7811
7812 let ack = handler
7814 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
7815 .await;
7816 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
7817 if module_loop_error.is_some() {
7818 handler.cleanup_connection(module_connection).unwrap();
7822 }
7823 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7827 .await
7828 .ok()
7829 .flatten();
7830
7831 assert!(
7833 registry
7834 .get_module_by_connection(module_connection)
7835 .unwrap()
7836 .is_some(),
7837 "one client's closing connection ended the shared module connection: \
7838 {module_loop_error:?}"
7839 );
7840 let cotenant = ConnectionId::new(32);
7842 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
7843 let (cotenant_task, cotenant_bind) = relay_route_open(
7844 &handler,
7845 cotenant,
7846 &cotenant_ctx.egress,
7847 &mut module_rx,
7848 102,
7849 "aft",
7850 "closing-cotenant",
7851 )
7852 .await;
7853 handler
7854 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
7855 .await
7856 .unwrap();
7857 assert!(cotenant_task.await.unwrap().is_empty());
7858 let (cotenant_channel, cotenant_epoch) =
7859 published_route(&cotenant_rx.recv().await.unwrap());
7860 assert!(matches!(
7861 forwarding
7862 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
7863 .unwrap(),
7864 DataRoute::Client(DataRouteState::Bound(_))
7865 ));
7866
7867 let goodbye = post_ack_module_frame
7870 .expect("module receives a GOODBYE for the abandoned route channel");
7871 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
7872 assert_eq!(goodbye.header.channel, abandoned_channel);
7873 assert_eq!(goodbye.header.epoch, abandoned_epoch);
7874
7875 assert!(dying_rx.try_recv().is_err());
7879 let second_response = second_task.await.unwrap();
7880 assert_eq!(second_response.len(), 1);
7881 assert_eq!(
7882 parse_error(&second_response[0])["code"],
7883 "target_unavailable"
7884 );
7885 }
7886
7887 #[test]
7894 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
7895 let handler = ControlHandler::default();
7896 let module_connection = ConnectionId::new(30);
7897 let client_connection = ConnectionId::new(31);
7898
7899 handler
7900 .refuse_to_end_module_connection_for_a_client(
7901 module_connection,
7902 77,
7903 ForwardingError::ConnectionClosing {
7904 connection_id: client_connection,
7905 },
7906 )
7907 .expect("a closing client must never end the module connection");
7908
7909 assert!(matches!(
7910 handler.refuse_to_end_module_connection_for_a_client(
7911 module_connection,
7912 78,
7913 ForwardingError::ConnectionClosing {
7914 connection_id: module_connection,
7915 },
7916 ),
7917 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
7918 connection_id
7919 })) if connection_id == module_connection
7920 ));
7921 assert!(matches!(
7922 handler.refuse_to_end_module_connection_for_a_client(
7923 module_connection,
7924 79,
7925 ForwardingError::Poisoned,
7926 ),
7927 Err(RouterError::Forwarding(ForwardingError::Poisoned))
7928 ));
7929 assert!(matches!(
7930 handler.refuse_to_end_module_connection_for_a_client(
7931 module_connection,
7932 80,
7933 ForwardingError::StaleModuleEndpoint,
7934 ),
7935 Err(RouterError::Forwarding(
7936 ForwardingError::StaleModuleEndpoint
7937 ))
7938 ));
7939 }
7940
7941 #[tokio::test]
7954 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
7955 let registry = Arc::new(Registry::default());
7956 let forwarding = Arc::new(ForwardingTable::default());
7957 let supervisor = SupervisorHandle::new();
7958 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7959 let handler =
7960 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
7961 .with_supervisor(supervisor);
7962
7963 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
7964 hello_via_sink(
7965 &handler,
7966 &target_ctx,
7967 &mut target_rx,
7968 hello_frame("target", PROTOCOL_VERSION, 1),
7969 )
7970 .await;
7971
7972 let wrong_nonce = handler
7976 .handle_control_frame(
7977 &route_ctx(ConnectionId::new(91)).0,
7978 route_open_frame_with_admission_facts(
7979 20,
7980 "target",
7981 unique_project_root("admission-facts"),
7982 Some(subc_control::ConsumerIdentity {
7983 module_id: "fed".to_string(),
7984 launch_nonce: "not-the-real-nonce".to_string(),
7985 }),
7986 None,
7987 ),
7988 )
7989 .await
7990 .unwrap();
7991 assert_eq!(
7992 parse_error(&wrong_nonce[0])["code"],
7993 "bad_consumer_identity",
7994 "a mismatched launch nonce must be refused, not stamped Reserved"
7995 );
7996
7997 let never_spawned = handler
8001 .handle_control_frame(
8002 &route_ctx(ConnectionId::new(92)).0,
8003 route_open_frame_with_admission_facts(
8004 21,
8005 "target",
8006 unique_project_root("admission-facts"),
8007 Some(subc_control::ConsumerIdentity {
8008 module_id: "never-spawned".to_string(),
8009 launch_nonce: "any-nonce".to_string(),
8010 }),
8011 None,
8012 ),
8013 )
8014 .await
8015 .unwrap();
8016 assert_eq!(
8017 parse_error(&never_spawned[0])["code"],
8018 "bad_consumer_identity",
8019 "an unspawned module_id must be refused rather than accepted for lack of a record"
8020 );
8021 }
8022
8023 #[tokio::test]
8046 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
8047 let registry = Arc::new(Registry::default());
8048 let forwarding = Arc::new(ForwardingTable::default());
8049 let supervisor = SupervisorHandle::new();
8050 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8051 let handler =
8052 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8053 .with_supervisor(supervisor);
8054
8055 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
8056 hello_via_sink(
8057 &handler,
8058 &target_ctx,
8059 &mut target_rx,
8060 hello_frame("target", PROTOCOL_VERSION, 1),
8061 )
8062 .await;
8063
8064 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
8065 let route_handler = handler.clone();
8066 let route_task = tokio::spawn(async move {
8067 route_handler
8068 .handle_control_frame(
8069 &client_ctx,
8070 route_open_frame_with_admission_facts(
8071 30,
8072 "target",
8073 unique_project_root("admission-facts"),
8074 Some(subc_control::ConsumerIdentity {
8075 module_id: "fed".to_string(),
8076 launch_nonce: "fed-nonce".to_string(),
8077 }),
8078 None,
8079 ),
8080 )
8081 .await
8082 .unwrap()
8083 });
8084
8085 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8092 .await
8093 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
8094 .expect("module control channel closed before route.bind");
8095 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8096 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
8097 panic!("expected route.bind")
8098 };
8099 assert_eq!(
8100 principal,
8101 Some(Principal::Reserved {
8102 module_id: "fed".to_string()
8103 }),
8104 "a correctly attested consumer must be stamped Reserved for its own id"
8105 );
8106
8107 handler
8108 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8109 .await
8110 .unwrap();
8111 assert!(route_task.await.unwrap().is_empty());
8112 assert!(
8113 matches!(
8114 serde_json::from_slice::<ClientControlResponse>(
8115 &client_rx.recv().await.unwrap().body
8116 )
8117 .unwrap(),
8118 ClientControlResponse::RouteOpen { .. }
8119 ),
8120 "the route must actually open, not merely avoid an error"
8121 );
8122 }
8123
8124 #[tokio::test(start_paused = true)]
8125 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
8126 let registry = Arc::new(Registry::default());
8127 let forwarding = Arc::new(ForwardingTable::default());
8128 let supervisor = SupervisorHandle::new();
8129 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8130 let handler =
8131 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8132 .with_supervisor(supervisor);
8133
8134 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
8135 hello_via_sink(
8136 &handler,
8137 &target_ctx,
8138 &mut target_rx,
8139 hello_frame("target", PROTOCOL_VERSION, 1),
8140 )
8141 .await;
8142
8143 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
8144 let direct_handler = handler.clone();
8145 let direct_open = tokio::spawn(async move {
8146 direct_handler
8147 .handle_control_frame(
8148 &direct_ctx,
8149 route_open_frame(2, "target", unique_project_root("route-census-direct")),
8150 )
8151 .await
8152 .unwrap()
8153 });
8154 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8155 .await
8156 .expect("no direct route.bind within 5s")
8157 .expect("target control channel closed before direct route.bind");
8158 handler
8159 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
8160 .await
8161 .unwrap();
8162 assert!(direct_open.await.unwrap().is_empty());
8163 let _ = direct_rx.recv().await.unwrap();
8164
8165 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
8166 let reserved_handler = handler.clone();
8167 let reserved_open = tokio::spawn(async move {
8168 reserved_handler
8169 .handle_control_frame(
8170 &reserved_ctx,
8171 route_open_frame_with_admission_facts(
8172 3,
8173 "target",
8174 unique_project_root("admission-facts"),
8175 Some(ConsumerIdentity {
8176 module_id: "fed".to_string(),
8177 launch_nonce: "fed-nonce".to_string(),
8178 }),
8179 None,
8180 ),
8181 )
8182 .await
8183 .unwrap()
8184 });
8185 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8186 .await
8187 .expect("no reserved route.bind within 5s")
8188 .expect("target control channel closed before reserved route.bind");
8189 handler
8190 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
8191 .await
8192 .unwrap();
8193 assert!(reserved_open.await.unwrap().is_empty());
8194 let _ = reserved_rx.recv().await.unwrap();
8195
8196 forwarding
8197 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
8198 .unwrap();
8199 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
8200 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
8201 module_id: Some("target".to_string()),
8202 })
8203 .unwrap();
8204 let census_frame =
8205 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
8206 let response = handler
8207 .handle_control_frame(&census_ctx, census_frame)
8208 .await
8209 .unwrap()
8210 .pop()
8211 .unwrap();
8212 let actual: Value = serde_json::from_slice(&response.body).unwrap();
8213 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
8214 assert!(matches!(
8215 decoded,
8216 ClientControlResponse::SupervisorRoutes { .. }
8217 ));
8218 let routes = actual["modules"][0]["routes"].as_array().unwrap();
8219 assert_eq!(routes.len(), 2);
8220 assert!(routes.iter().all(|route| route["draining"] == true));
8221 assert!(
8224 routes.iter().all(|route| route["drain_reason"] == "reload"),
8225 "draining routes must name the drain's reason: {routes:?}"
8226 );
8227 assert!(routes.iter().any(|route| {
8228 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
8229 }));
8230 assert!(routes.iter().any(|route| {
8231 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
8232 }));
8233
8234 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8235 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
8236 if std::env::var_os("UPDATE_GOLDEN").is_some() {
8237 std::fs::write(
8238 &golden_path,
8239 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
8240 )
8241 .unwrap();
8242 }
8243 let expected: Value =
8244 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
8245 assert_eq!(actual, expected);
8246 }
8247
8248 async fn query_live_roots(
8249 handler: &ControlHandler,
8250 module_ctx: &RouteCtx,
8251 ) -> ModuleControlResponseToModule {
8252 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
8253 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
8254 let response = handler
8255 .handle_control_frame(module_ctx, frame)
8256 .await
8257 .unwrap()
8258 .pop()
8259 .unwrap();
8260 serde_json::from_slice(&response.body).unwrap()
8261 }
8262
8263 #[tokio::test(start_paused = true)]
8264 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
8265 let registry = Arc::new(Registry::default());
8266 let forwarding = Arc::new(ForwardingTable::default());
8267 let handler = ControlHandler::with_forwarding(registry, forwarding);
8268 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
8269 hello_via_sink(
8270 &handler,
8271 &target_ctx,
8272 &mut target_rx,
8273 hello_frame("target", PROTOCOL_VERSION, 1),
8274 )
8275 .await;
8276 let root = unique_project_root("live-roots-known");
8277 let path = ProjectRootId::from_path_allowing_missing(root.path())
8278 .unwrap()
8279 .as_path()
8280 .to_path_buf();
8281 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
8282 let open_handler = handler.clone();
8283 let opened = tokio::spawn(async move {
8284 open_handler
8285 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
8286 .await
8287 .unwrap()
8288 });
8289 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8290 .await
8291 .unwrap()
8292 .unwrap();
8293 handler
8294 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
8295 .await
8296 .unwrap();
8297 assert!(opened.await.unwrap().is_empty());
8298 let _ = client_rx.recv().await.unwrap();
8299
8300 let root = unique_project_root("live-roots-pending");
8301 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
8302 .unwrap()
8303 .as_path()
8304 .to_path_buf();
8305 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
8306 let open_handler = handler.clone();
8307 let pending = tokio::spawn(async move {
8308 open_handler
8309 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
8310 .await
8311 .unwrap()
8312 });
8313 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8314 .await
8315 .unwrap()
8316 .unwrap();
8317 let actual = query_live_roots(&handler, &target_ctx).await;
8318 let ModuleControlResponseToModule::LiveRoots {
8319 roots,
8320 unknown_root_bindings,
8321 total_bindings,
8322 } = actual
8323 else {
8324 panic!("expected live roots")
8325 };
8326 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
8327 assert_eq!(unknown_root_bindings, 0);
8328 assert_eq!(
8329 roots.len(),
8330 2,
8331 "root-known arm must retain each canonical root"
8332 );
8333 assert_eq!(
8334 total_bindings,
8335 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
8336 );
8337 let counts = roots
8338 .iter()
8339 .map(|root| (root.project_root.clone(), root.bound, root.pending))
8340 .collect::<Vec<_>>();
8341 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
8342 expected.sort_by(|a, b| a.0.cmp(&b.0));
8343 assert_eq!(
8344 counts, expected,
8345 "roots must sort by path and count pending separately"
8346 );
8347 handler
8348 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
8349 .await
8350 .unwrap();
8351 assert!(pending.await.unwrap().is_empty());
8352 }
8353
8354 #[tokio::test(start_paused = true)]
8355 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
8356 let forwarding = Arc::new(ForwardingTable::default());
8357 let handler =
8358 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
8359 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
8360 hello_via_sink(
8361 &handler,
8362 &target_ctx,
8363 &mut target_rx,
8364 hello_frame("target", PROTOCOL_VERSION, 1),
8365 )
8366 .await;
8367 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
8368 let pending = forwarding
8369 .begin_route_bind_relay_for_test(
8370 client_ctx.connection_id,
8371 client_ctx.egress.clone(),
8372 2,
8373 "target",
8374 )
8375 .unwrap();
8376 forwarding
8377 .complete_pending_relay(
8378 target_ctx.connection_id,
8379 pending.corr,
8380 RouteBindRelayOutcome::Accepted,
8381 )
8382 .unwrap();
8383 let actual = query_live_roots(&handler, &target_ctx).await;
8384 let ModuleControlResponseToModule::LiveRoots {
8385 roots,
8386 unknown_root_bindings,
8387 total_bindings,
8388 } = actual
8389 else {
8390 panic!("expected live roots")
8391 };
8392 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
8393 assert_eq!(
8394 unknown_root_bindings, 1,
8395 "unknown-root arm must not read as no bindings"
8396 );
8397 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
8398 assert_eq!(
8399 total_bindings,
8400 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
8401 );
8402 }
8403
8404 #[tokio::test(start_paused = true)]
8412 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
8413 let forwarding = Arc::new(ForwardingTable::default());
8414 let handler =
8415 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
8416 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
8417 let replies = handler
8418 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
8419 .await
8420 .unwrap();
8421 let queued_by_hello = module_rx.len();
8422
8423 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
8424 let open_handler = handler.clone();
8425 let open = tokio::spawn(async move {
8426 open_handler
8427 .handle_control_frame(
8428 &client_ctx,
8429 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
8430 )
8431 .await
8432 .unwrap()
8433 });
8434 let mut spins = 0;
8436 while module_rx.len() == queued_by_hello {
8437 spins += 1;
8438 assert!(spins < 10_000, "route.open never queued a route.bind");
8439 tokio::task::yield_now().await;
8440 }
8441
8442 for reply in replies {
8444 module_ctx.egress.send(reply).await.unwrap();
8445 }
8446
8447 let first = module_rx.recv().await.unwrap().frame;
8448 assert_eq!(
8449 first.header.ty,
8450 FrameType::HelloAck,
8451 "the first frame a registering module reads must be its HELLO_ACK"
8452 );
8453 assert_eq!(first.header.corr, 7);
8454 let second = module_rx.recv().await.unwrap().frame;
8455 assert_eq!(second.header.ty, FrameType::Request);
8456 assert!(
8457 matches!(
8458 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
8459 ModuleControlRequest::RouteBind { .. }
8460 ),
8461 "the route.bind follows the ack"
8462 );
8463 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
8464
8465 handler
8466 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
8467 .await
8468 .unwrap();
8469 assert!(open.await.unwrap().is_empty());
8470 let _ = client_rx.recv().await.unwrap();
8471 }
8472
8473 #[tokio::test(start_paused = true)]
8474 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
8475 let handler = ControlHandler::with_forwarding(
8476 Arc::new(Registry::default()),
8477 Arc::new(ForwardingTable::default()),
8478 );
8479 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
8480 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
8481 hello_via_sink(
8482 &handler,
8483 &first_ctx,
8484 &mut first_rx,
8485 hello_frame("first", PROTOCOL_VERSION, 1),
8486 )
8487 .await;
8488 hello_via_sink(
8489 &handler,
8490 &second_ctx,
8491 &mut second_rx,
8492 hello_frame("second", PROTOCOL_VERSION, 2),
8493 )
8494 .await;
8495 let root = unique_project_root("second-only");
8496 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
8497 let cloned = handler.clone();
8498 let open = tokio::spawn(async move {
8499 cloned
8500 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
8501 .await
8502 .unwrap()
8503 });
8504 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
8505 .await
8506 .unwrap()
8507 .unwrap();
8508 let first = query_live_roots(&handler, &first_ctx).await;
8509 let second = query_live_roots(&handler, &second_ctx).await;
8510 assert!(
8511 matches!(
8512 first,
8513 ModuleControlResponseToModule::LiveRoots {
8514 total_bindings: 0,
8515 ..
8516 }
8517 ),
8518 "cross-module scope must not expose another module's roots"
8519 );
8520 assert!(
8521 matches!(
8522 second,
8523 ModuleControlResponseToModule::LiveRoots {
8524 total_bindings: 1,
8525 ..
8526 }
8527 ),
8528 "second module must see its pending route"
8529 );
8530 handler
8531 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
8532 .await
8533 .unwrap();
8534 assert!(open.await.unwrap().is_empty());
8535 }
8536
8537 #[tokio::test(start_paused = true)]
8538 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
8539 let handler = ControlHandler::with_forwarding(
8540 Arc::new(Registry::default()),
8541 Arc::new(ForwardingTable::default()),
8542 );
8543 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
8544 hello_via_sink(
8545 &handler,
8546 &target_ctx,
8547 &mut target_rx,
8548 hello_frame("target", PROTOCOL_VERSION, 1),
8549 )
8550 .await;
8551 let actual = query_live_roots(&handler, &target_ctx).await;
8552 let ModuleControlResponseToModule::LiveRoots {
8553 roots,
8554 unknown_root_bindings,
8555 total_bindings,
8556 } = actual
8557 else {
8558 panic!("expected live roots")
8559 };
8560 assert!(roots.is_empty());
8561 assert_eq!(unknown_root_bindings, 0);
8562 assert_eq!(total_bindings, 0);
8563 assert_eq!(
8564 total_bindings,
8565 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
8566 );
8567 }
8568
8569 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
8576 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8577 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
8578 let text = std::fs::read_to_string(&path)
8579 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
8580 let vectors: Vec<(String, Value)> = text
8581 .lines()
8582 .filter(|line| !line.trim().is_empty())
8583 .map(|line| {
8584 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
8585 let id = entry["corpus_id"]
8586 .as_str()
8587 .expect("every vector carries a corpus_id")
8588 .to_string();
8589 (id, entry["package"].clone())
8590 })
8591 .collect();
8592 assert_eq!(
8596 vectors.len(),
8597 3,
8598 "vendored fed corpus changed size; re-sync from subc-federation"
8599 );
8600
8601 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
8616 let richest = vectors
8617 .iter()
8618 .filter_map(|(_, package)| package.as_object())
8619 .map(|object| {
8620 object
8621 .keys()
8622 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
8623 .count()
8624 })
8625 .max()
8626 .unwrap_or(0);
8627 assert!(
8628 richest >= 2,
8629 "vendored corpus no longer carries a package with unmodelled fields, \
8630 so the relay test can no longer distinguish a verbatim relay from a lossy one"
8631 );
8632
8633 vectors
8634 }
8635
8636 #[tokio::test]
8644 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
8645 for (corpus_id, package) in fed_admission_facts_vectors() {
8646 let registry = Arc::new(Registry::default());
8647 let forwarding = Arc::new(ForwardingTable::default());
8648 let supervisor = SupervisorHandle::new();
8649 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8650 let handler =
8651 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8652 .with_supervisor(supervisor)
8653 .with_admission_facts_config(
8654 Some("fed".to_string()),
8655 Some(vec!["target".to_string()]),
8656 );
8657
8658 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
8659 hello_via_sink(
8660 &handler,
8661 &target_ctx,
8662 &mut target_rx,
8663 hello_frame("target", PROTOCOL_VERSION, 1),
8664 )
8665 .await;
8666
8667 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
8668 let route_handler = handler.clone();
8669 let expected = package.clone();
8670 let route_task = tokio::spawn(async move {
8671 route_handler
8672 .handle_control_frame(
8673 &client_ctx,
8674 route_open_frame_with_admission_facts(
8675 20,
8676 "target",
8677 unique_project_root("admission-facts"),
8678 Some(subc_control::ConsumerIdentity {
8679 module_id: "fed".to_string(),
8680 launch_nonce: "fed-nonce".to_string(),
8681 }),
8682 Some(package),
8683 ),
8684 )
8685 .await
8686 .unwrap()
8687 });
8688
8689 let bind_frame = target_rx.recv().await.unwrap();
8690 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8691 let ModuleControlRequest::RouteBind {
8692 admission_facts, ..
8693 } = bind
8694 else {
8695 panic!("{corpus_id}: expected route.bind")
8696 };
8697 assert_eq!(
8698 admission_facts,
8699 Some(expected),
8700 "{corpus_id}: relay must not add, drop or reshape any field"
8701 );
8702
8703 handler
8704 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8705 .await
8706 .unwrap();
8707 route_task.await.unwrap();
8708 }
8709 }
8710
8711 #[tokio::test]
8712 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
8713 let registry = Arc::new(Registry::default());
8714 let forwarding = Arc::new(ForwardingTable::default());
8715 let supervisor = SupervisorHandle::new();
8716 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8717 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
8718 let handler =
8719 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8720 .with_supervisor(supervisor)
8721 .with_admission_facts_config(
8722 Some("fed".to_string()),
8723 Some(vec!["target".to_string()]),
8724 );
8725
8726 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
8727 hello_via_sink(
8728 &handler,
8729 &target_ctx,
8730 &mut target_rx,
8731 hello_frame("target", PROTOCOL_VERSION, 1),
8732 )
8733 .await;
8734 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
8735 hello_via_sink(
8736 &handler,
8737 &other_ctx,
8738 &mut other_rx,
8739 hello_frame("other", PROTOCOL_VERSION, 2),
8740 )
8741 .await;
8742
8743 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
8744 let expected_facts = facts.clone();
8745 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
8746 let route_handler = handler.clone();
8747 let route_task = tokio::spawn(async move {
8748 route_handler
8749 .handle_control_frame(
8750 &client_ctx,
8751 route_open_frame_with_admission_facts(
8752 10,
8753 "target",
8754 unique_project_root("admission-facts"),
8755 Some(subc_control::ConsumerIdentity {
8756 module_id: "fed".to_string(),
8757 launch_nonce: "fed-nonce".to_string(),
8758 }),
8759 Some(facts.clone()),
8760 ),
8761 )
8762 .await
8763 .unwrap()
8764 });
8765 let bind_frame = target_rx.recv().await.unwrap();
8766 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8767 let ModuleControlRequest::RouteBind {
8768 admission_facts, ..
8769 } = bind
8770 else {
8771 panic!("expected route.bind")
8772 };
8773 assert_eq!(admission_facts, Some(expected_facts));
8774 handler
8775 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8776 .await
8777 .unwrap();
8778 assert!(route_task.await.unwrap().is_empty());
8779 assert!(matches!(
8780 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
8781 .unwrap(),
8782 ClientControlResponse::RouteOpen { .. }
8783 ));
8784
8785 let direct = handler
8786 .handle_control_frame(
8787 &route_ctx(ConnectionId::new(73)).0,
8788 route_open_frame_with_admission_facts(
8789 11,
8790 "target",
8791 unique_project_root("admission-facts"),
8792 None,
8793 Some(json!({"x": 1})),
8794 ),
8795 )
8796 .await
8797 .unwrap();
8798 assert_eq!(
8799 parse_error(&direct[0])["code"],
8800 "admission_facts_not_permitted"
8801 );
8802
8803 let different_reserved = handler
8804 .handle_control_frame(
8805 &route_ctx(ConnectionId::new(77)).0,
8806 route_open_frame_with_admission_facts(
8807 15,
8808 "target",
8809 unique_project_root("admission-facts"),
8810 Some(subc_control::ConsumerIdentity {
8811 module_id: "other".to_string(),
8812 launch_nonce: "other-nonce".to_string(),
8813 }),
8814 Some(json!({"x": 1})),
8815 ),
8816 )
8817 .await
8818 .unwrap();
8819 assert_eq!(
8820 parse_error(&different_reserved[0])["code"],
8821 "admission_facts_not_permitted"
8822 );
8823
8824 let other_target = handler
8825 .handle_control_frame(
8826 &route_ctx(ConnectionId::new(74)).0,
8827 route_open_frame_with_admission_facts(
8828 12,
8829 "other",
8830 unique_project_root("admission-facts"),
8831 Some(subc_control::ConsumerIdentity {
8832 module_id: "fed".to_string(),
8833 launch_nonce: "fed-nonce".to_string(),
8834 }),
8835 Some(json!({"x": 1})),
8836 ),
8837 )
8838 .await
8839 .unwrap();
8840 assert_eq!(
8841 parse_error(&other_target[0])["code"],
8842 "admission_facts_target_not_allowed"
8843 );
8844
8845 let nonexistent = handler
8846 .handle_control_frame(
8847 &route_ctx(ConnectionId::new(75)).0,
8848 route_open_frame_with_admission_facts(
8849 13,
8850 "missing",
8851 unique_project_root("admission-facts"),
8852 None,
8853 Some(json!({"x": 1})),
8854 ),
8855 )
8856 .await
8857 .unwrap();
8858 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
8859
8860 let described = handler
8861 .handle_control_frame(
8862 &route_ctx(ConnectionId::new(76)).0,
8863 Frame::build(
8864 FrameType::Request,
8865 control_flags(),
8866 0,
8867 0,
8868 14,
8869 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
8870 )
8871 .unwrap(),
8872 )
8873 .await
8874 .unwrap();
8875 let ClientControlResponse::ServerDescribe { capabilities, .. } =
8876 serde_json::from_slice(&described[0].body).unwrap()
8877 else {
8878 panic!("expected server.describe response")
8879 };
8880 assert!(capabilities
8881 .iter()
8882 .any(|cap| cap == "admission_facts_relay_v1"));
8883 }
8884
8885 #[tokio::test]
8886 async fn admission_facts_without_configured_carrier_are_rejected() {
8887 let registry = Arc::new(Registry::default());
8888 let forwarding = Arc::new(ForwardingTable::default());
8889 let handler = ControlHandler::with_forwarding(registry, forwarding);
8890 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
8891 hello_via_sink(
8892 &handler,
8893 &target_ctx,
8894 &mut target_rx,
8895 hello_frame("target", PROTOCOL_VERSION, 1),
8896 )
8897 .await;
8898
8899 let responses = handler
8900 .handle_control_frame(
8901 &route_ctx(ConnectionId::new(79)).0,
8902 route_open_frame_with_admission_facts(
8903 16,
8904 "target",
8905 unique_project_root("admission-facts"),
8906 None,
8907 Some(json!({"x": 1})),
8908 ),
8909 )
8910 .await
8911 .unwrap();
8912 assert_eq!(
8913 parse_error(&responses[0])["code"],
8914 "admission_facts_not_permitted"
8915 );
8916 }
8917
8918 #[tokio::test]
8919 async fn route_open_relays_consumer_capabilities_verbatim() {
8920 let registry = Arc::new(Registry::default());
8921 let forwarding = Arc::new(ForwardingTable::default());
8922 let handler =
8923 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8924 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
8925 hello_via_sink(
8926 &handler,
8927 &module_ctx,
8928 &mut module_rx,
8929 hello_frame("aft", PROTOCOL_VERSION, 7),
8930 )
8931 .await;
8932
8933 let expected = vec!["elicitation".to_string(), "roots".to_string()];
8934 let expected_for_request = expected.clone();
8935 let project_root = unique_project_root("consumer-capabilities-present");
8936 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
8937 let route_handler = handler.clone();
8938 let route_task = tokio::spawn(async move {
8939 route_handler
8940 .handle_control_frame(
8941 &client_ctx,
8942 route_open_frame_with_consumer_capabilities(
8943 401,
8944 "aft",
8945 project_root,
8946 Some(expected_for_request),
8947 ),
8948 )
8949 .await
8950 .unwrap()
8951 });
8952 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8953 .await
8954 .unwrap()
8955 .unwrap();
8956 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8957 let ModuleControlRequest::RouteBind {
8958 consumer_capabilities,
8959 ..
8960 } = bind
8961 else {
8962 panic!("expected route.bind request, got {bind:?}");
8963 };
8964 assert_eq!(consumer_capabilities, Some(expected.clone()));
8965
8966 handler
8967 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8968 .await
8969 .unwrap();
8970 let route_response = route_task.await.unwrap();
8971 assert!(route_response.is_empty());
8972 let published = client_rx.recv().await.unwrap();
8973 assert!(matches!(
8974 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8975 ClientControlResponse::RouteOpen { .. }
8976 ));
8977 }
8978
8979 #[tokio::test]
8980 async fn route_open_without_consumer_capabilities_relays_none() {
8981 let registry = Arc::new(Registry::default());
8982 let forwarding = Arc::new(ForwardingTable::default());
8983 let handler =
8984 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8985 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
8986 hello_via_sink(
8987 &handler,
8988 &module_ctx,
8989 &mut module_rx,
8990 hello_frame("aft", PROTOCOL_VERSION, 7),
8991 )
8992 .await;
8993
8994 let project_root = unique_project_root("consumer-capabilities-absent");
8995 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
8996 let route_handler = handler.clone();
8997 let route_task = tokio::spawn(async move {
8998 route_handler
8999 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
9000 .await
9001 .unwrap()
9002 });
9003 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9004 .await
9005 .unwrap()
9006 .unwrap();
9007 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9008 let ModuleControlRequest::RouteBind {
9009 consumer_capabilities,
9010 ..
9011 } = bind
9012 else {
9013 panic!("expected route.bind request, got {bind:?}");
9014 };
9015 assert_eq!(consumer_capabilities, None);
9016
9017 handler
9018 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9019 .await
9020 .unwrap();
9021 let route_response = route_task.await.unwrap();
9022 assert!(route_response.is_empty());
9023 let published = client_rx.recv().await.unwrap();
9024 assert!(matches!(
9025 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9026 ClientControlResponse::RouteOpen { .. }
9027 ));
9028 }
9029
9030 async fn bind_role_versions_for(
9034 sent: Option<BTreeMap<String, String>>,
9035 connection: u64,
9036 ) -> Option<BTreeMap<String, String>> {
9037 let registry = Arc::new(Registry::default());
9038 let forwarding = Arc::new(ForwardingTable::default());
9039 let handler =
9040 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9041 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
9042 hello_via_sink(
9043 &handler,
9044 &module_ctx,
9045 &mut module_rx,
9046 hello_frame("aft", PROTOCOL_VERSION, 7),
9047 )
9048 .await;
9049 let project_root = unique_project_root("role-versions");
9050 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
9051 let route_handler = handler.clone();
9052 let route_task = tokio::spawn(async move {
9053 route_handler
9054 .handle_control_frame(
9055 &client_ctx,
9056 route_open_frame_with_role_versions(403, "aft", project_root, sent),
9057 )
9058 .await
9059 .unwrap()
9060 });
9061 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9062 .await
9063 .expect("a well-formed route.open reaches the module as a bind")
9064 .unwrap();
9065 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9066 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
9067 panic!("expected route.bind request, got {bind:?}");
9068 };
9069 handler
9070 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9071 .await
9072 .unwrap();
9073 assert!(route_task.await.unwrap().is_empty());
9074 let published = client_rx.recv().await.unwrap();
9075 assert!(matches!(
9076 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9077 ClientControlResponse::RouteOpen { .. }
9078 ));
9079 role_versions
9080 }
9081
9082 #[tokio::test]
9083 async fn route_open_relays_role_versions_verbatim() {
9084 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
9085 assert_eq!(
9086 bind_role_versions_for(Some(sent.clone()), 141).await,
9087 Some(sent)
9088 );
9089 }
9090
9091 #[tokio::test]
9094 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
9095 assert_eq!(bind_role_versions_for(None, 143).await, None);
9096 assert_eq!(
9097 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
9098 None
9099 );
9100 }
9101
9102 #[tokio::test]
9103 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
9104 let registry = Arc::new(Registry::default());
9105 let forwarding = Arc::new(ForwardingTable::default());
9106 let handler =
9107 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9108 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
9109 hello_via_sink(
9110 &handler,
9111 &module_ctx,
9112 &mut module_rx,
9113 hello_frame("aft", PROTOCOL_VERSION, 7),
9114 )
9115 .await;
9116
9117 let nine: BTreeMap<String, String> = (0..9)
9118 .map(|index| (format!("role-{index}"), "v1".to_string()))
9119 .collect();
9120 for (label, malformed) in [
9121 (
9122 "invalid role name",
9123 role_versions(&[("Tool_Provider", "v1")]),
9124 ),
9125 ("invalid version", role_versions(&[("tool-provider", "v0")])),
9126 ("nine entries", nine),
9127 ] {
9128 let responses = tokio::time::timeout(
9131 Duration::from_secs(1),
9132 handler.handle_control_frame(
9133 &route_ctx(ConnectionId::new(148)).0,
9134 route_open_frame_with_role_versions(
9135 404,
9136 "aft",
9137 unique_project_root("role-versions-malformed"),
9138 Some(malformed),
9139 ),
9140 ),
9141 )
9142 .await
9143 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
9144 .unwrap();
9145 assert_eq!(responses.len(), 1, "{label}");
9146 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
9147 let error = parse_error(&responses[0]);
9148 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
9149 assert_eq!(
9150 error["detail"]["field"], "role_versions",
9151 "{label}: {error}"
9152 );
9153 assert!(
9154 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
9155 "{label}: a malformed declaration is terminal"
9156 );
9157 assert!(
9158 module_rx.try_recv().is_err(),
9159 "{label}: the module must never see a bind"
9160 );
9161 }
9162 }
9163
9164 #[tokio::test]
9168 async fn route_role_versions_capability_is_advertised() {
9169 let handler = ControlHandler::new(Arc::new(Registry::default()));
9170 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
9171 let ack = hello_via_sink(
9172 &handler,
9173 &ctx,
9174 &mut rx,
9175 hello_frame("m", PROTOCOL_VERSION, 1),
9176 )
9177 .await;
9178 let ack = parse_ack(&ack);
9179 assert!(
9180 ack.subc_capabilities
9181 .iter()
9182 .any(|c| c == "route-role-versions/v1"),
9183 "{:?}",
9184 ack.subc_capabilities
9185 );
9186
9187 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
9188 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
9189 let reply = handler
9190 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
9191 .await
9192 .unwrap()
9193 .pop()
9194 .unwrap();
9195 let ClientControlResponse::ServerDescribe { capabilities, .. } =
9196 serde_json::from_slice(&reply.body).unwrap()
9197 else {
9198 panic!("not a server.describe reply");
9199 };
9200 assert!(
9201 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
9202 "{capabilities:?}"
9203 );
9204 }
9205
9206 #[tokio::test]
9207 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
9208 let registry = Arc::new(Registry::default());
9209 let forwarding = Arc::new(ForwardingTable::default());
9210 let handler =
9211 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9212 .with_health_probe_timeout(Duration::from_secs(5));
9213 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
9214 hello_via_sink(
9215 &handler,
9216 &module_ctx,
9217 &mut module_rx,
9218 non_routable_hello_frame_with_control_ops(
9219 "mcp",
9220 300,
9221 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9222 ),
9223 )
9224 .await;
9225 assert!(registry
9226 .get_module("mcp")
9227 .unwrap()
9228 .unwrap()
9229 .manifest
9230 .provides
9231 .is_empty());
9232
9233 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
9234 let route_response = handler
9235 .handle_control_frame(
9236 &route_client_ctx,
9237 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
9238 )
9239 .await
9240 .unwrap();
9241 assert_eq!(route_response[0].header.ty, FrameType::Error);
9242 assert_eq!(
9243 parse_error(&route_response[0])["code"],
9244 "target_unavailable"
9245 );
9246 assert!(parse_error(&route_response[0])["message"]
9247 .as_str()
9248 .unwrap()
9249 .contains("does not provide the requested target"));
9250 assert!(module_rx.try_recv().is_err());
9251
9252 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
9253 let health_handler = handler.clone();
9254 let health_task = tokio::spawn(async move {
9255 health_handler
9256 .handle_control_frame(
9257 &health_client_ctx,
9258 supervisor_health_probe_frame(302, "mcp"),
9259 )
9260 .await
9261 .unwrap()
9262 });
9263 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9264 .await
9265 .unwrap()
9266 .unwrap();
9267 assert_eq!(
9268 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
9269 ModuleControlRequest::HealthCheck {}
9270 );
9271 handler
9272 .handle_control_frame(
9273 &module_ctx,
9274 health_response(health_frame.header.corr, HealthStatus::Ok),
9275 )
9276 .await
9277 .unwrap();
9278 let health_response = health_task.await.unwrap();
9279 assert_eq!(health_response[0].header.ty, FrameType::Response);
9280 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
9281 ClientControlResponse::SupervisorHealthProbe {
9282 module_id, status, ..
9283 } => {
9284 assert_eq!(module_id, "mcp");
9285 assert_eq!(status, HealthStatus::Ok);
9286 }
9287 other => panic!("unexpected health response: {other:?}"),
9288 }
9289
9290 forwarding
9295 .cleanup_connection(module_ctx.connection_id)
9296 .unwrap();
9297 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
9298 let cleanup_response = tokio::time::timeout(
9299 Duration::from_millis(200),
9300 handler.handle_control_frame(
9301 &cleanup_probe_ctx,
9302 supervisor_health_probe_frame(303, "mcp"),
9303 ),
9304 )
9305 .await
9306 .expect("probe should fail immediately when the control lane is gone")
9307 .unwrap();
9308 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
9309 assert_eq!(
9310 parse_error(&cleanup_response[0])["code"],
9311 "target_unavailable"
9312 );
9313 assert!(parse_error(&cleanup_response[0])["message"]
9314 .as_str()
9315 .unwrap()
9316 .contains("no module connection"));
9317
9318 handler
9319 .cleanup_connection(module_ctx.connection_id)
9320 .unwrap();
9321 }
9322
9323 #[tokio::test]
9324 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
9325 let registry = Arc::new(Registry::default());
9326 let supervisor_handle = SupervisorHandle::new();
9327 let supervisor =
9328 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
9329 .with_handle(supervisor_handle.clone())
9330 .with_connection_file_path(
9331 std::env::temp_dir()
9332 .join(format!("subc-route-open-warming-{}", std::process::id())),
9333 );
9334 let module = supervisor
9335 .supervise_configured(
9336 ModuleSpec {
9337 module_id: "warming".to_string(),
9338 program: fake_aft_stub_path(),
9339 args: Vec::new(),
9340 env: Vec::new(),
9341 reserved: false,
9342 reserved_prefixes: Vec::new(),
9343 protocol: ModuleProtocol::Subc,
9344 overlap: Default::default(),
9345 },
9346 true,
9347 )
9348 .unwrap();
9349 assert_eq!(module.state().unwrap(), ModuleState::Running);
9350
9351 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9352 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
9353 let response = handler
9354 .handle_control_frame(
9355 &ctx,
9356 route_open_frame(304, "warming", unique_project_root("warming")),
9357 )
9358 .await
9359 .unwrap();
9360 module.stop().await.unwrap();
9361
9362 assert_eq!(response[0].header.ty, FrameType::Error);
9363 let error = parse_error(&response[0]);
9364 assert_eq!(error["code"], "module_warming");
9365 assert!(error["message"]
9366 .as_str()
9367 .unwrap()
9368 .contains("state=running, enabled=true, live=false"));
9369 }
9370
9371 #[test]
9372 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
9373 let handler = ControlHandler::new(Arc::new(Registry::default()));
9374 let capture = EventCapture::default();
9375 let _subscriber =
9376 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9377 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
9378 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
9379 let pending = (0..limit).collect::<Vec<_>>();
9380 let response = handler
9381 .route_open_capacity_refusal(
9382 &ctx,
9383 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
9384 "busy",
9385 pending.len(),
9386 limit,
9387 )
9388 .unwrap();
9389 assert_eq!(parse_error(&response)["code"], "target_unavailable");
9390 let event = capture
9391 .events()
9392 .into_iter()
9393 .find(|event| {
9394 event.target == "control"
9395 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
9396 })
9397 .expect("connection admission refusal event");
9398 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
9399 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
9400 }
9401
9402 #[test]
9403 fn route_open_target_cap_logs_admission_reason_and_capacity() {
9404 let handler = ControlHandler::new(Arc::new(Registry::default()));
9405 let capture = EventCapture::default();
9406 let _subscriber =
9407 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9408 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
9409 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
9410 let guards = (0..limit)
9411 .map(|_| {
9412 handler
9413 .route_bind_concurrency
9414 .try_admit("busy", limit)
9415 .unwrap()
9416 })
9417 .collect::<Vec<_>>();
9418 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
9419 Err(in_flight) => in_flight,
9420 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
9421 };
9422 let response = handler
9423 .route_open_target_capacity_refusal(
9424 &ctx,
9425 &route_open_frame(397, "busy", unique_project_root("target-cap")),
9426 "busy",
9427 in_flight,
9428 )
9429 .unwrap();
9430 assert_eq!(parse_error(&response)["code"], "target_unavailable");
9431 let event = capture
9432 .events()
9433 .into_iter()
9434 .find(|event| {
9435 event.target == "control"
9436 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
9437 })
9438 .expect("target admission refusal event");
9439 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
9440 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
9441 drop(guards);
9442 }
9443
9444 #[tokio::test]
9449 async fn route_open_refusal_names_the_check_that_refused() {
9450 let handler = ControlHandler::new(Arc::new(Registry::default()));
9451 let capture = EventCapture::default();
9452 let _subscriber =
9453 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9454 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
9455 let response = handler
9456 .handle_control_frame(
9457 &ctx,
9458 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
9459 )
9460 .await
9461 .unwrap();
9462
9463 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
9464 let event = capture
9465 .events()
9466 .into_iter()
9467 .find(|event| {
9468 event.target == "control"
9469 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
9470 })
9471 .expect("route.open refusal event");
9472 assert_eq!(
9473 event.fields.get("reason"),
9474 Some(&"\"not_registered\"".to_string())
9475 );
9476 }
9477
9478 #[tokio::test]
9479 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
9480 let registry = Arc::new(Registry::default());
9481 let supervisor_handle = SupervisorHandle::new();
9482 let supervisor =
9483 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
9484 .with_handle(supervisor_handle.clone())
9485 .with_connection_file_path(std::env::temp_dir().join(format!(
9486 "subc-route-open-refusal-info-{}",
9487 std::process::id()
9488 )));
9489 let module = supervisor
9490 .supervise_configured(
9491 ModuleSpec {
9492 module_id: "warming".to_string(),
9493 program: fake_aft_stub_path(),
9494 args: Vec::new(),
9495 env: Vec::new(),
9496 reserved: false,
9497 reserved_prefixes: Vec::new(),
9498 protocol: ModuleProtocol::Subc,
9499 overlap: Default::default(),
9500 },
9501 true,
9502 )
9503 .unwrap();
9504 assert_eq!(module.state().unwrap(), ModuleState::Running);
9505
9506 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9507 assert!(handler
9508 .counters()
9509 .snapshot()
9510 .get("route_open_refused_by_code")
9511 .is_none());
9512 let capture = EventCapture::default();
9513 let _subscriber =
9514 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9515 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
9516 let response = handler
9517 .handle_control_frame(
9518 &ctx,
9519 route_open_frame(394, "warming", unique_project_root("refusal-info")),
9520 )
9521 .await
9522 .unwrap();
9523 module.stop().await.unwrap();
9524
9525 assert_eq!(parse_error(&response[0])["code"], "module_warming");
9526 let event = capture
9527 .events()
9528 .into_iter()
9529 .find(|event| {
9530 event.target == "control"
9531 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
9532 })
9533 .expect("route.open refusal event");
9534 assert_eq!(
9535 event.fields.get("module_id"),
9536 Some(&"\"warming\"".to_string())
9537 );
9538 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
9539 assert_eq!(
9540 event.fields.get("reason"),
9541 Some(&"\"supervised_not_registered\"".to_string())
9542 );
9543 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
9544 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
9545 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
9546 assert_eq!(
9547 handler.counters().snapshot()["route_open_refused_by_code"],
9548 json!({ "module_warming": 1 })
9549 );
9550 }
9551
9552 const OUTAGE_START: &str = "route.open refusing module: not serving";
9553 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
9554
9555 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
9556 capture
9557 .events()
9558 .into_iter()
9559 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
9560 .collect()
9561 }
9562
9563 fn supervise_stub(
9564 registry: &Arc<Registry>,
9565 module_id: &str,
9566 enabled: bool,
9567 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
9568 let supervisor_handle = SupervisorHandle::new();
9569 let supervisor =
9570 Supervisor::new(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
9571 .with_handle(supervisor_handle.clone())
9572 .with_connection_file_path(std::env::temp_dir().join(format!(
9573 "subc-route-outage-{module_id}-{}",
9574 std::process::id()
9575 )));
9576 let module = supervisor
9577 .supervise_configured(
9578 ModuleSpec {
9579 module_id: module_id.to_string(),
9580 program: fake_aft_stub_path(),
9581 args: Vec::new(),
9582 env: Vec::new(),
9583 reserved: false,
9584 reserved_prefixes: Vec::new(),
9585 protocol: ModuleProtocol::Subc,
9586 overlap: Default::default(),
9587 },
9588 enabled,
9589 )
9590 .unwrap();
9591 (supervisor_handle, module)
9592 }
9593
9594 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
9595 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
9596 module_id: module_id.to_string(),
9597 drain_timeout_ms: Some(50),
9598 })
9599 .unwrap();
9600 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9601 }
9602
9603 #[test]
9607 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
9608 let registry = Arc::new(Registry::default());
9609 let forwarding = Arc::new(ForwardingTable::default());
9610 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9611 let second = ControlHandler::with_forwarding(registry, forwarding);
9612 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
9613 }
9614
9615 #[tokio::test(flavor = "current_thread")]
9619 async fn route_open_unknown_module_refusals_add_no_outage_state() {
9620 let handler = ControlHandler::new(Arc::new(Registry::default()));
9621 let capture = EventCapture::default();
9622 let _subscriber =
9623 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9624 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
9625 for corr in 0..8 {
9626 let response = handler
9627 .handle_control_frame(
9628 &ctx,
9629 route_open_frame(
9630 380 + corr,
9631 &format!("nobody-{corr}"),
9632 unique_project_root("outage-unknown"),
9633 ),
9634 )
9635 .await
9636 .unwrap();
9637 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
9638 }
9639
9640 assert_eq!(handler.route_outages.tracked_module_count(), 0);
9641 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
9642 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
9643 }
9644
9645 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9650 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
9651 let registry = Arc::new(Registry::default());
9652 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
9653 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9654 let capture = EventCapture::default();
9655 let _subscriber =
9656 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9657 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
9658 handler.route_outages.record_accepted("outage-restart");
9661
9662 let response = handler
9663 .handle_control_frame(
9664 &ctx,
9665 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
9666 )
9667 .await
9668 .unwrap();
9669 assert_eq!(response[0].header.ty, FrameType::Error);
9670 let starts = outage_lines(&capture, OUTAGE_START);
9671 assert_eq!(starts.len(), 1, "{starts:?}");
9672 assert_eq!(starts[0].level, tracing::Level::WARN);
9673 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
9674 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
9675 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
9676 handler.route_outages.record_accepted("outage-restart");
9677 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
9678
9679 let restart = handler
9680 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
9681 .await
9682 .unwrap();
9683 assert_eq!(
9684 restart[0].header.ty,
9685 FrameType::Response,
9686 "{:?}",
9687 parse_error(&restart[0])
9688 );
9689 handler
9690 .handle_control_frame(
9691 &ctx,
9692 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
9693 )
9694 .await
9695 .unwrap();
9696 module.stop().await.unwrap();
9697
9698 let starts = outage_lines(&capture, OUTAGE_START);
9699 assert_eq!(starts.len(), 2, "{starts:?}");
9700 assert_eq!(starts[1].level, tracing::Level::INFO);
9701 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
9702 }
9703
9704 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9708 async fn failed_operator_restart_leaves_no_operator_mark() {
9709 let registry = Arc::new(Registry::default());
9710 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
9711 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9712 let capture = EventCapture::default();
9713 let _subscriber =
9714 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9715 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
9716 handler.route_outages.record_accepted("outage-disabled");
9717
9718 let restart = handler
9719 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
9720 .await
9721 .unwrap();
9722 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
9723 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
9724
9725 handler
9726 .handle_control_frame(
9727 &ctx,
9728 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
9729 )
9730 .await
9731 .unwrap();
9732 let starts = outage_lines(&capture, OUTAGE_START);
9733 assert_eq!(starts.len(), 1, "{starts:?}");
9734 assert_eq!(starts[0].level, tracing::Level::WARN);
9735 }
9736
9737 #[tokio::test(flavor = "current_thread")]
9738 async fn route_open_unknown_module_escapes_target_module_id() {
9739 let handler = ControlHandler::new(Arc::new(Registry::default()));
9740 let capture = EventCapture::default();
9741 let _subscriber =
9742 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9743 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
9744 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
9745 let response = handler
9746 .handle_control_frame(
9747 &ctx,
9748 route_open_frame(
9749 395,
9750 hostile_module_id,
9751 unique_project_root("hostile-target-module-id"),
9752 ),
9753 )
9754 .await
9755 .unwrap();
9756
9757 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
9758 let event = capture
9759 .events()
9760 .into_iter()
9761 .find(|event| {
9762 event.target == "control"
9763 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
9764 })
9765 .expect("route.open unknown-module refusal event");
9766 let logged = event.fields.get("module_id").expect("module_id field");
9767 assert!(!logged.bytes().any(|byte| byte < 0x20));
9768 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
9769 }
9770
9771 #[tokio::test(flavor = "current_thread")]
9772 async fn route_open_module_rejection_uses_daemon_counter_key() {
9773 let registry = Arc::new(Registry::default());
9774 let forwarding = Arc::new(ForwardingTable::default());
9775 let handler =
9776 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9777 let module_connection = ConnectionId::new(95);
9778 let (module_ctx, mut module_rx) = route_ctx(module_connection);
9779 hello_via_sink(
9780 &handler,
9781 &module_ctx,
9782 &mut module_rx,
9783 hello_frame("aft", PROTOCOL_VERSION, 395),
9784 )
9785 .await;
9786
9787 let client_connection = ConnectionId::new(96);
9788 let (client_ctx, _client_rx) = route_ctx(client_connection);
9789 let capture = EventCapture::default();
9790 let _subscriber =
9791 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9792 let (route_task, bind) = relay_route_open(
9793 &handler,
9794 client_connection,
9795 &client_ctx.egress,
9796 &mut module_rx,
9797 396,
9798 "aft",
9799 "hostile-module-code",
9800 )
9801 .await;
9802 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
9803 let rejection = Frame::build(
9804 FrameType::Error,
9805 control_flags(),
9806 0,
9807 0,
9808 bind.header.corr,
9809 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
9810 )
9811 .unwrap();
9812 handler
9813 .handle_control_frame(&module_ctx, rejection)
9814 .await
9815 .unwrap();
9816
9817 let response = route_task.await.unwrap();
9818 assert_eq!(parse_error(&response[0])["code"], hostile_code);
9819 let counters = handler.counters().snapshot();
9820 assert_eq!(
9821 counters["route_open_refused_by_code"],
9822 json!({ "module_rejected": 1 })
9823 );
9824 assert!(counters["route_open_refused_by_code"]
9825 .get(hostile_code)
9826 .is_none());
9827
9828 let event = capture
9829 .events()
9830 .into_iter()
9831 .find(|event| {
9832 event.target == "control"
9833 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
9834 })
9835 .expect("route.open module-rejection refusal event");
9836 let logged = event.fields.get("module_code").expect("module_code field");
9837 assert!(!logged.bytes().any(|byte| byte < 0x20));
9838 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
9839 }
9840
9841 #[tokio::test]
9842 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
9843 let registry = Arc::new(Registry::default());
9844 let supervisor_handle = SupervisorHandle::new();
9845 let missing_program = std::env::temp_dir().join(format!(
9846 "subc-route-open-missing-program-{}",
9847 std::process::id()
9848 ));
9849 let supervisor =
9850 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
9851 .with_handle(supervisor_handle.clone());
9852 let module = supervisor
9853 .supervise_configured(
9854 ModuleSpec {
9855 module_id: "failed".to_string(),
9856 program: missing_program,
9857 args: Vec::new(),
9858 env: Vec::new(),
9859 reserved: false,
9860 reserved_prefixes: Vec::new(),
9861 protocol: ModuleProtocol::Subc,
9862 overlap: Default::default(),
9863 },
9864 true,
9865 )
9866 .unwrap();
9867 assert_eq!(module.state().unwrap(), ModuleState::Failed);
9868
9869 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9870 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
9871 let response = handler
9872 .handle_control_frame(
9873 &ctx,
9874 route_open_frame(305, "failed", unique_project_root("failed")),
9875 )
9876 .await
9877 .unwrap();
9878
9879 assert_eq!(response[0].header.ty, FrameType::Error);
9880 let error = parse_error(&response[0]);
9881 assert_eq!(error["code"], "target_unavailable");
9882 assert!(error["message"]
9883 .as_str()
9884 .unwrap()
9885 .contains("state=failed, enabled=true, live=false"));
9886 }
9887
9888 #[tokio::test]
9889 async fn route_open_role_mismatch_remains_target_unavailable() {
9890 let registry = Arc::new(Registry::default());
9891 let handler = ControlHandler::new(Arc::clone(®istry));
9892 handler
9893 .handle_control(
9894 ConnectionId::new(41),
9895 non_routable_hello_frame_with_control_ops("health-only", 306, None),
9896 )
9897 .unwrap();
9898
9899 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
9900 let response = handler
9901 .handle_control_frame(
9902 &ctx,
9903 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
9904 )
9905 .await
9906 .unwrap();
9907
9908 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
9909 assert!(parse_error(&response[0])["message"]
9910 .as_str()
9911 .unwrap()
9912 .contains("does not provide the requested target"));
9913 }
9914
9915 #[tokio::test]
9916 async fn route_open_inactive_registration_remains_target_unavailable() {
9917 let registry = Arc::new(Registry::default());
9918 let handler = ControlHandler::new(Arc::clone(®istry));
9919 handler
9920 .handle_control(
9921 ConnectionId::new(43),
9922 hello_frame("inactive", PROTOCOL_VERSION, 308),
9923 )
9924 .unwrap();
9925 assert!(registry
9926 .set_module_state_for_test("inactive", ChannelState::Closed)
9927 .unwrap());
9928
9929 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
9930 let response = handler
9931 .handle_control_frame(
9932 &ctx,
9933 route_open_frame(309, "inactive", unique_project_root("inactive")),
9934 )
9935 .await
9936 .unwrap();
9937
9938 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
9939 assert!(parse_error(&response[0])["message"]
9940 .as_str()
9941 .unwrap()
9942 .contains("is not active"));
9943 }
9944
9945 #[tokio::test]
9946 async fn late_health_reply_is_recorded_through_the_module_response_path() {
9947 let registry = Arc::new(Registry::default());
9948 let forwarding = Arc::new(ForwardingTable::default());
9949 let supervisor_handle = SupervisorHandle::new();
9950 let supervisor = Supervisor::new(Arc::clone(®istry), crate::RestartPolicy::default())
9951 .with_forwarding(Arc::clone(&forwarding))
9952 .with_handle(supervisor_handle.clone());
9953 let module = supervisor
9954 .supervise_configured(
9955 crate::ModuleSpec {
9956 module_id: "late-health-response".to_string(),
9957 program: PathBuf::from("disabled-module"),
9958 args: Vec::new(),
9959 env: Vec::new(),
9960 reserved: false,
9961 reserved_prefixes: Vec::new(),
9962 protocol: ModuleProtocol::Subc,
9963 overlap: Default::default(),
9964 },
9965 false,
9966 )
9967 .unwrap();
9968 let handler =
9969 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9970 .with_supervisor(supervisor_handle);
9971 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
9972 handler
9973 .handle_control_frame(
9974 &module_ctx,
9975 hello_frame_with_control_ops(
9976 "late-health-response",
9977 PROTOCOL_VERSION,
9978 7,
9979 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9980 ),
9981 )
9982 .await
9983 .unwrap();
9984 let probe_started_at = Instant::now() - Duration::from_millis(80);
9985 let pending = forwarding
9986 .begin_health_probe_rpc_for(
9987 "late-health-response",
9988 MODULE_CONTROL_OP_HEALTH_CHECK,
9989 probe_started_at,
9990 Instant::now() - Duration::from_millis(1),
9991 )
9992 .unwrap();
9993 assert!(forwarding
9994 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
9995 .unwrap());
9996
9997 let responses = handler
9998 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
9999 .await
10000 .unwrap();
10001
10002 assert!(responses.is_empty());
10003 let health = module.status().unwrap().health;
10004 assert_eq!(health.late_answer_count, 1);
10005 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
10006 }
10007
10008 #[tokio::test]
10009 async fn health_probe_timeout_and_module_death_are_typed() {
10010 let registry = Arc::new(Registry::default());
10011 let forwarding = Arc::new(ForwardingTable::default());
10012 let handler =
10013 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10014 .with_health_probe_timeout(Duration::from_millis(50));
10015 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
10016 hello_via_sink(
10017 &handler,
10018 &module_ctx,
10019 &mut module_rx,
10020 hello_frame_with_control_ops(
10021 "aft",
10022 PROTOCOL_VERSION,
10023 7,
10024 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10025 ),
10026 )
10027 .await;
10028
10029 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10030 let responses = handler
10031 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
10032 .await
10033 .unwrap();
10034 assert_eq!(responses[0].header.ty, FrameType::Error);
10035 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
10036 let _ = module_rx.try_recv();
10037
10038 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
10039 let health_handler = handler.clone();
10040 let death_task = tokio::spawn(async move {
10041 health_handler
10042 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
10043 .await
10044 .unwrap()
10045 });
10046 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10047 .await
10048 .unwrap()
10049 .unwrap();
10050 handler
10051 .cleanup_connection(module_ctx.connection_id)
10052 .unwrap();
10053 let responses = death_task.await.unwrap();
10054 assert_eq!(responses[0].header.ty, FrameType::Error);
10055 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
10056 }
10057
10058 #[test]
10059 fn hello_requires_exact_protocol_version() {
10060 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
10061 let registry = Arc::new(Registry::default());
10062 let handler = ControlHandler::new(Arc::clone(®istry));
10063 let responses = handler
10064 .handle_control(
10065 ConnectionId::new(connection),
10066 hello_frame("aft", offered, 9),
10067 )
10068 .unwrap();
10069
10070 assert_eq!(responses.len(), 1);
10071 assert_eq!(responses[0].header.ty, FrameType::Error);
10072 let error = parse_error(&responses[0]);
10073 assert_eq!(error["code"], "version_unsupported");
10074 assert!(registry.get_module("aft").unwrap().is_none());
10075 assert_eq!(registry.active_registration_count().unwrap(), 0);
10076 }
10077 }
10078
10079 #[test]
10080 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
10081 let registry = Arc::new(Registry::default());
10082 let forwarding = Arc::new(ForwardingTable::default());
10083 let handler =
10084 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10085 let module_connection = ConnectionId::new(301);
10086 let registration = registry
10087 .register_with_control_ops(
10088 manifest("aft-push", PROTOCOL_VERSION),
10089 PROTOCOL_VERSION,
10090 module_connection,
10091 module_baseline_control_ops(),
10092 )
10093 .unwrap();
10094 let (module_tx, _module_rx) = mpsc::channel(8);
10095 let endpoint = forwarding
10096 .register_module_connection(
10097 module_connection,
10098 "aft-push".to_string(),
10099 PROTOCOL_VERSION,
10100 manifest_concurrency(®istration.manifest),
10101 FrameSink::new(module_tx),
10102 )
10103 .unwrap();
10104
10105 let unknown = Frame::build(
10107 FrameType::Push,
10108 control_flags(),
10109 0,
10110 0,
10111 5,
10112 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
10113 )
10114 .unwrap();
10115 let out = handler.handle_status_update(endpoint, unknown).unwrap();
10116 assert!(
10117 out.is_empty(),
10118 "unknown push op must be ignored, got {out:?}"
10119 );
10120
10121 let malformed = Frame::build(
10123 FrameType::Push,
10124 control_flags(),
10125 0,
10126 0,
10127 6,
10128 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
10129 )
10130 .unwrap();
10131 let out = handler.handle_status_update(endpoint, malformed).unwrap();
10132 assert_eq!(out.len(), 1);
10133 assert_eq!(out[0].header.ty, FrameType::Error);
10134 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
10135 }
10136
10137 #[test]
10138 fn hello_rejected_when_connection_already_owns_client_routes() {
10139 let registry = Arc::new(Registry::default());
10140 let forwarding = Arc::new(ForwardingTable::default());
10141 let handler =
10142 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10143 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
10145 let client_connection = ConnectionId::new(202);
10146
10147 let responses = handler
10150 .handle_control(
10151 client_connection,
10152 hello_frame("aft-second", PROTOCOL_VERSION, 9),
10153 )
10154 .unwrap();
10155 assert_eq!(responses[0].header.ty, FrameType::Error);
10156 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
10157 assert!(registry.get_module("aft-second").unwrap().is_none());
10158 }
10159
10160 #[test]
10161 fn reserved_module_hello_requires_matching_launch_nonce() {
10162 let registry = Arc::new(Registry::default());
10163 let supervisor = SupervisorHandle::new();
10164 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
10167 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
10168
10169 let no_nonce = handler
10171 .handle_control(
10172 ConnectionId::new(1),
10173 hello_frame("vault", PROTOCOL_VERSION, 1),
10174 )
10175 .unwrap();
10176 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
10177 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
10178 assert!(registry.get_module("vault").unwrap().is_none());
10179
10180 let wrong = handler
10182 .handle_control(
10183 ConnectionId::new(2),
10184 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
10185 )
10186 .unwrap();
10187 assert_eq!(wrong[0].header.ty, FrameType::Error);
10188 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
10189 assert!(registry.get_module("vault").unwrap().is_none());
10190
10191 let ok = handler
10193 .handle_control(
10194 ConnectionId::new(3),
10195 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
10196 )
10197 .unwrap();
10198 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
10199 assert!(registry.get_module("vault").unwrap().is_some());
10200 }
10201
10202 #[test]
10203 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
10204 let registry = Arc::new(Registry::default());
10205 let supervisor = SupervisorHandle::new();
10206 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
10207 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
10208 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
10209
10210 let squat = handler
10211 .handle_control(
10212 ConnectionId::new(1),
10213 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
10214 )
10215 .unwrap();
10216 assert_eq!(squat[0].header.ty, FrameType::Error);
10217 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
10218 assert!(parse_error(&squat[0])["message"]
10219 .as_str()
10220 .unwrap()
10221 .contains("fed:"));
10222
10223 let accepted_peer = handler
10224 .handle_control(
10225 ConnectionId::new(2),
10226 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
10227 )
10228 .unwrap();
10229 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
10230
10231 let accepted_short = handler
10232 .handle_control(
10233 ConnectionId::new(3),
10234 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
10235 )
10236 .unwrap();
10237 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
10238
10239 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
10240 let response = handler
10241 .handle_control(
10242 ConnectionId::new(conn),
10243 hello_frame(module_id, PROTOCOL_VERSION, conn),
10244 )
10245 .unwrap();
10246 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
10247 }
10248 }
10249
10250 #[test]
10251 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
10252 let registry = Arc::new(Registry::default());
10253 let supervisor = SupervisorHandle::new();
10254 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
10255 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
10256 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
10257 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
10258
10259 let owner_nonce = handler
10260 .handle_control(
10261 ConnectionId::new(1),
10262 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
10263 )
10264 .unwrap();
10265 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
10266 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
10267 assert!(registry.get_module("fed:special").unwrap().is_none());
10268
10269 let exact_nonce = handler
10270 .handle_control(
10271 ConnectionId::new(2),
10272 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
10273 )
10274 .unwrap();
10275 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
10276 assert!(registry.get_module("fed:special").unwrap().is_some());
10277 }
10278
10279 #[test]
10280 fn non_reserved_module_ignores_launch_nonce() {
10281 let registry = Arc::new(Registry::default());
10282 let handler = ControlHandler::new(Arc::clone(®istry));
10285 let no_nonce = handler
10286 .handle_control(
10287 ConnectionId::new(1),
10288 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
10289 )
10290 .unwrap();
10291 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
10292 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
10293
10294 let echoed_nonce = handler
10295 .handle_control(
10296 ConnectionId::new(2),
10297 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
10298 )
10299 .unwrap();
10300 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
10301 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
10302 }
10303
10304 #[test]
10305 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
10306 let handler = ControlHandler::default();
10307 let conn = ConnectionId::new(1);
10308 let malformed = Frame::build(
10309 FrameType::Hello,
10310 control_flags(),
10311 0,
10312 0,
10313 3,
10314 b"{not json".to_vec(),
10315 )
10316 .unwrap();
10317
10318 let error = handler.handle_control(conn, malformed).unwrap();
10319 assert_eq!(error[0].header.ty, FrameType::Error);
10320 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
10321
10322 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
10323 let pong = handler.handle_control(conn, ping).unwrap();
10324 assert_eq!(pong[0].header.ty, FrameType::Pong);
10325 assert_eq!(pong[0].header.corr, 4);
10326 }
10327
10328 #[test]
10329 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
10330 let registry = Arc::new(Registry::default());
10331 let handler = ControlHandler::new(Arc::clone(®istry));
10332
10333 handler
10334 .handle_control(
10335 ConnectionId::new(1),
10336 hello_frame("aft", PROTOCOL_VERSION, 1),
10337 )
10338 .unwrap();
10339 let duplicate = handler
10340 .handle_control(
10341 ConnectionId::new(2),
10342 hello_frame("aft", PROTOCOL_VERSION, 2),
10343 )
10344 .unwrap();
10345
10346 assert_eq!(duplicate[0].header.ty, FrameType::Error);
10347 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
10348 let registration = registry.get_module("aft").unwrap().unwrap();
10349 assert_eq!(registration.connection_id, ConnectionId::new(1));
10350 }
10351
10352 #[test]
10353 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
10354 let registry = Arc::new(Registry::default());
10355 let forwarding = Arc::new(ForwardingTable::default());
10356 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
10357 let handler =
10358 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10359 .with_process_liveness(process_liveness);
10360 let (ctx, route_channel, route_epoch) =
10361 bind_liveness_route(®istry, &forwarding, "aft-dead");
10362 let responses = handler
10363 .handle_route_poll(
10364 &ctx,
10365 route_poll_frame(41, PollKind::Liveness, route_channel),
10366 route_channel,
10367 route_epoch,
10368 PollKind::Liveness,
10369 )
10370 .unwrap();
10371
10372 assert_eq!(responses.len(), 1);
10373 assert_eq!(responses[0].header.ty, FrameType::Response);
10374 assert_route_poll_liveness(&responses[0], false);
10375 }
10376
10377 #[test]
10378 fn liveness_poll_without_process_source_uses_bound_route() {
10379 let registry = Arc::new(Registry::default());
10380 let forwarding = Arc::new(ForwardingTable::default());
10381 let handler =
10382 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10383 let (ctx, route_channel, route_epoch) =
10384 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
10385 let responses = handler
10386 .handle_route_poll(
10387 &ctx,
10388 route_poll_frame(42, PollKind::Liveness, route_channel),
10389 route_channel,
10390 route_epoch,
10391 PollKind::Liveness,
10392 )
10393 .unwrap();
10394
10395 assert_route_poll_liveness(&responses[0], true);
10396 }
10397
10398 #[test]
10399 fn liveness_poll_untracked_process_source_uses_bound_route() {
10400 let registry = Arc::new(Registry::default());
10401 let forwarding = Arc::new(ForwardingTable::default());
10402 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
10403 let handler =
10404 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10405 .with_process_liveness(process_liveness);
10406 let (ctx, route_channel, route_epoch) =
10407 bind_liveness_route(®istry, &forwarding, "aft-untracked");
10408 let responses = handler
10409 .handle_route_poll(
10410 &ctx,
10411 route_poll_frame(43, PollKind::Liveness, route_channel),
10412 route_channel,
10413 route_epoch,
10414 PollKind::Liveness,
10415 )
10416 .unwrap();
10417
10418 assert_route_poll_liveness(&responses[0], true);
10419 }
10420
10421 #[tokio::test]
10422 async fn unknown_op_returns_unknown_control_op() {
10423 let handler = ControlHandler::default();
10424 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
10425 let request = Frame::build(
10426 FrameType::Request,
10427 control_flags(),
10428 0,
10429 0,
10430 55,
10431 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
10432 )
10433 .unwrap();
10434
10435 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
10436
10437 assert_eq!(response.len(), 1);
10438 assert_eq!(response[0].header.ty, FrameType::Error);
10439 assert_eq!(response[0].header.corr, 55);
10440 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
10441 }
10442
10443 #[tokio::test]
10444 async fn supervisor_provenance_rejects_unknown_exact_module() {
10445 let handler = ControlHandler::default();
10446 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
10447 let request = Frame::build(
10448 FrameType::Request,
10449 control_flags(),
10450 0,
10451 0,
10452 57,
10453 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
10454 )
10455 .unwrap();
10456
10457 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
10458
10459 assert_eq!(response.len(), 1);
10460 assert_eq!(response[0].header.ty, FrameType::Error);
10461 assert_eq!(response[0].header.corr, 57);
10462 let error = parse_error(&response[0]);
10463 assert_eq!(error["code"], "unknown_module");
10464 assert_eq!(error["message"], "module_id 'missing' is not supervised");
10465 }
10466
10467 #[test]
10468 fn provenance_probe_override_keeps_handler_tests_deterministic() {
10469 let expected = subc_control::RunningImageAgreement::Unavailable {
10470 reason: subc_control::RunningImageUnavailableReason::HashFailed,
10471 };
10472 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
10473 assert_eq!(handler.provenance_probe_override, Some(expected));
10474 }
10475
10476 #[test]
10477 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
10478 let verdict = reload_verdict(
10479 std::path::Path::new("/bin/new"),
10480 Some(std::path::Path::new("/bin/old")),
10481 subc_control::RunningImageAgreement::Unavailable {
10482 reason: subc_control::RunningImageUnavailableReason::HashFailed,
10483 },
10484 );
10485 assert!(matches!(
10486 verdict.path,
10487 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
10488 if configured == std::path::Path::new("/bin/new")
10489 && spawned_from == std::path::Path::new("/bin/old")
10490 ));
10491 }
10492
10493 #[test]
10494 fn reload_verdict_detects_replaced_image_at_same_path() {
10495 let image = subc_control::RunningImageAgreement::Mismatch {
10496 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
10497 digest: "old".into(),
10498 },
10499 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
10500 digest: "new".into(),
10501 },
10502 };
10503 let verdict = reload_verdict(
10504 std::path::Path::new("/bin/same"),
10505 Some(std::path::Path::new("/bin/same")),
10506 image.clone(),
10507 );
10508 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
10509 assert_eq!(verdict.image, image);
10510 }
10511
10512 #[test]
10513 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
10514 let image = subc_control::RunningImageAgreement::Unavailable {
10515 reason: subc_control::RunningImageUnavailableReason::NotRunning,
10516 };
10517 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
10518 assert_eq!(
10519 verdict.path,
10520 subc_control::ReloadPathAgreement::Unavailable {
10521 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
10522 }
10523 );
10524 assert_eq!(verdict.image, image);
10525
10526 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
10527 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
10528 };
10529 let verdict = reload_verdict(
10530 std::path::Path::new("/bin/same"),
10531 Some(std::path::Path::new("/bin/same")),
10532 unconfirmed.clone(),
10533 );
10534 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
10535 assert_eq!(verdict.image, unconfirmed);
10536 }
10537
10538 #[test]
10539 fn reload_verdict_preserves_each_image_unavailability_reason() {
10540 use subc_control::RunningImageUnavailableReason as Reason;
10541
10542 for reason in [
10543 Reason::NotRunning,
10544 Reason::UnsupportedPlatform,
10545 Reason::RunningExecutableUnreadable,
10546 Reason::SpawnedPathUnreadable,
10547 Reason::HashFailed,
10548 Reason::ProcessIdentityUnconfirmed,
10549 Reason::Unknown("future_probe_reason".to_string()),
10550 ] {
10551 let image = subc_control::RunningImageAgreement::Unavailable {
10552 reason: reason.clone(),
10553 };
10554 let verdict = reload_verdict(
10555 std::path::Path::new("/bin/same"),
10556 Some(std::path::Path::new("/bin/same")),
10557 image.clone(),
10558 );
10559 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
10560 assert_eq!(verdict.image, image, "{reason:?}");
10561 }
10562 }
10563
10564 #[tokio::test]
10565 async fn malformed_control_bodies_return_invalid_control_body() {
10566 let handler = ControlHandler::default();
10567 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
10568
10569 for (corr, body) in [
10570 (56, br#"{"route_channel":1}"#.as_slice()),
10571 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
10572 (
10573 58,
10574 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
10575 ),
10576 ] {
10577 let request = Frame::build(
10578 FrameType::Request,
10579 control_flags(),
10580 0,
10581 0,
10582 corr,
10583 body.to_vec(),
10584 )
10585 .unwrap();
10586 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
10587
10588 assert_eq!(response.len(), 1);
10589 assert_eq!(response[0].header.ty, FrameType::Error);
10590 assert_eq!(response[0].header.corr, corr);
10591 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
10592 }
10593 }
10594
10595 #[tokio::test]
10596 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
10597 let registry = Arc::new(Registry::default());
10598 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
10599 let router = Router::with_control_handler(Arc::clone(&control));
10600 let connection = router.begin_connection();
10601 let (ctx, mut rx) = route_ctx(connection.id());
10602
10603 router
10604 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
10605 .await
10606 .unwrap();
10607 let response = rx.recv().await.unwrap();
10608 let ack = parse_ack(&response);
10609 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
10610 let channel = 1;
10611
10612 let goodbye =
10613 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
10614 router.route_for_connection(&ctx, goodbye).await.unwrap();
10615 assert!(rx.try_recv().is_err());
10616 assert!(registry.get_module("aft").unwrap().is_none());
10617
10618 router
10619 .route_for_connection(&ctx, channel_request(channel, 13))
10620 .await
10621 .unwrap();
10622 let error_frame = rx.recv().await.unwrap();
10623 assert_eq!(error_frame.header.ty, FrameType::Error);
10624 assert_eq!(error_frame.header.channel, channel);
10625 }
10626
10627 #[tokio::test]
10628 async fn dropping_router_connection_releases_registration() {
10629 let registry = Arc::new(Registry::default());
10630 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
10631 let router = Router::with_control_handler(control);
10632 let connection = router.begin_connection();
10633 let (ctx, mut rx) = route_ctx(connection.id());
10634
10635 router
10636 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
10637 .await
10638 .unwrap();
10639 let response = rx.recv().await.unwrap();
10640 let ack = parse_ack(&response);
10641 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
10642 assert!(registry.get_module("aft").unwrap().is_some());
10643
10644 drop(connection);
10645
10646 assert!(registry.get_module("aft").unwrap().is_none());
10647 assert_eq!(registry.active_registration_count().unwrap(), 0);
10648 }
10649
10650 fn capability_manifest(
10651 module_id: &str,
10652 provides: &[&str],
10653 must_never_reach: &[&str],
10654 ) -> ModuleManifest {
10655 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
10656 manifest.capabilities = Some(CapabilityDeclarations {
10657 provides: provides
10658 .iter()
10659 .map(|capability| (*capability).to_string())
10660 .collect(),
10661 requires: Vec::new(),
10662 must_never_reach: must_never_reach
10663 .iter()
10664 .map(|capability| (*capability).to_string())
10665 .collect(),
10666 });
10667 manifest
10668 }
10669
10670 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
10671 Frame::build(
10672 FrameType::Hello,
10673 control_flags(),
10674 0,
10675 0,
10676 corr,
10677 serde_json::to_vec(&ModuleHelloBody {
10678 protocol_ver: manifest.protocol_ver,
10679 manifest,
10680 control_ops: None,
10681 launch_nonce: None,
10682 })
10683 .expect("capability test HELLO serializes"),
10684 )
10685 .expect("capability test HELLO frame builds")
10686 }
10687
10688 fn catalog_update_with_capabilities_frame(
10689 corr: u64,
10690 capabilities: CapabilityDeclarations,
10691 ) -> Frame {
10692 Frame::build(
10693 FrameType::Request,
10694 control_flags(),
10695 0,
10696 0,
10697 corr,
10698 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
10699 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
10700 capabilities: Some(capabilities),
10701 ready: None,
10702 })
10703 .expect("capability catalog.update serializes"),
10704 )
10705 .expect("capability catalog.update frame builds")
10706 }
10707
10708 async fn register_capability_manifest(
10709 handler: &ControlHandler,
10710 ctx: &RouteCtx,
10711 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
10712 manifest: ModuleManifest,
10713 corr: u64,
10714 ) {
10715 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
10716 }
10717
10718 async fn open_route_for_capability_test(
10719 handler: &ControlHandler,
10720 target_ctx: &RouteCtx,
10721 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
10722 client_connection_id: u64,
10723 corr: u64,
10724 target_module_id: &str,
10725 consumer_identity: Option<ConsumerIdentity>,
10726 ) -> (
10727 mpsc::Receiver<crate::router::OutboundFrame>,
10728 ModuleControlRequest,
10729 ) {
10730 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
10731 let route_handler = handler.clone();
10732 let target_module_id = target_module_id.to_string();
10733 let route_task = tokio::spawn(async move {
10734 route_handler
10735 .handle_control_frame(
10736 &client_ctx,
10737 route_open_frame_with_admission_facts(
10738 corr,
10739 &target_module_id,
10740 unique_project_root("admission-facts"),
10741 consumer_identity,
10742 None,
10743 ),
10744 )
10745 .await
10746 .expect("capability test route.open succeeds")
10747 });
10748 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
10749 .await
10750 .expect("capability test route.open must reach route.bind")
10751 .expect("target control receiver stays open");
10752 let bind_request: ModuleControlRequest =
10753 serde_json::from_slice(&bind.body).expect("route.bind decodes");
10754 handler
10755 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
10756 .await
10757 .expect("capability test route.bind ACK succeeds");
10758 assert!(route_task.await.expect("route.open task joins").is_empty());
10759 let opened = client_rx
10760 .recv()
10761 .await
10762 .expect("successful route.open publishes a response");
10763 assert!(matches!(
10764 serde_json::from_slice::<ClientControlResponse>(&opened.body),
10765 Ok(ClientControlResponse::RouteOpen { .. })
10766 ));
10767 (client_rx, bind_request)
10768 }
10769
10770 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
10771 assert_eq!(frame.header.ty, FrameType::Push);
10772 assert_eq!(frame.header.channel, 0);
10773 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
10774 .expect("route.closed control push decodes");
10775 let ClientControlPush::RouteClosed { channels, .. } = &push else {
10776 panic!("expected route.closed");
10777 };
10778 assert_eq!(channels.len(), 1, "exactly one violating route closed");
10779 let channels = channels.clone();
10780 assert_eq!(
10781 push,
10782 ClientControlPush::RouteClosed {
10783 module_id: target_module_id.to_string(),
10784 channels,
10785 reason: RouteCloseReason::CapabilityDenied,
10786 drained: false,
10787 abandoned: 0,
10788 excluded_subscriptions: 0,
10789 terminal: Some(false),
10790 }
10791 );
10792 }
10793
10794 #[tokio::test]
10795 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
10796 let registry = Arc::new(Registry::default());
10797 let forwarding = Arc::new(ForwardingTable::default());
10798 let supervisor = SupervisorHandle::new();
10799 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
10800 let handler =
10801 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10802 .with_supervisor(supervisor);
10803 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
10804 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
10805 register_capability_manifest(
10806 &handler,
10807 &target_ctx,
10808 &mut target_rx,
10809 capability_manifest("target", &["credentials-provider/v1"], &[]),
10810 1,
10811 )
10812 .await;
10813 register_capability_manifest(
10814 &handler,
10815 &opener_ctx,
10816 &mut opener_rx,
10817 capability_manifest("opener", &[], &["credentials-provider/v1"]),
10818 2,
10819 )
10820 .await;
10821
10822 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
10823 let replies = handler
10824 .handle_control_frame(
10825 &client_ctx,
10826 route_open_frame_with_admission_facts(
10827 3,
10828 "target",
10829 unique_project_root("admission-facts"),
10830 Some(ConsumerIdentity {
10831 module_id: "opener".to_string(),
10832 launch_nonce: "opener-nonce".to_string(),
10833 }),
10834 None,
10835 ),
10836 )
10837 .await
10838 .expect("denied route.open returns a typed frame");
10839 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
10840 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
10841 assert!(
10842 target_rx.try_recv().is_err(),
10843 "forbidden route.open must not relay route.bind"
10844 );
10845 }
10846
10847 #[tokio::test]
10848 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
10849 let registry = Arc::new(Registry::default());
10850 let forwarding = Arc::new(ForwardingTable::default());
10851 let supervisor = SupervisorHandle::new();
10852 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
10853 let handler =
10854 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10855 .with_supervisor(supervisor);
10856 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
10857 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
10858 register_capability_manifest(
10859 &handler,
10860 &target_ctx,
10861 &mut target_rx,
10862 capability_manifest("target", &["credentials-provider/v1"], &[]),
10863 1,
10864 )
10865 .await;
10866 register_capability_manifest(
10867 &handler,
10868 &old_opener_ctx,
10869 &mut old_opener_rx,
10870 capability_manifest("opener", &[], &[]),
10871 2,
10872 )
10873 .await;
10874 let (mut client_rx, _) = open_route_for_capability_test(
10875 &handler,
10876 &target_ctx,
10877 &mut target_rx,
10878 712,
10879 3,
10880 "target",
10881 Some(ConsumerIdentity {
10882 module_id: "opener".to_string(),
10883 launch_nonce: "opener-nonce".to_string(),
10884 }),
10885 )
10886 .await;
10887 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
10888
10889 handler
10890 .cleanup_connection(old_opener_ctx.connection_id)
10891 .expect("old opener registration cleans up");
10892 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
10893 register_capability_manifest(
10894 &handler,
10895 &new_opener_ctx,
10896 &mut new_opener_rx,
10897 capability_manifest("opener", &[], &["credentials-provider/v1"]),
10898 4,
10899 )
10900 .await;
10901
10902 assert_capability_denied_push(
10903 client_rx
10904 .try_recv()
10905 .expect("HELLO deny addition must emit route.closed")
10906 .frame,
10907 "target",
10908 );
10909 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
10910 assert!(matches!(
10911 target_rx.try_recv(),
10912 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
10913 ));
10914 }
10915
10916 #[tokio::test]
10917 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
10918 let registry = Arc::new(Registry::default());
10919 let forwarding = Arc::new(ForwardingTable::default());
10920 let supervisor = SupervisorHandle::new();
10921 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
10922 let handler =
10923 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10924 .with_supervisor(supervisor);
10925 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
10926 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
10927 register_capability_manifest(
10928 &handler,
10929 &target_ctx,
10930 &mut target_rx,
10931 capability_manifest("target", &[], &[]),
10932 1,
10933 )
10934 .await;
10935 register_capability_manifest(
10936 &handler,
10937 &opener_ctx,
10938 &mut opener_rx,
10939 capability_manifest("opener", &[], &["credentials-provider/v1"]),
10940 2,
10941 )
10942 .await;
10943 let (mut client_rx, _) = open_route_for_capability_test(
10944 &handler,
10945 &target_ctx,
10946 &mut target_rx,
10947 722,
10948 3,
10949 "target",
10950 Some(ConsumerIdentity {
10951 module_id: "opener".to_string(),
10952 launch_nonce: "opener-nonce".to_string(),
10953 }),
10954 )
10955 .await;
10956 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
10957
10958 let replies = handler
10959 .handle_control_frame(
10960 &target_ctx,
10961 catalog_update_with_capabilities_frame(
10962 4,
10963 CapabilityDeclarations {
10964 provides: vec!["credentials-provider/v1".to_string()],
10965 requires: Vec::new(),
10966 must_never_reach: Vec::new(),
10967 },
10968 ),
10969 )
10970 .await
10971 .expect("claim catalog.update succeeds");
10972 assert!(matches!(
10973 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
10974 Ok(ModuleControlResponseToModule::CatalogUpdate {})
10975 ));
10976 assert_capability_denied_push(
10977 client_rx
10978 .try_recv()
10979 .expect("claim addition must emit route.closed")
10980 .frame,
10981 "target",
10982 );
10983 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
10984 assert!(matches!(
10985 target_rx.try_recv(),
10986 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
10987 ));
10988 }
10989
10990 #[tokio::test]
10991 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
10992 let registry = Arc::new(Registry::default());
10993 let forwarding = Arc::new(ForwardingTable::default());
10994 let supervisor = SupervisorHandle::new();
10995 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
10996 let handler =
10997 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10998 .with_supervisor(supervisor);
10999 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
11000 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
11001 register_capability_manifest(
11002 &handler,
11003 &target_ctx,
11004 &mut target_rx,
11005 capability_manifest("target", &["credentials-provider/v1"], &[]),
11006 1,
11007 )
11008 .await;
11009 register_capability_manifest(
11010 &handler,
11011 &opener_ctx,
11012 &mut opener_rx,
11013 capability_manifest("opener", &[], &[]),
11014 2,
11015 )
11016 .await;
11017 let (mut client_rx, _) = open_route_for_capability_test(
11018 &handler,
11019 &target_ctx,
11020 &mut target_rx,
11021 732,
11022 3,
11023 "target",
11024 Some(ConsumerIdentity {
11025 module_id: "opener".to_string(),
11026 launch_nonce: "opener-nonce".to_string(),
11027 }),
11028 )
11029 .await;
11030 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
11031
11032 handler
11033 .handle_control_frame(
11034 &target_ctx,
11035 catalog_update_with_capabilities_frame(
11036 4,
11037 CapabilityDeclarations {
11038 provides: Vec::new(),
11039 requires: Vec::new(),
11040 must_never_reach: Vec::new(),
11041 },
11042 ),
11043 )
11044 .await
11045 .expect("claim removal catalog.update succeeds");
11046 assert_eq!(
11047 forwarding.active_binding_count().unwrap(),
11048 1,
11049 "removing an attested target claim must leave the route census unchanged"
11050 );
11051 assert!(
11052 client_rx.try_recv().is_err(),
11053 "claim removal must not emit route.closed capability_denied"
11054 );
11055 assert!(
11056 target_rx.try_recv().is_err(),
11057 "claim removal must not send the target a route GOODBYE"
11058 );
11059 }
11060
11061 #[tokio::test]
11064 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
11065 let registry = Arc::new(Registry::default());
11066 let forwarding = Arc::new(ForwardingTable::default());
11067 let supervisor = SupervisorHandle::new();
11068 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11069 let handler =
11070 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11071 .with_supervisor(supervisor);
11072 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
11073 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
11074 register_capability_manifest(
11075 &handler,
11076 &target_ctx,
11077 &mut target_rx,
11078 capability_manifest("target", &["credentials-provider/v1"], &[]),
11079 1,
11080 )
11081 .await;
11082 register_capability_manifest(
11083 &handler,
11084 &opener_ctx,
11085 &mut opener_rx,
11086 capability_manifest("opener", &[], &["credentials-provider/v1"]),
11087 2,
11088 )
11089 .await;
11090
11091 let (_client_rx, bind) = open_route_for_capability_test(
11092 &handler,
11093 &target_ctx,
11094 &mut target_rx,
11095 742,
11096 3,
11097 "target",
11098 None,
11099 )
11100 .await;
11101 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
11102 panic!("direct scope-honesty route must bind");
11103 };
11104 assert_eq!(principal, Some(Principal::Direct));
11105 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
11106 }
11107
11108 #[tokio::test]
11111 async fn must_never_reach_self_route_is_capability_forbidden() {
11112 let registry = Arc::new(Registry::default());
11113 let forwarding = Arc::new(ForwardingTable::default());
11114 let supervisor = SupervisorHandle::new();
11115 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
11116 let handler =
11117 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11118 .with_supervisor(supervisor);
11119 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
11120 register_capability_manifest(
11121 &handler,
11122 &self_ctx,
11123 &mut self_rx,
11124 capability_manifest(
11125 "self-provider",
11126 &["credentials-provider/v1"],
11127 &["credentials-provider/v1"],
11128 ),
11129 1,
11130 )
11131 .await;
11132
11133 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
11134 let replies = handler
11135 .handle_control_frame(
11136 &client_ctx,
11137 route_open_frame_with_admission_facts(
11138 2,
11139 "self-provider",
11140 unique_project_root("admission-facts"),
11141 Some(ConsumerIdentity {
11142 module_id: "self-provider".to_string(),
11143 launch_nonce: "self-nonce".to_string(),
11144 }),
11145 None,
11146 ),
11147 )
11148 .await
11149 .expect("self-route refusal returns a typed frame");
11150 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
11151 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
11152 assert!(
11153 self_rx.try_recv().is_err(),
11154 "self denial must not relay route.bind"
11155 );
11156 }
11157
11158 #[test]
11159 fn unsupported_channel_zero_frame_returns_error() {
11160 let handler = ControlHandler::default();
11161 let request = Frame::build(
11162 FrameType::Request,
11163 control_flags(),
11164 0,
11165 0,
11166 21,
11167 b"opaque".to_vec(),
11168 )
11169 .unwrap();
11170
11171 let response = handler
11172 .handle_control(ConnectionId::new(1), request)
11173 .unwrap();
11174
11175 assert_eq!(response[0].header.ty, FrameType::Error);
11176 assert_eq!(
11177 parse_error(&response[0])["code"],
11178 "unsupported_control_frame"
11179 );
11180 }
11181
11182 mod swap {
11187 use super::*;
11188
11189 const INCUMBENT: ConnectionId = ConnectionId::new(30);
11190 const CANDIDATE: ConnectionId = ConnectionId::new(40);
11191
11192 struct Swap {
11193 registry: Arc<Registry>,
11194 forwarding: Arc<ForwardingTable>,
11195 handler: ControlHandler,
11196 incumbent_ctx: RouteCtx,
11197 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
11198 candidate_ctx: RouteCtx,
11199 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
11200 }
11201
11202 async fn swap_with_incumbent() -> Swap {
11203 let registry = Arc::new(Registry::default());
11204 let forwarding = Arc::new(ForwardingTable::default());
11205 let handler =
11206 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11207 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
11208 hello_via_sink(
11209 &handler,
11210 &incumbent_ctx,
11211 &mut incumbent_rx,
11212 hello_frame("aft", PROTOCOL_VERSION, 7),
11213 )
11214 .await;
11215 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
11216 Swap {
11217 registry,
11218 forwarding,
11219 handler,
11220 incumbent_ctx,
11221 incumbent_rx,
11222 candidate_ctx,
11223 candidate_rx,
11224 }
11225 }
11226
11227 fn register_candidate(swap: &Swap, ready: Option<bool>) {
11228 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
11229 candidate_manifest.ready = ready;
11230 let registration = swap
11231 .registry
11232 .register_candidate_with_control_ops(
11233 candidate_manifest,
11234 PROTOCOL_VERSION,
11235 CANDIDATE,
11236 module_baseline_control_ops(),
11237 )
11238 .unwrap();
11239 swap.forwarding
11240 .register_candidate_module_connection(
11241 CANDIDATE,
11242 "aft".to_string(),
11243 PROTOCOL_VERSION,
11244 manifest_concurrency(®istration.manifest),
11245 swap.candidate_ctx.egress.clone(),
11246 )
11247 .unwrap();
11248 }
11249
11250 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
11251 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
11252 swap.registry.promote_candidate("aft").unwrap().unwrap();
11253 cutover.incumbent.unwrap()
11254 }
11255
11256 fn keyed_total(counters: &Value, key: &str) -> u64 {
11257 counters[key]
11258 .as_object()
11259 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
11260 .unwrap_or(0)
11261 }
11262
11263 #[tokio::test]
11269 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
11270 let mut swap = swap_with_incumbent().await;
11271 let handler = swap.handler.clone();
11272
11273 let cotenant = ConnectionId::new(31);
11275 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
11276 let (cotenant_task, cotenant_bind) = relay_route_open(
11277 &handler,
11278 cotenant,
11279 &cotenant_ctx.egress,
11280 &mut swap.incumbent_rx,
11281 100,
11282 "aft",
11283 "swap-cotenant",
11284 )
11285 .await;
11286 handler
11287 .handle_control_frame(
11288 &swap.incumbent_ctx,
11289 route_bind_ack(cotenant_bind.header.corr),
11290 )
11291 .await
11292 .unwrap();
11293 assert!(cotenant_task.await.unwrap().is_empty());
11294 let (cotenant_channel, cotenant_epoch) =
11295 published_route(&cotenant_rx.recv().await.unwrap());
11296
11297 let caller = ConnectionId::new(32);
11299 let (caller_ctx, mut caller_rx) = route_ctx(caller);
11300 let (caller_task, caller_bind) = relay_route_open(
11301 &handler,
11302 caller,
11303 &caller_ctx.egress,
11304 &mut swap.incumbent_rx,
11305 101,
11306 "aft",
11307 "swap-caller",
11308 )
11309 .await;
11310 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
11311
11312 register_candidate(&swap, None);
11313 cutover(&swap);
11314
11315 let ack = handler
11317 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
11318 .await;
11319 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
11320 if module_loop_error.is_some() {
11321 handler.cleanup_connection(INCUMBENT).unwrap();
11324 }
11325
11326 assert!(
11328 cotenant_rx.try_recv().is_err(),
11329 "the co-tenant route on the incumbent was torn down by one late ack: \
11330 {module_loop_error:?}"
11331 );
11332 assert!(matches!(
11333 swap.forwarding
11334 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
11335 .unwrap(),
11336 DataRoute::Client(DataRouteState::Bound(_))
11337 ));
11338 assert_eq!(module_loop_error, None);
11339 assert!(swap
11340 .registry
11341 .get_module_by_connection(INCUMBENT)
11342 .unwrap()
11343 .is_some());
11344
11345 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
11347 .await
11348 .expect("the incumbent is told to drop the abandoned binding")
11349 .unwrap()
11350 .frame;
11351 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11352 assert_eq!(goodbye.header.channel, abandoned_channel);
11353 assert_eq!(goodbye.header.epoch, abandoned_epoch);
11354 assert!(swap.incumbent_rx.try_recv().is_err());
11355
11356 let response = caller_task.await.unwrap();
11358 assert_eq!(response.len(), 1);
11359 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
11360 assert!(caller_rx.try_recv().is_err());
11361
11362 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
11366 let counters = handler.counters().snapshot();
11367 assert_eq!(
11368 keyed_total(&counters, "route_open_accepted_by_principal"),
11369 1
11370 );
11371 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
11372 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
11373 }
11374
11375 #[tokio::test]
11379 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
11380 let mut swap = swap_with_incumbent().await;
11381 register_candidate(&swap, None);
11382 let incumbent = cutover(&swap);
11383 swap.forwarding
11384 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
11385 .unwrap()
11386 .expect("the incumbent is still registered");
11387
11388 let client = ConnectionId::new(33);
11389 let (client_ctx, mut client_rx) = route_ctx(client);
11390 let route_handler = swap.handler.clone();
11391 let open_ctx = RouteCtx {
11392 connection_id: client,
11393 egress: client_ctx.egress.clone(),
11394 };
11395 let mut route_task = tokio::spawn(async move {
11396 route_handler
11397 .handle_control_frame(
11398 &open_ctx,
11399 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
11400 )
11401 .await
11402 .unwrap()
11403 });
11404 let bind = tokio::select! {
11405 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
11406 response = &mut route_task => {
11407 let response = response.unwrap();
11408 panic!(
11409 "post-cutover route.open was refused instead of relayed to the candidate: {}",
11410 parse_error(&response[0])["code"]
11411 );
11412 }
11413 };
11414 swap.handler
11415 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
11416 .await
11417 .unwrap();
11418 assert!(route_task.await.unwrap().is_empty());
11419 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
11420 match swap
11421 .forwarding
11422 .lookup_data_route(client, channel, epoch)
11423 .unwrap()
11424 {
11425 DataRoute::Client(DataRouteState::Bound(route)) => {
11426 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
11427 }
11428 other => panic!("expected a bound route on the candidate, got {other:?}"),
11429 }
11430 assert!(swap.incumbent_rx.try_recv().is_err());
11431 }
11432
11433 #[tokio::test]
11438 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
11439 let swap = swap_with_incumbent().await;
11440 register_candidate(&swap, Some(false));
11441 let update = Frame::build(
11442 FrameType::Request,
11443 control_flags(),
11444 0,
11445 0,
11446 55,
11447 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
11448 provides: manifest("aft", PROTOCOL_VERSION).provides,
11449 capabilities: None,
11450 ready: Some(true),
11451 })
11452 .unwrap(),
11453 )
11454 .unwrap();
11455
11456 let replies = swap
11457 .handler
11458 .handle_control_frame(&swap.candidate_ctx, update)
11459 .await
11460 .unwrap();
11461
11462 assert_eq!(replies.len(), 1);
11463 assert_eq!(
11464 replies[0].header.ty,
11465 FrameType::Response,
11466 "candidate catalog.update was refused: {:?}",
11467 serde_json::from_slice::<Value>(&replies[0].body).ok()
11468 );
11469 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
11470 assert_eq!(
11471 swap.registry
11472 .get_module("aft")
11473 .unwrap()
11474 .unwrap()
11475 .connection_id,
11476 INCUMBENT
11477 );
11478 }
11479 }
11480
11481 mod swap_admission {
11485 use super::*;
11486
11487 const INCUMBENT_NONCE: &str = "incumbent-nonce";
11488 const CANDIDATE_NONCE: &str = "candidate-nonce";
11489
11490 fn handler_with_incumbent(
11491 module_id: &str,
11492 reserved: bool,
11493 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
11494 let registry = Arc::new(Registry::default());
11495 let supervisor = SupervisorHandle::new();
11496 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
11497 if reserved {
11498 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
11499 }
11500 let handler =
11501 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
11502 let incumbent = handler
11503 .handle_control(
11504 ConnectionId::new(1),
11505 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
11506 )
11507 .unwrap();
11508 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
11509 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
11510 (registry, supervisor, handler)
11511 }
11512
11513 #[test]
11519 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
11520 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
11521
11522 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
11523 let replies = handler
11524 .handle_control(
11525 ConnectionId::new(connection),
11526 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
11527 )
11528 .unwrap();
11529 assert_eq!(replies[0].header.ty, FrameType::Error);
11530 assert_eq!(
11531 parse_error(&replies[0])["code"],
11532 "swap_token_invalid",
11533 "nonce {nonce:?}"
11534 );
11535 }
11536 assert!(registry.get_candidate("aft").unwrap().is_none());
11537 assert_eq!(
11538 registry.get_module("aft").unwrap().unwrap().connection_id,
11539 ConnectionId::new(1)
11540 );
11541
11542 let admitted = handler
11545 .handle_control(
11546 ConnectionId::new(4),
11547 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
11548 )
11549 .unwrap();
11550 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
11551 assert_eq!(
11552 registry
11553 .get_candidate("aft")
11554 .unwrap()
11555 .unwrap()
11556 .connection_id,
11557 ConnectionId::new(4)
11558 );
11559 assert_eq!(
11560 registry.get_module("aft").unwrap().unwrap().connection_id,
11561 ConnectionId::new(1),
11562 "the candidate must not take the active slot"
11563 );
11564 let replayed = handler
11565 .handle_control(
11566 ConnectionId::new(5),
11567 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
11568 )
11569 .unwrap();
11570 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
11571
11572 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
11576 let squatter = handler
11577 .handle_control(
11578 ConnectionId::new(6),
11579 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
11580 )
11581 .unwrap();
11582 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
11583 assert!(
11584 registry.get_module("aft").unwrap().is_none(),
11585 "a squatter took the active slot of an id being swapped"
11586 );
11587 }
11588
11589 #[test]
11594 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
11595 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
11596
11597 let replies = handler
11598 .handle_control(
11599 ConnectionId::new(2),
11600 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
11601 )
11602 .unwrap();
11603
11604 assert_eq!(
11605 replies[0].header.ty,
11606 FrameType::HelloAck,
11607 "reserved candidate refused: {:?}",
11608 serde_json::from_slice::<Value>(&replies[0].body).ok()
11609 );
11610 assert_eq!(
11611 registry
11612 .get_candidate("vault")
11613 .unwrap()
11614 .unwrap()
11615 .connection_id,
11616 ConnectionId::new(2)
11617 );
11618 }
11619
11620 #[test]
11623 fn without_an_open_swap_the_ordinary_gates_decide() {
11624 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
11625 supervisor.close_swap("vault");
11626
11627 let candidate = handler
11628 .handle_control(
11629 ConnectionId::new(2),
11630 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
11631 )
11632 .unwrap();
11633 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
11634 let duplicate = handler
11635 .handle_control(
11636 ConnectionId::new(3),
11637 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
11638 )
11639 .unwrap();
11640 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
11641 assert!(registry.get_candidate("vault").unwrap().is_none());
11642 }
11643 }
11644
11645 mod scopes {
11649 use subc_protocol::scope::{
11650 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
11651 ScopeStatus,
11652 };
11653
11654 use super::*;
11655
11656 const OWNER: &str = "prefrontal-core";
11657
11658 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
11659 ScopeRecord {
11660 scope_ref: scope_ref.to_string(),
11661 scope_epoch,
11662 kind: ScopeKind::Head,
11663 parent: None,
11664 child_owners: Vec::new(),
11665 carriers: Vec::new(),
11666 attributes: Default::default(),
11667 }
11668 }
11669
11670 async fn call(
11671 handler: &ControlHandler,
11672 ctx: &RouteCtx,
11673 request: &ModuleControlRequestFromModule,
11674 ) -> Frame {
11675 let body = serde_json::to_vec(request).unwrap();
11676 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
11677 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
11678 assert_eq!(replies.len(), 1, "{replies:?}");
11679 replies.pop().unwrap()
11680 }
11681
11682 async fn sync(
11683 handler: &ControlHandler,
11684 ctx: &RouteCtx,
11685 generation: u64,
11686 scopes: Vec<ScopeRecord>,
11687 ) -> Result<ModuleControlResponseToModule, String> {
11688 let reply = call(
11689 handler,
11690 ctx,
11691 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
11692 )
11693 .await;
11694 match reply.header.ty {
11695 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
11696 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
11697 }
11698 }
11699
11700 async fn describe(
11701 handler: &ControlHandler,
11702 ctx: &RouteCtx,
11703 owner: &str,
11704 scope_ref: &str,
11705 ) -> ModuleControlResponseToModule {
11706 let reply = call(
11707 handler,
11708 ctx,
11709 &ModuleControlRequestFromModule::ScopeDescribe {
11710 owner: Principal::Reserved {
11711 module_id: owner.to_string(),
11712 },
11713 scope_ref: scope_ref.to_string(),
11714 },
11715 )
11716 .await;
11717 assert_eq!(
11718 reply.header.ty,
11719 FrameType::Response,
11720 "{:?}",
11721 parse_error(&reply)
11722 );
11723 serde_json::from_slice(&reply.body).unwrap()
11724 }
11725
11726 async fn module(
11728 handler: &ControlHandler,
11729 connection: u64,
11730 module_id: &str,
11731 nonce: Option<&str>,
11732 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
11733 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
11734 hello_via_sink(
11735 handler,
11736 &ctx,
11737 &mut rx,
11738 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
11739 )
11740 .await;
11741 (ctx, rx)
11742 }
11743
11744 #[tokio::test]
11747 async fn a_client_connection_cannot_sync_or_describe() {
11748 let handler = ControlHandler::new(Arc::new(Registry::default()));
11749 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
11750 for request in [
11751 ModuleControlRequestFromModule::ScopeSync {
11752 generation: 1,
11753 scopes: vec![head("s", 1)],
11754 },
11755 ModuleControlRequestFromModule::ScopeDescribe {
11756 owner: Principal::Direct,
11757 scope_ref: "s".to_string(),
11758 },
11759 ] {
11760 let reply = call(&handler, &ctx, &request).await;
11761 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
11762 }
11763 assert!(
11764 !handler
11765 .scopes
11766 .read()
11767 .unwrap()
11768 .describe(
11769 &Principal::Reserved {
11770 module_id: OWNER.to_string()
11771 },
11772 "s"
11773 )
11774 .owner_synced
11775 );
11776 }
11777
11778 #[tokio::test]
11781 async fn a_module_without_a_supervised_launch_cannot_sync() {
11782 let handler = ControlHandler::new(Arc::new(Registry::default()));
11783 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
11784 assert_eq!(
11785 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
11786 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
11787 );
11788 }
11789
11790 #[tokio::test]
11791 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
11792 let supervisor = SupervisorHandle::new();
11793 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
11794 let handler = ControlHandler::new(Arc::new(Registry::default()))
11795 .with_supervisor(supervisor.clone());
11796 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
11797 sync(&handler, &incumbent, 1, vec![head("s", 1)])
11798 .await
11799 .expect("the current launch syncs");
11800
11801 supervisor.open_swap(OWNER, "n2".to_string());
11804 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
11805 assert_eq!(
11806 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
11807 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
11808 );
11809 sync(&handler, &incumbent, 2, vec![head("s", 1)])
11810 .await
11811 .expect("the serving owner syncs during the swap");
11812
11813 supervisor.close_swap(OWNER);
11816 assert_eq!(
11817 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
11818 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
11819 );
11820 sync(&handler, &incumbent, 3, vec![head("s", 1)])
11821 .await
11822 .expect("the serving owner syncs after the rollback");
11823 handler.cleanup_connection(candidate.connection_id).unwrap();
11824
11825 supervisor.open_swap(OWNER, "n3".to_string());
11830 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
11831 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
11832 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
11833 .await
11834 .expect("the promoted launch takes authority");
11835 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
11836 panic!("unexpected reply {reply:?}");
11837 };
11838 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
11839 assert_eq!(
11840 sync(&handler, &incumbent, 4, Vec::new()).await,
11841 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
11842 );
11843 }
11844
11845 #[tokio::test]
11848 async fn closing_the_authority_connection_frees_sync_authority() {
11849 let supervisor = SupervisorHandle::new();
11850 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
11851 let handler = ControlHandler::new(Arc::new(Registry::default()))
11852 .with_supervisor(supervisor.clone());
11853 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
11854 sync(&handler, &first, 10, vec![head("s", 1)])
11855 .await
11856 .unwrap();
11857 handler.cleanup_connection(first.connection_id).unwrap();
11858
11859 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
11860 sync(&handler, &second, 1, vec![head("s", 1)])
11861 .await
11862 .expect("the next connection takes the released authority");
11863 }
11864
11865 #[tokio::test]
11866 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
11867 let registry = Arc::new(Registry::default());
11868 let supervisor_handle = SupervisorHandle::new();
11869 let supervisor = Supervisor::new(Arc::clone(®istry), RestartPolicy::default())
11870 .with_handle(supervisor_handle.clone())
11871 .with_daemon_incarnation("incarnation-7".to_string());
11872 supervisor
11875 .supervise_configured(
11876 ModuleSpec {
11877 module_id: OWNER.to_string(),
11878 program: PathBuf::from("/nonexistent/prefrontal-core"),
11879 args: Vec::new(),
11880 env: Vec::new(),
11881 reserved: false,
11882 reserved_prefixes: Vec::new(),
11883 protocol: ModuleProtocol::Subc,
11884 overlap: Default::default(),
11885 },
11886 false,
11887 )
11888 .unwrap();
11889 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
11890 let handler =
11891 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11892 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
11893
11894 let ModuleControlResponseToModule::ScopeDescribe {
11896 status,
11897 daemon_incarnation,
11898 owner_synced,
11899 owner_configured,
11900 scope,
11901 ..
11902 } = describe(&handler, &reader, OWNER, "s").await
11903 else {
11904 panic!("not a describe reply");
11905 };
11906 assert_eq!(status, ScopeStatus::NotLive);
11907 assert_eq!(daemon_incarnation, "incarnation-7");
11908 assert!(!owner_synced);
11909 assert!(owner_configured);
11910 assert!(scope.is_none());
11911
11912 let ModuleControlResponseToModule::ScopeDescribe {
11915 status,
11916 owner_configured,
11917 ..
11918 } = describe(&handler, &reader, "ghost", "s").await
11919 else {
11920 panic!("not a describe reply");
11921 };
11922 assert_eq!(status, ScopeStatus::NotLive);
11923 assert!(!owner_configured);
11924
11925 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
11927 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
11928 let ModuleControlResponseToModule::ScopeDescribe {
11929 status,
11930 scope_epoch,
11931 owner_synced,
11932 scope,
11933 ..
11934 } = describe(&handler, &reader, OWNER, "s").await
11935 else {
11936 panic!("not a describe reply");
11937 };
11938 assert_eq!(status, ScopeStatus::Live);
11939 assert_eq!(scope_epoch, Some(4));
11940 assert!(owner_synced);
11941 let stamp = scope.expect("a live scope carries its stamp");
11942 assert!(
11943 stamp.owner_authorized,
11944 "prefrontal-core is the default authority"
11945 );
11946 assert_eq!(stamp.kind, ScopeKind::Head);
11947 }
11948
11949 #[tokio::test]
11950 async fn scope_authority_owners_decides_owner_authorized() {
11951 let supervisor = SupervisorHandle::new();
11952 supervisor.set_spawn_nonce("broca", "b1".to_string());
11953 let handler = ControlHandler::new(Arc::new(Registry::default()))
11954 .with_supervisor(supervisor)
11955 .with_scope_authority_owners(vec!["broca".to_string()]);
11956 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
11957 let mut gated = head("s", 1);
11958 gated.attributes.agent_id = Some("agent".to_string());
11959 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
11960 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
11961 describe(&handler, &broca, "broca", "s").await
11962 else {
11963 panic!("not a describe reply");
11964 };
11965 assert!(scope.unwrap().owner_authorized);
11966 }
11967
11968 #[tokio::test]
11972 async fn scope_ops_and_the_scopes_capability_are_advertised() {
11973 let handler = ControlHandler::new(Arc::new(Registry::default()));
11974 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
11975 let ack = hello_via_sink(
11976 &handler,
11977 &ctx,
11978 &mut rx,
11979 hello_frame("m", PROTOCOL_VERSION, 1),
11980 )
11981 .await;
11982 let ack = parse_ack(&ack);
11983 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
11984 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
11985 }
11986 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
11987
11988 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
11989 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11990 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11991 let reply = handler
11992 .handle_control_frame(&client, frame)
11993 .await
11994 .unwrap()
11995 .pop()
11996 .unwrap();
11997 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11998 serde_json::from_slice(&reply.body).unwrap()
11999 else {
12000 panic!("not a server.describe reply");
12001 };
12002 assert!(
12003 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
12004 "{capabilities:?}"
12005 );
12006 }
12007
12008 const PLEXUS: &str = "plexus";
12011 const OTHER: &str = "other";
12012 const AFT: &str = "aft";
12013 const BROCA: &str = "broca";
12014 const MAGIC: &str = "magic-context";
12015
12016 fn nonce(module_id: &str) -> String {
12017 format!("nonce-{module_id}")
12018 }
12019
12020 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
12021 let (tx, rx) = mpsc::channel(64);
12022 (
12023 RouteCtx {
12024 connection_id: ConnectionId::new(connection),
12025 egress: FrameSink::new(tx),
12026 },
12027 rx,
12028 )
12029 }
12030
12031 struct Rig {
12035 handler: ControlHandler,
12036 forwarding: Arc<ForwardingTable>,
12037 owner: RouteCtx,
12038 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12039 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
12040 generation: u64,
12041 next_connection: u64,
12042 _supervisor: Supervisor,
12043 }
12044
12045 async fn rig() -> Rig {
12046 let registry = Arc::new(Registry::default());
12047 let forwarding = Arc::new(ForwardingTable::default());
12048 let supervisor_handle = SupervisorHandle::new();
12049 let supervisor = Supervisor::new(Arc::clone(®istry), RestartPolicy::default())
12050 .with_handle(supervisor_handle.clone());
12051 supervisor
12052 .supervise_configured(
12053 ModuleSpec {
12054 module_id: OWNER.to_string(),
12055 program: PathBuf::from("/nonexistent/prefrontal-core"),
12056 args: Vec::new(),
12057 env: Vec::new(),
12058 reserved: false,
12059 reserved_prefixes: Vec::new(),
12060 protocol: ModuleProtocol::Subc,
12061 overlap: Default::default(),
12062 },
12063 false,
12064 )
12065 .unwrap();
12066 for module_id in [OWNER, AFT, BROCA, MAGIC] {
12067 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
12068 }
12069 let handler =
12070 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12071 .with_supervisor(supervisor_handle);
12072 let (owner, mut owner_rx) = wide_ctx(1);
12073 hello_via_sink(
12074 &handler,
12075 &owner,
12076 &mut owner_rx,
12077 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
12078 )
12079 .await;
12080 let mut modules = BTreeMap::new();
12081 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
12082 let (ctx, mut rx) = wide_ctx(connection);
12083 hello_via_sink(
12084 &handler,
12085 &ctx,
12086 &mut rx,
12087 hello_frame(module_id, PROTOCOL_VERSION, connection),
12088 )
12089 .await;
12090 modules.insert(module_id.to_string(), (ctx, rx));
12091 }
12092 Rig {
12093 handler,
12094 forwarding,
12095 owner,
12096 _owner_rx: owner_rx,
12097 modules,
12098 generation: 0,
12099 next_connection: 100,
12100 _supervisor: supervisor,
12101 }
12102 }
12103
12104 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
12105 ScopeCarrier {
12106 principal: Principal::Reserved {
12107 module_id: module_id.to_string(),
12108 },
12109 targets: targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()),
12110 }
12111 }
12112
12113 fn session(scope_epoch: u64) -> ScopeRecord {
12116 let mut record = head("s", scope_epoch);
12117 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
12118 record.attributes.agent_id = Some("agent-1".to_string());
12119 record.attributes.delegates = true;
12120 record
12121 }
12122
12123 impl Rig {
12124 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
12125 self.generation += 1;
12126 sync(&self.handler, &self.owner, self.generation, scopes)
12127 .await
12128 .expect("the owner's sync is accepted");
12129 }
12130
12131 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
12132 ScopeSelector {
12133 owner: Principal::Reserved {
12134 module_id: OWNER.to_string(),
12135 },
12136 scope_ref: scope_ref.to_string(),
12137 scope_epoch,
12138 }
12139 }
12140
12141 fn open_frame(
12142 &mut self,
12143 opener: Option<&str>,
12144 target: &str,
12145 scope: Option<ScopeSelector>,
12146 ) -> (
12147 RouteCtx,
12148 mpsc::Receiver<crate::router::OutboundFrame>,
12149 Frame,
12150 ) {
12151 self.next_connection += 1;
12152 let (ctx, rx) = wide_ctx(self.next_connection);
12153 let root = unique_project_root("scoped-open");
12154 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
12155 target: RouteTarget::ToolProvider {
12156 module_id: target.to_string(),
12157 },
12158 identity: BindIdentity::new(
12159 root.path().to_path_buf(),
12160 "unit".to_string(),
12161 "session".to_string(),
12162 ),
12163 consumer_identity: opener.map(|module_id| ConsumerIdentity {
12164 module_id: module_id.to_string(),
12165 launch_nonce: nonce(module_id),
12166 }),
12167 consumer_capabilities: None,
12168 role_versions: None,
12169 admission_facts: None,
12170 scope,
12171 })
12172 .unwrap();
12173 let frame = Frame::build(
12174 FrameType::Request,
12175 control_flags(),
12176 0,
12177 0,
12178 self.next_connection,
12179 body,
12180 )
12181 .unwrap();
12182 (ctx, rx, frame)
12183 }
12184
12185 async fn refused(
12187 &mut self,
12188 opener: Option<&str>,
12189 target: &str,
12190 scope: Option<ScopeSelector>,
12191 ) -> String {
12192 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
12193 let replies = self
12194 .handler
12195 .handle_control_frame(&ctx, frame)
12196 .await
12197 .unwrap();
12198 assert_eq!(replies.len(), 1, "{replies:?}");
12199 assert_eq!(replies[0].header.ty, FrameType::Error);
12200 let (_, module_rx) = self.modules.get_mut(target).unwrap();
12201 assert!(
12202 module_rx.try_recv().is_err(),
12203 "a refused open relays nothing"
12204 );
12205 parse_error(&replies[0])["code"]
12206 .as_str()
12207 .unwrap()
12208 .to_string()
12209 }
12210
12211 async fn relayed(
12213 &mut self,
12214 opener: Option<&str>,
12215 target: &str,
12216 scope: Option<ScopeSelector>,
12217 ) -> Relayed {
12218 let (ctx, rx, frame) = self.open_frame(opener, target, scope);
12219 let handler = self.handler.clone();
12220 let task_ctx = ctx.clone();
12221 let task = tokio::spawn(async move {
12222 handler
12223 .handle_control_frame(&task_ctx, frame)
12224 .await
12225 .unwrap()
12226 });
12227 let (_, module_rx) = self.modules.get_mut(target).unwrap();
12228 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
12229 .await
12230 .expect("the target receives the relayed route.bind")
12231 .unwrap()
12232 .frame;
12233 Relayed {
12234 target: target.to_string(),
12235 client: ctx,
12236 client_rx: rx,
12237 task,
12238 bind,
12239 }
12240 }
12241
12242 async fn ack(&self, relayed: &Relayed) {
12243 let (module, _) = &self.modules[&relayed.target];
12244 self.handler
12245 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
12246 .await
12247 .unwrap();
12248 }
12249
12250 async fn bound(
12252 &mut self,
12253 opener: Option<&str>,
12254 target: &str,
12255 scope: Option<ScopeSelector>,
12256 ) -> Bound {
12257 let relayed = self.relayed(opener, target, scope).await;
12258 self.ack(&relayed).await;
12259 let Relayed {
12260 target,
12261 client,
12262 mut client_rx,
12263 task,
12264 bind,
12265 } = relayed;
12266 assert!(
12267 task.await.unwrap().is_empty(),
12268 "the open is answered by commit"
12269 );
12270 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
12271 Bound {
12272 target,
12273 client,
12274 client_rx,
12275 channel,
12276 epoch,
12277 bind,
12278 }
12279 }
12280
12281 fn live(&self, route: &Bound) -> bool {
12282 matches!(
12283 self.forwarding
12284 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
12285 .unwrap(),
12286 DataRoute::Client(DataRouteState::Bound(_))
12287 )
12288 }
12289 }
12290
12291 struct Relayed {
12292 target: String,
12293 client: RouteCtx,
12294 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12295 task: tokio::task::JoinHandle<Vec<Frame>>,
12296 bind: Frame,
12297 }
12298
12299 struct Bound {
12300 target: String,
12301 client: RouteCtx,
12302 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12303 channel: u16,
12304 epoch: u32,
12305 bind: Frame,
12306 }
12307
12308 impl Bound {
12309 fn closed_reason(&mut self) -> RouteCloseReason {
12312 let mut reason = None;
12313 let mut goodbye = false;
12314 while let Ok(outbound) = self.client_rx.try_recv() {
12315 let frame = outbound.frame;
12316 match frame.header.ty {
12317 FrameType::Goodbye => {
12318 assert_eq!(
12319 (frame.header.channel, frame.header.epoch),
12320 (self.channel, self.epoch)
12321 );
12322 goodbye = true;
12323 }
12324 FrameType::Push => {
12325 let ClientControlPush::RouteClosed {
12326 reason: r,
12327 module_id,
12328 ..
12329 } = serde_json::from_slice(&frame.body).unwrap()
12330 else {
12331 panic!("unexpected push");
12332 };
12333 assert_eq!(module_id, self.target);
12334 reason = Some(r);
12335 }
12336 other => panic!("unexpected frame {other:?}"),
12337 }
12338 }
12339 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
12340 reason.expect("the client is told why the route closed")
12341 }
12342
12343 fn untouched(&mut self) -> bool {
12344 self.client_rx.try_recv().is_err()
12345 }
12346
12347 fn stamp(&self) -> Option<ScopeStamp> {
12348 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
12349 ModuleControlRequest::RouteBind { scope, .. } => scope,
12350 other => panic!("expected a route.bind, got {other:?}"),
12351 }
12352 }
12353 }
12354
12355 #[tokio::test]
12356 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
12357 ) {
12358 let mut rig = rig().await;
12359 let mut record = session(1);
12360 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
12361 record.child_owners = vec![Principal::Reserved {
12362 module_id: MAGIC.to_string(),
12363 }];
12364 rig.sync(vec![record]).await;
12365 let scope = || Some(rig_selector("s", Some(1)));
12366
12367 rig.bound(Some(OWNER), PLEXUS, scope()).await;
12370 rig.bound(Some(AFT), OTHER, scope()).await;
12371 rig.bound(Some(BROCA), PLEXUS, scope()).await;
12372
12373 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
12377 assert_eq!(
12378 rig.refused(opener, target, scope()).await,
12379 error_codes::SCOPE_NOT_CARRIER,
12380 "{opener:?} -> {target}"
12381 );
12382 }
12383 }
12384
12385 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
12386 ScopeSelector {
12387 owner: Principal::Reserved {
12388 module_id: OWNER.to_string(),
12389 },
12390 scope_ref: scope_ref.to_string(),
12391 scope_epoch,
12392 }
12393 }
12394
12395 #[tokio::test]
12396 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
12397 let mut rig = rig().await;
12398 rig.sync(vec![session(1)]).await;
12399 for opener in [OWNER, AFT] {
12400 assert_eq!(
12401 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
12402 .await,
12403 error_codes::SCOPE_EPOCH_REQUIRED,
12404 "{opener}"
12405 );
12406 }
12407 }
12408
12409 #[tokio::test]
12410 async fn admission_separates_not_synced_not_live_and_ended() {
12411 let mut rig = rig().await;
12412 let code = rig
12414 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12415 .await;
12416 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
12417 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
12418
12419 let ghost = ScopeSelector {
12421 owner: Principal::Reserved {
12422 module_id: "ghost".to_string(),
12423 },
12424 scope_ref: "s".to_string(),
12425 scope_epoch: Some(1),
12426 };
12427 assert_eq!(
12428 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
12429 error_codes::SCOPE_NOT_LIVE
12430 );
12431
12432 rig.sync(vec![session(2)]).await;
12433 assert_eq!(
12434 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
12435 .await,
12436 error_codes::SCOPE_NOT_LIVE
12437 );
12438 for epoch in [1, 3] {
12439 assert_eq!(
12440 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
12441 .await,
12442 error_codes::SCOPE_ENDED,
12443 "epoch {epoch}"
12444 );
12445 }
12446 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
12448 .await;
12449 }
12450
12451 #[tokio::test]
12452 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
12453 let mut rig = rig().await;
12454 rig.sync(vec![session(1)]).await;
12455 let route = rig
12456 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12457 .await;
12458 let stamp = route.stamp().expect("a scoped bind carries the stamp");
12459 assert_eq!(stamp.scope_ref, "s");
12460 assert_eq!(stamp.scope_epoch, 1);
12461 assert_eq!(stamp.kind, ScopeKind::Head);
12462 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
12463 assert!(stamp.attributes.delegates);
12464 assert!(stamp.owner_authorized);
12465 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
12466 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
12467
12468 let (broca, mut broca_rx) = wide_ctx(50);
12471 hello_via_sink(
12472 &rig.handler,
12473 &broca,
12474 &mut broca_rx,
12475 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
12476 )
12477 .await;
12478 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
12479 .await
12480 .unwrap();
12481 let own = ScopeSelector {
12482 owner: Principal::Reserved {
12483 module_id: BROCA.to_string(),
12484 },
12485 scope_ref: "b".to_string(),
12486 scope_epoch: Some(1),
12487 };
12488 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
12489 assert!(!route.stamp().unwrap().owner_authorized);
12490 }
12491
12492 #[tokio::test]
12497 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
12498 let mut rig = rig().await;
12499 rig.sync(vec![session(1)]).await;
12500 let mut cotenant = rig
12501 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
12502 .await;
12503
12504 let mut changed = session(1);
12505 changed.child_owners.push(Principal::Reserved {
12506 module_id: MAGIC.to_string(),
12507 });
12508 let mut ended = None;
12509 for (code, next) in [
12510 (error_codes::SCOPE_CHANGED, vec![changed]),
12511 (error_codes::SCOPE_ENDED, Vec::new()),
12512 ] {
12513 let relayed = rig
12514 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12515 .await;
12516 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
12517 ended = Some(next.is_empty());
12518 rig.sync(next).await;
12519 rig.ack(&relayed).await;
12520 let replies = relayed.task.await.unwrap();
12521 assert_eq!(replies.len(), 1, "{replies:?}");
12522 assert_eq!(parse_error(&replies[0])["code"], code);
12523 assert_eq!(
12524 subc_protocol::error_codes::is_retryable_route_open(code),
12525 code == error_codes::SCOPE_CHANGED
12526 );
12527 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
12528 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
12530 let mut goodbyes = Vec::new();
12533 while let Ok(outbound) = plexus_rx.try_recv() {
12534 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
12535 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
12536 }
12537 assert!(
12538 goodbyes.contains(&(bind_channel, bind_epoch)),
12539 "{goodbyes:?}"
12540 );
12541 assert!(rig
12542 .handler
12543 .registry
12544 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
12545 .unwrap()
12546 .is_some());
12547 }
12548 assert_eq!(ended, Some(true));
12549 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
12552 }
12553
12554 #[tokio::test]
12557 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
12558 struct Case {
12559 name: &'static str,
12560 change: fn(&mut ScopeRecord),
12561 closed: [Option<RouteCloseReason>; 4],
12564 }
12565 use RouteCloseReason::*;
12566 let cases = [
12567 Case {
12568 name: "a carrier entry removed",
12569 change: |r| {
12570 r.carriers.retain(|c| {
12571 c.principal
12572 != Principal::Reserved {
12573 module_id: AFT.to_string(),
12574 }
12575 })
12576 },
12577 closed: [None, Some(ScopeCarrierRemoved), None, None],
12578 },
12579 Case {
12580 name: "a target removed from a carrier",
12581 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
12582 closed: [None, None, None, Some(ScopeCarrierRemoved)],
12583 },
12584 Case {
12585 name: "a bare carrier narrowed to targets",
12586 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
12587 closed: [None, Some(ScopeCarrierRemoved), None, None],
12588 },
12589 Case {
12590 name: "delegates turned off",
12591 change: |r| r.attributes.delegates = false,
12592 closed: [Some(ScopeDelegationChanged); 4],
12593 },
12594 Case {
12595 name: "agent_id changed",
12596 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
12597 closed: [Some(ScopeDelegationChanged); 4],
12598 },
12599 Case {
12600 name: "a carrier added, child owners changed, the record re-sent",
12601 change: |r| {
12602 r.carriers.push(carrier(MAGIC, None));
12603 r.child_owners.push(Principal::Reserved {
12604 module_id: MAGIC.to_string(),
12605 });
12606 },
12607 closed: [None; 4],
12608 },
12609 Case {
12610 name: "a target added",
12611 change: |r| {
12612 r.carriers[1]
12613 .targets
12614 .as_mut()
12615 .unwrap()
12616 .push("third".to_string())
12617 },
12618 closed: [None; 4],
12619 },
12620 Case {
12621 name: "delegates turned on",
12622 change: |r| r.attributes.delegates = true,
12623 closed: [None; 4],
12624 },
12625 ];
12626 for case in cases {
12627 let mut rig = rig().await;
12628 rig.sync(vec![session(1)]).await;
12629 let scope = || Some(rig_selector("s", Some(1)));
12630 let mut routes = [
12631 rig.bound(Some(OWNER), PLEXUS, scope()).await,
12632 rig.bound(Some(AFT), PLEXUS, scope()).await,
12633 rig.bound(Some(BROCA), PLEXUS, scope()).await,
12634 rig.bound(Some(BROCA), OTHER, scope()).await,
12635 ];
12636 let mut record = session(1);
12637 (case.change)(&mut record);
12638 rig.sync(vec![record]).await;
12639 for (index, expected) in case.closed.iter().enumerate() {
12640 let route = &mut routes[index];
12641 match expected {
12642 Some(reason) => {
12643 assert!(!rig.live(route), "{}: route {index} still live", case.name);
12644 assert_eq!(
12645 route.closed_reason(),
12646 *reason,
12647 "{}: route {index}",
12648 case.name
12649 );
12650 }
12651 None => {
12652 assert!(rig.live(route), "{}: route {index} closed", case.name);
12653 assert!(
12654 route.untouched(),
12655 "{}: route {index} was told something",
12656 case.name
12657 );
12658 }
12659 }
12660 }
12661 }
12662 }
12663
12664 #[tokio::test]
12665 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
12666 for next in [Vec::new(), vec![session(2)]] {
12668 let mut rig = rig().await;
12669 rig.sync(vec![session(1)]).await;
12670 let mut route = rig
12671 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12672 .await;
12673 rig.sync(next).await;
12674 assert!(!rig.live(&route));
12675 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
12676 }
12677
12678 let mut rig = rig().await;
12681 let mut child = session(1);
12682 child.scope_ref = "child".to_string();
12683 child.kind = ScopeKind::Worker;
12684 child.parent = Some(ScopeParent {
12685 owner: Principal::Reserved {
12686 module_id: OWNER.to_string(),
12687 },
12688 scope_ref: "s".to_string(),
12689 scope_epoch: 1,
12690 });
12691 rig.sync(vec![session(1), child.clone()]).await;
12692 let mut child_route = rig
12693 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
12694 .await;
12695 assert_eq!(
12696 child_route.stamp().unwrap().parent_state,
12697 Some(ParentState::Linked)
12698 );
12699 rig.sync(vec![child]).await;
12700 assert!(!rig.live(&child_route));
12701 assert_eq!(
12702 child_route.closed_reason(),
12703 RouteCloseReason::ScopeParentEnded
12704 );
12705 }
12706
12707 #[tokio::test]
12708 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
12709 ) {
12710 let mut rig = rig().await;
12711 rig.sync(vec![session(1)]).await;
12712 let mut route = rig
12713 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12714 .await;
12715 let before = rig.forwarding.published_scope_tag(OWNER, "s");
12716 rig.sync(vec![session(1)]).await;
12717 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
12718 assert!(rig.live(&route) && route.untouched());
12719
12720 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
12724 .forwarding
12725 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
12726 .unwrap()
12727 else {
12728 panic!("the route is bound");
12729 };
12730 binding.flow.acquire_tagged(9, false).await.unwrap();
12731 let mut widened = session(1);
12732 widened.carriers.push(carrier(MAGIC, None));
12733 rig.sync(vec![widened]).await;
12734 assert!(rig.live(&route) && route.untouched());
12735 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
12736 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
12737 assert_eq!(binding.flow.in_flight(), 1);
12740 binding
12741 .flow
12742 .acquire_tagged(10, false)
12743 .await
12744 .expect("the flow is still open");
12745 }
12746
12747 #[tokio::test]
12750 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
12751 let mut rig = rig().await;
12752 rig.sync(vec![session(1)]).await;
12753 let mut on_incumbent = rig
12754 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12755 .await;
12756
12757 let (candidate, _candidate_rx) = wide_ctx(9);
12760 let registration = rig
12761 .handler
12762 .registry
12763 .register_candidate_with_control_ops(
12764 manifest(PLEXUS, PROTOCOL_VERSION),
12765 PROTOCOL_VERSION,
12766 candidate.connection_id,
12767 module_baseline_control_ops(),
12768 )
12769 .unwrap();
12770 rig.forwarding
12771 .register_candidate_module_connection(
12772 candidate.connection_id,
12773 PLEXUS.to_string(),
12774 PROTOCOL_VERSION,
12775 manifest_concurrency(®istration.manifest),
12776 candidate.egress.clone(),
12777 )
12778 .unwrap();
12779 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
12780 rig.handler
12781 .registry
12782 .promote_candidate(PLEXUS)
12783 .unwrap()
12784 .unwrap();
12785 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
12786
12787 rig.sync(Vec::new()).await;
12788 assert!(!rig.live(&on_incumbent));
12789 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
12790 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
12791 let goodbye = incumbent_rx
12792 .try_recv()
12793 .expect("the superseded endpoint is told")
12794 .frame;
12795 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
12796 }
12797 }
12798}
12799
12800#[cfg(test)]
12801mod concurrency_default_exposure_tests {
12802 use super::*;
12803
12804 fn hello_body(role_json: &str) -> Vec<u8> {
12805 format!(
12806 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
12807 )
12808 .into_bytes()
12809 }
12810
12811 fn manifest_from(body: &[u8]) -> ModuleManifest {
12812 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
12813 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
12814 .expect("manifest parses")
12815 }
12816
12817 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
12818
12819 #[test]
12820 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
12821 let body = hello_body(&format!(
12822 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
12823 ));
12824 let manifest = manifest_from(&body);
12825 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
12828 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
12829 }
12830
12831 #[test]
12832 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
12833 let body = hello_body(&format!(
12834 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
12835 ));
12836 let manifest = manifest_from(&body);
12837 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
12838 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
12839 }
12840
12841 #[test]
12842 fn non_management_roles_are_never_reported() {
12843 let body = hello_body(
12844 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
12845 );
12846 let manifest = manifest_from(&body);
12847 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
12848 }
12849}