1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 path::{Path, PathBuf},
5 sync::{Arc, Mutex, RwLock},
6 time::{Duration, Instant as StdInstant},
7};
8
9use serde::{Deserialize, Serialize};
10use subc_control::{
11 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
12 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
13 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
14 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
15 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
16 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
17 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
18};
19use subc_protocol::{
20 error_codes,
21 manifest::{
22 validate_hello_capability_grammar, validate_hello_self_signal_declarations,
23 CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
24 ProviderRole,
25 },
26 scope::{
27 ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
28 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_DESCRIBE_OP, SCOPE_SYNC_OP,
29 },
30 session::{
31 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
32 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
33 MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE, ROLE_VERSIONS_FIELD,
34 },
35 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
36 Priority, RouteTarget, PROTOCOL_VERSION,
37};
38use tokio::time::{timeout_at, Instant};
39use tracing::{debug, info, warn};
40
41use crate::{
42 capability_requirements::{
43 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
44 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
45 RuntimeModule,
46 },
47 daemon_config::RestartRequiredSection,
48 forwarding::{
49 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
50 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
51 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
52 },
53 observability::{
54 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
55 },
56 provenance::{
57 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
58 },
59 registry::{ChannelState, ConnectionId, Registry, RegistryError},
60 router::{RouteCtx, RouterError},
61 scopes::{BoundScope, HelloLaunchNonces, ScopeTable},
62 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
63 stderr_tail::{CaptureState, TailEntry},
64 supervise::{
65 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
66 SupervisorHandle, SwapHelloAdmission,
67 },
68 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
69};
70
71pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
77
78const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
79const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
80const CAP_PING_PONG: &str = "ping_pong_v1";
81const CAP_SESSION_ATTACH: &str = "session_attach_v1";
82const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
83
84const SUBC_CONTROL_OPS: &[&str] = &[
85 ops::SERVER_DESCRIBE,
86 ops::CATALOG_LIST,
87 ops::ROUTE_OPEN,
88 ops::ROUTE_POLL,
89 ops::ROUTE_CLOSING,
90 ops::ROUTE_CLOSED,
91 ops::SUPERVISOR_LIST,
92 ops::SUPERVISOR_RESTART,
93 ops::SUPERVISOR_SWAP,
94 ops::SUPERVISOR_RELOAD,
95 ops::SUPERVISOR_RESCAN,
96 ops::SUPERVISOR_RELEASE_RESERVED,
97 ops::SUPERVISOR_SET_ENABLED,
98 ops::SUPERVISOR_HEALTH_PROBE,
99 ops::SUPERVISOR_HEALTH,
100 ops::SUPERVISOR_STDERR_TAIL,
101 ops::SUPERVISOR_TERMINALS,
102 ops::SUPERVISOR_ROUTES,
103 ops::SUPERVISOR_PROVENANCE,
104 ops::SUPERVISOR_SPAWN_SNAPSHOT,
105 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
106];
107
108const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
109 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
110 "supervisor.live_roots",
111 SCOPE_SYNC_OP,
112 SCOPE_DESCRIBE_OP,
113];
114
115const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
119
120const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
121
122pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
130
131pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
146
147pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
165
166const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
167const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
168
169fn reload_verdict(
170 configured: &Path,
171 spawned_from: Option<&Path>,
172 image: subc_control::RunningImageAgreement,
173) -> PendingReloadVerdict {
174 let path = match spawned_from {
175 Some(spawned_from) if configured == spawned_from => ReloadPathAgreement::Match,
176 Some(spawned_from) => ReloadPathAgreement::Mismatch {
177 configured: configured.to_path_buf(),
178 spawned_from: spawned_from.to_path_buf(),
179 },
180 None => ReloadPathAgreement::Unavailable {
181 reason: if matches!(
182 image,
183 subc_control::RunningImageAgreement::Unavailable {
184 reason: subc_control::RunningImageUnavailableReason::NotRunning
185 }
186 ) {
187 ReloadPathUnavailableReason::NotRunning
188 } else {
189 ReloadPathUnavailableReason::SpawnedPathUnavailable
190 },
191 },
192 };
193 PendingReloadVerdict { path, image }
194}
195
196#[derive(Clone)]
197struct DaemonProvenanceFacts {
198 build: DaemonBuildProvenance,
199 pid: Option<u32>,
200 started_at_ms: Option<u64>,
201 start_clock: Option<crate::clock::StartClock>,
202 executable_path: Option<PathBuf>,
203 executable_identity: Option<SpawnedFileIdentity>,
204 process_start_time: Option<u64>,
205 probe: ExecutableIdentityProbe,
206}
207
208impl Default for DaemonProvenanceFacts {
209 fn default() -> Self {
210 Self {
211 build: DaemonBuildProvenance {
212 build_git_sha: None,
213 build_lock_digest: None,
214 },
215 pid: None,
216 started_at_ms: None,
217 start_clock: None,
218 executable_path: None,
219 executable_identity: None,
220 process_start_time: None,
221 probe: ExecutableIdentityProbe::default(),
222 }
223 }
224}
225
226#[derive(Debug, Clone)]
227struct SupervisorRescanContext {
228 supervisor: Supervisor,
229 config_path: PathBuf,
230 configured_port: Option<u16>,
231 storage_config: Option<crate::daemon_config::StorageConfig>,
232 admission_facts_carrier_module_id: Option<String>,
233 admission_facts_targets: Option<Vec<String>>,
234 scope_authority_owners: Vec<String>,
235}
236
237const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
258 "reloading",
259 "supervisor_not_live",
260 "registration_not_active",
261 "no_forwarding_connection",
262 "relay_send_failed",
263];
264
265#[derive(Clone)]
267pub struct ControlHandler {
268 registry: Arc<Registry>,
269 forwarding: Arc<ForwardingTable>,
270 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
271 supervisor: SupervisorHandle,
272 subc_capabilities: Arc<[String]>,
273 route_bind_relay_timeout: Duration,
277 route_bind_relay_timeouts: BTreeMap<String, Duration>,
281 route_bind_breakers: RouteBindBreakers,
284 route_bind_concurrency: RouteBindConcurrency,
287 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
292 route_bind_breaker_threshold: u32,
294 route_bind_breaker_cooldown: Duration,
296 health_probe_timeout: Duration,
297 storage_config: Option<crate::daemon_config::StorageConfig>,
300 machine_id: Option<crate::machine_id::MachineId>,
304 admission_facts_carrier_module_id: Option<String>,
305 admission_facts_targets: Option<Vec<String>>,
306 scopes: Arc<RwLock<ScopeTable>>,
310 scope_authority_owners: Vec<String>,
313 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
316 rescan: Option<SupervisorRescanContext>,
317 connected_clients: ConnectedClients,
318 counters: DaemonCounters,
319 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
320 daemon_provenance: DaemonProvenanceFacts,
321 #[cfg(test)]
322 control_dispatch_delay: Option<Duration>,
323 #[cfg(test)]
324 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
325}
326
327impl fmt::Debug for ControlHandler {
328 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
329 f.debug_struct("ControlHandler")
330 .field("registry", &self.registry)
331 .field("forwarding", &self.forwarding)
332 .field("process_liveness", &self.process_liveness.is_some())
333 .field("supervisor", &self.supervisor)
334 .field("subc_capabilities", &self.subc_capabilities)
335 .finish()
336 }
337}
338
339struct RouteOpenRequest {
340 target: RouteTarget,
341 identity: BindIdentity,
342 consumer_identity: Option<ConsumerIdentity>,
343 consumer_capabilities: Option<Vec<String>>,
344 role_versions: Option<BTreeMap<String, String>>,
345 admission_facts: Option<serde_json::Value>,
346 scope: Option<ScopeSelector>,
347}
348
349struct RouteBindReservationGuard {
350 forwarding: Arc<ForwardingTable>,
351 endpoint: ModuleEndpointId,
352 relay_corr: u64,
353 armed: bool,
354}
355
356struct ModuleControlRpcGuard {
357 forwarding: Arc<ForwardingTable>,
358 endpoint: ModuleEndpointId,
359 corr: u64,
360 armed: bool,
361}
362
363impl ModuleControlRpcGuard {
364 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
365 Self {
366 forwarding,
367 endpoint,
368 corr,
369 armed: true,
370 }
371 }
372
373 fn disarm(&mut self) {
374 self.armed = false;
375 }
376}
377
378impl Drop for ModuleControlRpcGuard {
379 fn drop(&mut self) {
380 if self.armed {
381 let _ = self
382 .forwarding
383 .cancel_module_control_rpc(self.endpoint, self.corr);
384 }
385 }
386}
387
388impl RouteBindReservationGuard {
389 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
390 Self {
391 forwarding,
392 endpoint,
393 relay_corr,
394 armed: true,
395 }
396 }
397
398 fn release_and_disarm(&mut self) {
399 if !self.armed {
400 return;
401 }
402 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
403 self.endpoint,
404 self.relay_corr,
405 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
406 ) {
407 send_goodbye_target_best_effort(
408 &self.forwarding.counters(),
409 &target,
410 "canceled route.bind",
411 );
412 }
413 self.armed = false;
414 }
415
416 fn disarm(&mut self) {
417 self.armed = false;
418 }
419}
420
421impl Drop for RouteBindReservationGuard {
422 fn drop(&mut self) {
423 self.release_and_disarm();
424 }
425}
426
427#[derive(Debug, Clone, Default)]
457pub(crate) struct RouteBindBreakers {
458 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
459}
460
461#[derive(Debug, Clone, Default)]
462pub(crate) struct RouteBindConcurrency {
463 modules: Arc<Mutex<HashMap<String, usize>>>,
464}
465
466struct RouteBindConcurrencyGuard {
467 concurrency: RouteBindConcurrency,
468 module_id: String,
469}
470
471impl RouteBindConcurrency {
472 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
475 let mut modules = self
476 .modules
477 .lock()
478 .expect("route.bind concurrency mutex poisoned");
479 let in_flight = modules.entry(module_id.to_string()).or_default();
480 if *in_flight >= limit {
481 return Err(*in_flight);
482 }
483 *in_flight += 1;
484 Ok(RouteBindConcurrencyGuard {
485 concurrency: self.clone(),
486 module_id: module_id.to_string(),
487 })
488 }
489}
490
491impl Drop for RouteBindConcurrencyGuard {
492 fn drop(&mut self) {
493 let mut modules = self
494 .concurrency
495 .modules
496 .lock()
497 .expect("route.bind concurrency mutex poisoned");
498 let remove = {
499 let in_flight = modules
500 .get_mut(&self.module_id)
501 .expect("admitted route.bind has a concurrency entry");
502 *in_flight -= 1;
503 *in_flight == 0
504 };
505 if remove {
506 modules.remove(&self.module_id);
507 }
508 }
509}
510
511#[derive(Debug, Default)]
512struct ModuleBreakerState {
513 consecutive_timeouts: u32,
515 cooldown_until: Option<Instant>,
518 probe_in_flight: bool,
523}
524
525enum RouteBindAdmission<'a> {
527 Admitted {
528 guard: RouteBindBreakerGuard<'a>,
529 probe: bool,
532 },
533 Refused {
534 consecutive_timeouts: u32,
535 retry_in: Duration,
539 probe_in_flight: bool,
540 },
541}
542
543struct RouteBindBreakerGuard<'a> {
550 breakers: RouteBindBreakers,
551 module_id: &'a str,
552 settled: bool,
553}
554
555impl RouteBindBreakerGuard<'_> {
556 fn record_accepted(&mut self) -> bool {
560 self.settled = true;
561 self.breakers.record_accepted(self.module_id)
562 }
563
564 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
567 self.settled = true;
568 self.breakers
569 .record_timeout(self.module_id, threshold, cooldown)
570 }
571
572 fn record_inconclusive(&mut self) {
581 self.settled = true;
582 self.breakers.record_inconclusive(self.module_id);
583 }
584}
585
586impl Drop for RouteBindBreakerGuard<'_> {
587 fn drop(&mut self) {
588 if !self.settled {
589 self.breakers.record_inconclusive(self.module_id);
590 }
591 }
592}
593
594struct BreakerOpened {
598 consecutive_timeouts: u32,
599 reopened_after_probe: bool,
602}
603
604impl RouteBindBreakers {
605 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
606 self.modules
607 .lock()
608 .expect("route.bind breaker mutex poisoned")
609 }
610
611 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
614 let admitted = |probe| RouteBindAdmission::Admitted {
615 guard: RouteBindBreakerGuard {
616 breakers: self.clone(),
617 module_id,
618 settled: false,
619 },
620 probe,
621 };
622
623 let mut modules = self.lock();
624 let Some(state) = modules.get_mut(module_id) else {
625 return admitted(false);
626 };
627 let Some(cooldown_until) = state.cooldown_until else {
628 return admitted(false);
629 };
630 if state.probe_in_flight {
631 return RouteBindAdmission::Refused {
632 consecutive_timeouts: state.consecutive_timeouts,
633 retry_in: Duration::ZERO,
634 probe_in_flight: true,
635 };
636 }
637 let now = Instant::now();
638 if now < cooldown_until {
639 return RouteBindAdmission::Refused {
640 consecutive_timeouts: state.consecutive_timeouts,
641 retry_in: cooldown_until - now,
642 probe_in_flight: false,
643 };
644 }
645 state.probe_in_flight = true;
646 admitted(true)
647 }
648
649 fn record_accepted(&self, module_id: &str) -> bool {
650 self.lock()
651 .remove(module_id)
652 .is_some_and(|state| state.cooldown_until.is_some())
653 }
654
655 fn record_timeout(
656 &self,
657 module_id: &str,
658 threshold: u32,
659 cooldown: Duration,
660 ) -> Option<BreakerOpened> {
661 let mut modules = self.lock();
662 let state = modules.entry(module_id.to_string()).or_default();
663 let was_open = state.cooldown_until.is_some();
664 let was_probe = state.probe_in_flight;
665 state.probe_in_flight = false;
666 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
667 if state.consecutive_timeouts < threshold {
668 return None;
669 }
670 state.cooldown_until = Some(Instant::now() + cooldown);
671 Some(BreakerOpened {
672 consecutive_timeouts: state.consecutive_timeouts,
673 reopened_after_probe: was_open && was_probe,
674 })
675 }
676
677 fn record_inconclusive(&self, module_id: &str) {
678 if let Some(state) = self.lock().get_mut(module_id) {
679 state.probe_in_flight = false;
680 }
681 }
682
683 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
698 self.lock()
699 .remove(module_id)
700 .map(|state| state.consecutive_timeouts)
701 .filter(|discarded| *discarded > 0)
702 }
703
704 fn open_snapshot(&self) -> Option<serde_json::Value> {
711 let now = Instant::now();
712 let modules = self.lock();
713 let open = modules
714 .iter()
715 .filter_map(|(module_id, state)| {
716 let cooldown_until = state.cooldown_until?;
717 Some((
718 module_id.clone(),
719 serde_json::json!({
720 "consecutive_timeouts": state.consecutive_timeouts,
721 "cooldown_remaining_ms":
722 cooldown_until.saturating_duration_since(now).as_millis() as u64,
723 "probe_in_flight": state.probe_in_flight,
724 }),
725 ))
726 })
727 .collect::<serde_json::Map<String, serde_json::Value>>();
728 (!open.is_empty()).then_some(serde_json::Value::Object(open))
729 }
730}
731
732impl ControlHandler {
733 pub fn new(registry: Arc<Registry>) -> Self {
734 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
735 }
736
737 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
738 let counters = forwarding.counters();
739 let route_bind_breakers = forwarding.route_bind_breakers();
743 let route_bind_concurrency = forwarding.route_bind_concurrency();
744 let route_outages = forwarding.route_outages();
745 Self {
746 registry,
747 forwarding,
748 process_liveness: None,
749 supervisor: SupervisorHandle::new(),
750 subc_capabilities: Arc::from([
751 CAP_MANIFEST_REGISTRATION.to_string(),
752 CAP_CHANNEL_LIFECYCLE.to_string(),
753 CAP_PING_PONG.to_string(),
754 CAP_SESSION_ATTACH.to_string(),
755 CAP_ADMISSION_FACTS_RELAY.to_string(),
756 CAP_SCOPES_V1.to_string(),
757 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
758 ]),
759 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
760 route_bind_relay_timeouts: BTreeMap::new(),
761 route_bind_breakers,
762 route_bind_concurrency,
763 route_outages,
764 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
765 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
766 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
767 storage_config: None,
768 machine_id: None,
769 admission_facts_carrier_module_id: None,
770 admission_facts_targets: None,
771 scopes: Arc::new(RwLock::new(ScopeTable::new(
772 crate::daemon_config::default_scope_authority_owners(),
773 ))),
774 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
775 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
776 rescan: None,
777 connected_clients: ConnectedClients::new(),
778 counters,
779 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
780 daemon_provenance: DaemonProvenanceFacts::default(),
781 #[cfg(test)]
782 control_dispatch_delay: None,
783 #[cfg(test)]
784 provenance_probe_override: None,
785 }
786 }
787
788 pub fn with_storage_config(
791 mut self,
792 storage_config: Option<crate::daemon_config::StorageConfig>,
793 ) -> Self {
794 self.storage_config = storage_config;
795 self
796 }
797
798 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
801 self.machine_id = machine_id;
802 self
803 }
804
805 pub fn with_admission_facts_config(
809 mut self,
810 carrier_module_id: Option<String>,
811 targets: Option<Vec<String>>,
812 ) -> Self {
813 self.admission_facts_carrier_module_id = carrier_module_id;
814 self.admission_facts_targets = targets;
815 self
816 }
817
818 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
822 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
823 self.scope_authority_owners = owners;
824 self
825 }
826
827 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
830 self.route_bind_relay_timeout = timeout;
831 self
832 }
833
834 pub fn with_route_bind_relay_timeouts(
840 mut self,
841 timeouts: impl IntoIterator<Item = (String, Duration)>,
842 ) -> Self {
843 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
844 self
845 }
846
847 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
853 self.route_bind_relay_timeouts
854 .get(module_id)
855 .copied()
856 .unwrap_or(self.route_bind_relay_timeout)
857 }
858
859 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
867 self.route_bind_breaker_threshold = threshold.max(1);
868 self.route_bind_breaker_cooldown = cooldown;
869 self
870 }
871
872 #[cfg(test)]
873 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
874 self.health_probe_timeout = timeout;
875 self
876 }
877
878 #[cfg(test)]
879 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
880 self.control_dispatch_delay = Some(delay);
881 self
882 }
883
884 pub fn with_process_liveness(
885 mut self,
886 process_liveness: Arc<dyn ModuleProcessLiveness>,
887 ) -> Self {
888 self.process_liveness = Some(process_liveness);
889 self
890 }
891
892 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
893 self.supervisor = supervisor;
894 self
895 }
896
897 pub fn with_daemon_provenance(
898 mut self,
899 pid: u32,
900 started_at_ms: u64,
901 executable_path: Option<PathBuf>,
902 build_git_sha: Option<String>,
903 build_lock_digest: Option<String>,
904 ) -> Self {
905 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
906 let process_start_time = process_start_time(pid);
907 self.daemon_provenance = DaemonProvenanceFacts {
908 build: DaemonBuildProvenance {
909 build_git_sha,
910 build_lock_digest,
911 },
912 pid: Some(pid),
913 started_at_ms: Some(started_at_ms),
914 start_clock: None,
915 executable_path,
916 executable_identity,
917 process_start_time,
918 probe: ExecutableIdentityProbe::default(),
919 };
920 self
921 }
922
923 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
924 self.daemon_provenance.start_clock = Some(clock);
925 self
926 }
927
928 #[cfg(test)]
929 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
930 self.provenance_probe_override = Some(result);
931 self
932 }
933
934 pub fn with_capability_config(
938 self,
939 modules: impl IntoIterator<Item = (String, bool)>,
940 reserved_capabilities: BTreeMap<String, String>,
941 ) -> Self {
942 self.capability_evaluator
943 .configure(modules, reserved_capabilities);
944 self
945 }
946
947 pub fn with_supervisor_rescan(
948 mut self,
949 supervisor: Supervisor,
950 config_path: impl Into<PathBuf>,
951 configured_port: Option<u16>,
952 ) -> Self {
953 self.rescan = Some(SupervisorRescanContext {
954 supervisor,
955 config_path: config_path.into(),
956 configured_port,
957 storage_config: self.storage_config.clone(),
958 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
959 admission_facts_targets: self.admission_facts_targets.clone(),
960 scope_authority_owners: self.scope_authority_owners.clone(),
961 });
962 self
963 }
964
965 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
966 self.connected_clients = connected_clients;
967 self
968 }
969
970 pub fn forwarding(&self) -> Arc<ForwardingTable> {
971 Arc::clone(&self.forwarding)
972 }
973
974 pub(crate) fn counters(&self) -> DaemonCounters {
975 self.counters.clone()
976 }
977
978 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
981 tokio::spawn(async move {
982 loop {
983 self.capability_evaluator
984 .wait_for_change_or_deadline()
985 .await;
986 self.refresh_capability_requirements();
987 }
988 });
989 }
990
991 fn runtime_capability_snapshot(
992 &self,
993 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
994 let runtime = self
995 .supervisor
996 .list()
997 .into_iter()
998 .map(|module| {
999 let status = module.status().map_err(|err| {
1000 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1001 })?;
1002 Ok(RuntimeModule {
1003 module_id: status.module_id,
1004 state: status.state,
1005 enabled: status.enabled,
1006 })
1007 })
1008 .collect::<Result<Vec<_>, RouterError>>()?;
1009 let (_, registrations) = self.registry.list_modules().map_err(|err| {
1010 RouterError::backend(
1011 0,
1012 0,
1013 format!("failed to list capability registrations: {err}"),
1014 )
1015 })?;
1016 let registrations = registrations
1017 .into_iter()
1018 .map(|registration| RegisteredModule {
1019 module_id: registration.manifest.module_id,
1020 module_version: registration.manifest.module_version,
1021 capabilities: registration.manifest.capabilities,
1022 })
1023 .collect();
1024 Ok((runtime, registrations))
1025 }
1026
1027 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1034 let cached_registration = RegisteredModule {
1035 module_id: registration.manifest.module_id.clone(),
1036 module_version: registration.manifest.module_version.clone(),
1037 capabilities: registration.manifest.capabilities.clone(),
1038 };
1039 if self.capability_evaluator.record_hello(&cached_registration) {
1040 warn!(
1041 module_id = %cached_registration.module_id,
1042 "capability claims drifted from the cached manifest"
1043 );
1044 }
1045 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1046 self.enforce_capability_denies();
1047 }
1048 self.refresh_capability_requirements();
1049 }
1050
1051 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1055 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1056 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1057 self.supervisor.set_swap_promotion_observer(observer);
1058 }
1059
1060 pub fn refresh_capability_requirements(&self) {
1061 match self.runtime_capability_snapshot() {
1062 Ok((runtime, registrations)) => {
1063 log_requirement_events(
1064 self.capability_evaluator
1065 .evaluate_now(&runtime, ®istrations),
1066 );
1067 }
1068 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1069 }
1070 }
1071
1072 fn enforce_capability_denies(&self) {
1076 let (_, registrations) = match self.registry.list_modules() {
1077 Ok(snapshot) => snapshot,
1078 Err(err) => {
1079 warn!(error = %err, "failed to read registrations for capability deny census");
1080 return;
1081 }
1082 };
1083 let manifests = registrations
1084 .into_iter()
1085 .map(|registration| {
1086 (
1087 registration.manifest.module_id.clone(),
1088 registration.manifest,
1089 )
1090 })
1091 .collect::<BTreeMap<_, _>>();
1092 let census = match self.forwarding.route_census(None) {
1093 Ok(census) => census,
1094 Err(err) => {
1095 warn!(error = %err, "failed to read route census for capability deny enforcement");
1096 return;
1097 }
1098 };
1099
1100 for (target_module_id, routes) in census {
1101 let Some(target_manifest) = manifests.get(&target_module_id) else {
1102 continue;
1103 };
1104 let mut closed_routes = Vec::new();
1105 let mut module_goodbyes = Vec::new();
1106 for route in routes {
1107 let Principal::Reserved {
1108 module_id: opening_module_id,
1109 } = &route.principal
1110 else {
1111 continue;
1112 };
1113 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1114 continue;
1115 };
1116 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1117 continue;
1118 };
1119
1120 match self.forwarding.release_client_route(
1121 route.goodbye_target.connection_id,
1122 route.goodbye_target.channel,
1123 route.goodbye_target.epoch,
1124 ) {
1125 Ok(RouteRelease::Removed(module_goodbye)) => {
1126 warn!(
1127 opening_module_id,
1128 target_module_id,
1129 capability,
1130 "force-closing route because an attested capability deny edge now matches"
1131 );
1132 closed_routes.push(route);
1133 module_goodbyes.push(module_goodbye);
1134 }
1135 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1136 Err(err) => warn!(
1137 opening_module_id,
1138 target_module_id,
1139 capability,
1140 error = %err,
1141 "failed to force-close capability-denied route"
1142 ),
1143 }
1144 }
1145
1146 if closed_routes.is_empty() {
1147 continue;
1148 }
1149 send_route_control_pushes(
1150 &self.forwarding,
1151 closed_routes,
1152 ClientControlPush::RouteClosed {
1153 module_id: target_module_id,
1154 reason: RouteCloseReason::CapabilityDenied,
1155 drained: false,
1156 abandoned: 0,
1157 excluded_subscriptions: 0,
1158 terminal: Some(false),
1159 },
1160 );
1161 self.emit_route_goodbyes(module_goodbyes);
1162 }
1163 }
1164
1165 fn not_ready_reason(
1171 &self,
1172 registration: &crate::registry::ModuleRegistration,
1173 ) -> Option<NotReadyReason> {
1174 if !registration.ready {
1175 return Some(NotReadyReason {
1176 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1177 capability: None,
1178 });
1179 }
1180 self.first_unprovided_required_capability(registration)
1181 .map(|capability| NotReadyReason {
1182 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1183 capability: Some(capability),
1184 })
1185 }
1186
1187 fn first_unprovided_required_capability(
1205 &self,
1206 registration: &crate::registry::ModuleRegistration,
1207 ) -> Option<String> {
1208 let required = registration
1209 .manifest
1210 .capabilities
1211 .iter()
1212 .flat_map(|declarations| declarations.requires.iter())
1213 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1214 .map(|requirement| requirement.capability.as_str())
1215 .collect::<BTreeSet<_>>();
1216 if required.is_empty() {
1217 return None;
1218 }
1219 let module_id = registration.manifest.module_id.as_str();
1220 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1221 if required
1222 .iter()
1223 .any(|capability| verdict(capability).is_none())
1224 {
1225 self.refresh_capability_requirements();
1226 }
1227 required
1228 .into_iter()
1229 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1230 .map(str::to_string)
1231 }
1232
1233 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1234 self.capability_evaluator
1235 .statuses()
1236 .into_iter()
1237 .map(capability_requirement_status)
1238 .collect()
1239 }
1240
1241 fn deregister_connection(
1248 &self,
1249 connection_id: ConnectionId,
1250 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1251 self.registry.deregister_connection(connection_id)
1252 }
1253
1254 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1255 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1256 return None;
1257 }
1258 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1259 parse_client_control_request(&frame.body)
1260 else {
1261 return None;
1262 };
1263 Some(target_module_id(&target).to_string())
1264 }
1265
1266 pub(crate) fn route_open_capacity_refusal(
1267 &self,
1268 ctx: &RouteCtx,
1269 frame: &Frame,
1270 target_module_id: &str,
1271 in_flight: usize,
1272 limit: usize,
1273 ) -> Result<Frame, RouterError> {
1274 self.route_open_admission_refusal_frame(
1275 ctx,
1276 frame,
1277 target_module_id,
1278 "open_admission_full",
1279 (in_flight, limit),
1280 format!(
1281 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1282 ),
1283 )
1284 }
1285
1286 fn route_open_target_capacity_refusal(
1287 &self,
1288 ctx: &RouteCtx,
1289 frame: &Frame,
1290 target_module_id: &str,
1291 in_flight: usize,
1292 ) -> Result<Frame, RouterError> {
1293 self.route_open_admission_refusal_frame(
1294 ctx,
1295 frame,
1296 target_module_id,
1297 "target_binds_full",
1298 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1299 format!(
1300 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1301 ),
1302 )
1303 }
1304
1305 fn route_open_admission_refusal_frame(
1313 &self,
1314 ctx: &RouteCtx,
1315 frame: &Frame,
1316 target_module_id: &str,
1317 reason: &'static str,
1318 (in_flight, limit): (usize, usize),
1319 message: impl Into<String>,
1320 ) -> Result<Frame, RouterError> {
1321 let code = error_codes::TARGET_UNAVAILABLE;
1322 self.counters.increment_route_open_refused(code);
1323 info!(
1324 target: "control",
1325 code,
1326 reason,
1327 module_id = ?target_module_id,
1328 connection_id = ctx.connection_id.get(),
1329 in_flight,
1330 limit,
1331 "route.open refused"
1332 );
1333 control_error_frame(frame, code, message.into())
1334 }
1335
1336 #[cfg(test)]
1342 pub fn handle_control(
1343 &self,
1344 connection_id: ConnectionId,
1345 frame: Frame,
1346 ) -> Result<Vec<Frame>, RouterError> {
1347 match frame.header.ty {
1348 FrameType::Ping => Ok(vec![pong(&frame)?]),
1349 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1350 FrameType::Goodbye => self.handle_goodbye(connection_id),
1351 ty => Ok(vec![control_error_frame(
1352 &frame,
1353 "unsupported_control_frame",
1354 format!("unsupported channel-0 frame {ty:?}"),
1355 )?]),
1356 }
1357 }
1358
1359 pub async fn handle_control_frame(
1360 &self,
1361 ctx: &RouteCtx,
1362 frame: Frame,
1363 ) -> Result<Vec<Frame>, RouterError> {
1364 self.handle_control_frame_timed(ctx, frame, None).await
1365 }
1366
1367 pub(crate) async fn handle_control_frame_timed(
1368 &self,
1369 ctx: &RouteCtx,
1370 frame: Frame,
1371 dispatch_started_at: Option<StdInstant>,
1372 ) -> Result<Vec<Frame>, RouterError> {
1373 match frame.header.ty {
1374 FrameType::Ping => Ok(vec![pong(&frame)?]),
1375 FrameType::Hello => {
1376 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1377 }
1378 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1379 FrameType::Cancel => {
1380 if self
1381 .supervisor
1382 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1383 {
1384 Ok(Vec::new())
1385 } else {
1386 Ok(vec![control_error_frame(
1387 &frame,
1388 "unknown_subscription",
1389 "no supervisor spawn subscription has this correlation id",
1390 )?])
1391 }
1392 }
1393 FrameType::Request => {
1394 if self
1395 .forwarding
1396 .module_endpoint_for_connection(ctx.connection_id)
1397 .map_err(RouterError::Forwarding)?
1398 .is_some()
1399 {
1400 if !is_known_module_request_op(&frame.body) {
1401 return Ok(vec![control_error_frame(
1402 &frame,
1403 "unsupported_control_frame",
1404 "module-originated channel-0 REQUEST is not supported",
1405 )?]);
1406 }
1407 let request = match parse_module_control_request_from_module(&frame.body) {
1408 Ok(request) => request,
1409 Err((err, ControlRequestBodyError::UnknownOp)) => {
1410 return Ok(vec![control_error_frame(
1411 &frame,
1412 "unsupported_control_frame",
1413 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1414 )?])
1415 }
1416 Err((err, ControlRequestBodyError::InvalidBody)) => {
1417 return Ok(vec![control_error_frame(
1418 &frame,
1419 "invalid_control_body",
1420 format!("malformed module control body: {err}"),
1421 )?])
1422 }
1423 };
1424 let op = module_control_request_op(&request);
1425 let corr = frame.header.corr;
1426 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1427 let result =
1428 self.handle_module_control_request(ctx.connection_id, frame, request);
1429 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1430 return result;
1431 }
1432
1433 if is_known_module_request_op(&frame.body) {
1434 return Ok(vec![control_error_frame(
1435 &frame,
1436 "not_registered",
1437 "catalog.update requires an active module registration owned by this connection",
1438 )?]);
1439 }
1440
1441 let request = match parse_client_control_request(&frame.body) {
1442 Ok(request) => request,
1443 Err((err, ControlRequestBodyError::UnknownOp)) => {
1444 return Ok(vec![control_error_frame(
1445 &frame,
1446 "unknown_control_op",
1447 format!("unknown client control op: {err}"),
1448 )?])
1449 }
1450 Err((err, ControlRequestBodyError::InvalidBody)) => {
1451 return Ok(vec![control_error_frame(
1452 &frame,
1453 "invalid_control_body",
1454 format!("malformed client control body: {err}"),
1455 )?])
1456 }
1457 };
1458 let op = client_control_request_op(&request);
1459 let corr = frame.header.corr;
1460 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1461 #[cfg(test)]
1462 if let Some(delay) = self.control_dispatch_delay {
1463 tokio::time::sleep(delay).await;
1464 }
1465 let result = self
1466 .handle_client_control_request(ctx, frame, request)
1467 .await;
1468 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1469 result
1470 }
1471 FrameType::Push => {
1472 let Some(endpoint) = self
1473 .forwarding
1474 .module_endpoint_for_connection(ctx.connection_id)
1475 .map_err(RouterError::Forwarding)?
1476 else {
1477 return Ok(vec![control_error_frame(
1478 &frame,
1479 "unsupported_control_frame",
1480 "client-originated channel-0 PUSH is not supported",
1481 )?]);
1482 };
1483 self.handle_status_update(endpoint, frame)
1484 }
1485 FrameType::Response | FrameType::Error
1486 if self
1487 .forwarding
1488 .module_endpoint_for_connection(ctx.connection_id)
1489 .map_err(RouterError::Forwarding)?
1490 .is_some() =>
1491 {
1492 self.handle_module_relay_response(ctx.connection_id, frame)
1493 }
1494 ty => Ok(vec![control_error_frame(
1495 &frame,
1496 "unsupported_control_frame",
1497 format!("unsupported channel-0 frame {ty:?}"),
1498 )?]),
1499 }
1500 }
1501
1502 pub fn cleanup_connection(
1503 &self,
1504 connection_id: ConnectionId,
1505 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1506 let crash_closed = self
1507 .registry
1508 .get_module_by_connection(connection_id)?
1509 .and_then(|registration| {
1510 self.forwarding
1511 .module_endpoint_for_connection(connection_id)
1512 .ok()
1513 .flatten()
1514 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1515 .map(|routes| (registration.manifest.module_id, routes))
1516 });
1517 let crash_closed = crash_closed.map(|(module_id, routes)| {
1518 let terminal = match self.supervisor.get(&module_id) {
1519 None => false,
1520 Some(module) => match module.will_recover_after_connection_loss() {
1521 Ok(will_recover) => !will_recover,
1522 Err(err) => {
1523 warn!(
1524 %module_id,
1525 error = %err,
1526 "failed to read crash recovery verdict; reporting non-terminal conservatively"
1527 );
1528 false
1529 }
1530 },
1531 };
1532 let reason = match self.forwarding.is_daemon_draining() {
1536 Ok(true) => RouteCloseReason::Restart,
1537 Ok(false) => RouteCloseReason::Crash,
1538 Err(err) => {
1539 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
1540 RouteCloseReason::Crash
1541 }
1542 };
1543 (module_id, routes, reason, terminal)
1544 });
1545 let registrations = self.deregister_connection(connection_id);
1546 let cleanup = self.forwarding.cleanup_connection_counted(connection_id);
1547 if let Some((module_id, routes, reason, terminal)) = crash_closed {
1552 let abandoned = cleanup
1553 .as_ref()
1554 .map_or(0, |cleanup| cleanup.abandoned_relays);
1555 send_route_control_pushes(
1556 &self.forwarding,
1557 routes,
1558 ClientControlPush::RouteClosed {
1559 module_id,
1560 reason,
1561 drained: false,
1562 abandoned,
1563 excluded_subscriptions: 0,
1564 terminal: Some(terminal),
1565 },
1566 );
1567 }
1568 if let Ok(cleanup) = cleanup {
1569 self.emit_route_goodbyes(cleanup.released);
1570 }
1571 if matches!(®istrations, Ok(r) if !r.is_empty()) {
1575 crate::supervise::notify_registration_release();
1576 self.capability_evaluator.wake_deadline_loop();
1577 self.refresh_capability_requirements();
1578 }
1579 self.supervisor.remove_spawn_subscribers(connection_id);
1580 self.hello_launch_nonces
1583 .lock()
1584 .unwrap_or_else(|poisoned| poisoned.into_inner())
1585 .forget(connection_id);
1586 self.scopes
1587 .write()
1588 .unwrap_or_else(|poisoned| poisoned.into_inner())
1589 .release_connection(connection_id);
1590 registrations
1591 }
1592
1593 pub(crate) fn handle_route_goodbye(
1594 &self,
1595 connection_id: ConnectionId,
1596 route_channel: u16,
1597 route_epoch: u32,
1598 ) -> Result<bool, RouterError> {
1599 debug!(
1600 connection_id = connection_id.get(),
1601 route_channel, route_epoch, "handling route GOODBYE"
1602 );
1603 let RouteRelease::Removed(released_route) = self
1604 .forwarding
1605 .release_client_route(connection_id, route_channel, route_epoch)
1606 .map_err(RouterError::Forwarding)?
1607 else {
1608 return Ok(false);
1609 };
1610 self.emit_route_goodbyes(vec![released_route]);
1611 Ok(true)
1612 }
1613
1614 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
1615 for released in released_routes {
1616 let frame = match Frame::build_with_version(
1617 released.negotiated_ver,
1618 FrameType::Goodbye,
1619 control_flags(),
1620 released.channel,
1621 released.epoch,
1622 0,
1623 Vec::new(),
1624 ) {
1625 Ok(frame) => frame,
1626 Err(err) => {
1627 warn!(
1628 route_channel = released.channel,
1629 error = %err,
1630 "failed to build route GOODBYE frame"
1631 );
1632 continue;
1633 }
1634 };
1635 if !released.close_on_delivery_failure() {
1636 crate::forwarding::send_module_route_goodbye(
1637 &self.counters,
1638 &released.sink,
1639 frame,
1640 released.module_id.as_deref(),
1641 "client route released",
1642 );
1643 continue;
1644 }
1645 if let Err(err) = released.sink.try_send(frame) {
1646 warn!(
1647 target_connection_id = released.connection_id.get(),
1648 route_channel = released.channel,
1649 error = %err,
1650 "route GOODBYE was not delivered to client; closing target connection"
1651 );
1652 if self
1653 .forwarding
1654 .escalate_client_delivery_failure(
1655 released.connection_id,
1656 released.channel,
1657 released.epoch,
1658 CloseReason::new(
1659 "route_goodbye_delivery_failed",
1660 format!(
1661 "failed to enqueue route GOODBYE for channel {}: {err}",
1662 released.channel
1663 ),
1664 ),
1665 crate::forwarding::UndeliveredFrame {
1666 module_id: released.module_id.as_deref(),
1667 sink: &released.sink,
1668 },
1669 )
1670 .unwrap_or(false)
1671 {
1672 self.counters.increment_goodbye_relay_client_failed();
1673 }
1674 }
1675 }
1676 }
1677
1678 fn send_abandoned_route_bind_goodbye(
1690 &self,
1691 module_sink: &crate::FrameSink,
1692 negotiated_ver: u8,
1693 module_channel: u16,
1694 module_epoch: u32,
1695 ) {
1696 let frame = match Frame::build_with_version(
1697 negotiated_ver,
1698 FrameType::Goodbye,
1699 control_flags(),
1700 module_channel,
1701 module_epoch,
1702 0,
1703 Vec::new(),
1704 ) {
1705 Ok(frame) => frame,
1706 Err(err) => {
1707 warn!(
1708 route_channel = module_channel,
1709 error = %err,
1710 "failed to build GOODBYE for abandoned route.bind"
1711 );
1712 return;
1713 }
1714 };
1715 crate::forwarding::send_module_route_goodbye(
1716 &self.counters,
1717 module_sink,
1718 frame,
1719 None,
1720 "abandoned route.bind",
1721 );
1722 }
1723
1724 fn handle_hello(
1725 &self,
1726 connection_id: ConnectionId,
1727 sink: Option<crate::FrameSink>,
1728 frame: Frame,
1729 ) -> Result<Vec<Frame>, RouterError> {
1730 debug!(
1731 connection_id = connection_id.get(),
1732 corr = frame.header.corr,
1733 "handling HELLO"
1734 );
1735 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
1736 Ok(value) => value,
1737 Err(err) => {
1738 return Ok(vec![control_error_frame(
1739 &frame,
1740 "invalid_hello",
1741 format!("malformed HELLO body: {err}"),
1742 )?])
1743 }
1744 };
1745 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
1746 return Ok(vec![control_error_frame(
1747 &frame,
1748 "invalid_capability_grammar",
1749 err.to_string(),
1750 )?]);
1751 }
1752 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
1753 return Ok(vec![control_error_frame(
1754 &frame,
1755 "invalid_manifest",
1756 err.to_string(),
1757 )?]);
1758 }
1759 if let Some(provenance) = hello_value
1760 .get("manifest")
1761 .and_then(|manifest| manifest.get("provenance"))
1762 {
1763 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
1764 return Ok(vec![control_error_frame(
1765 &frame,
1766 "invalid_manifest",
1767 format!("malformed manifest provenance: {err}"),
1768 )?]);
1769 }
1770 }
1771 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
1772 Ok(hello) => hello,
1773 Err(err) => {
1774 return Ok(vec![control_error_frame(
1775 &frame,
1776 "invalid_hello",
1777 format!("malformed HELLO body: {err}"),
1778 )?])
1779 }
1780 };
1781
1782 if hello.protocol_ver != hello.manifest.protocol_ver {
1783 return Ok(vec![control_error_frame(
1784 &frame,
1785 "invalid_manifest",
1786 format!(
1787 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
1788 hello.protocol_ver, hello.manifest.protocol_ver
1789 ),
1790 )?]);
1791 }
1792
1793 if hello.manifest.module_id.trim().is_empty() {
1794 return Ok(vec![control_error_frame(
1795 &frame,
1796 "invalid_manifest",
1797 "manifest module_id must not be empty",
1798 )?]);
1799 }
1800
1801 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
1802 Ok(negotiated_ver) => negotiated_ver,
1803 Err(message) => {
1804 return Ok(vec![control_error_frame(
1805 &frame,
1806 "version_unsupported",
1807 message,
1808 )?])
1809 }
1810 };
1811
1812 let swap_admission = self
1820 .supervisor
1821 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
1822 if swap_admission == SwapHelloAdmission::Refused {
1823 warn!(
1824 module_id = %hello.manifest.module_id,
1825 connection_id = connection_id.get(),
1826 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
1827 );
1828 return Ok(vec![control_error_frame(
1829 &frame,
1830 "swap_token_invalid",
1831 format!(
1832 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
1833 hello.manifest.module_id
1834 ),
1835 )?]);
1836 }
1837 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
1838
1839 if let Some(rejection) = (!swap_candidate)
1847 .then(|| {
1848 self.supervisor.reserved_hello_rejection(
1849 &hello.manifest.module_id,
1850 hello.launch_nonce.as_deref(),
1851 )
1852 })
1853 .flatten()
1854 {
1855 let message = match rejection {
1856 ReservedHelloRejection::Exact { module_id } => format!(
1857 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
1858 ),
1859 ReservedHelloRejection::Prefix {
1860 prefix,
1861 owner_module_id,
1862 } => format!(
1863 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
1864 hello.manifest.module_id
1865 ),
1866 };
1867 return Ok(vec![control_error_frame(
1868 &frame,
1869 "reserved_module",
1870 message,
1871 )?]);
1872 }
1873
1874 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
1875 &hello.manifest.module_id,
1876 hello.manifest.capabilities.as_ref(),
1877 );
1878 if let Some(refusal) = reserved_capability_refusals.first() {
1879 let capability = refusal.capability.clone();
1880 let bound_module = refusal.claimants[0].clone();
1881 log_duplicate_claim_events(reserved_capability_refusals);
1882 return Ok(vec![control_error_frame(
1883 &frame,
1884 "reserved_capability",
1885 format!(
1886 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
1887 capability, bound_module, hello.manifest.module_id
1888 ),
1889 )?]);
1890 }
1891
1892 if self
1895 .forwarding
1896 .connection_has_client_routes(connection_id)
1897 .map_err(RouterError::Forwarding)?
1898 {
1899 return Ok(vec![control_error_frame(
1900 &frame,
1901 "invalid_hello",
1902 "connection has open client routes and cannot also register as a module",
1903 )?]);
1904 }
1905
1906 self.hello_launch_nonces
1911 .lock()
1912 .unwrap_or_else(|poisoned| poisoned.into_inner())
1913 .record(connection_id, hello.launch_nonce.as_deref());
1914 let control_ops = effective_module_control_ops(hello.control_ops);
1915 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
1918 if swap_candidate {
1919 return self.register_swap_candidate(
1920 connection_id,
1921 sink,
1922 &frame,
1923 hello.manifest,
1924 negotiated_ver,
1925 control_ops,
1926 hello_ack,
1927 );
1928 }
1929 let registration = match self.registry.register_with_control_ops(
1930 hello.manifest,
1931 negotiated_ver,
1932 connection_id,
1933 control_ops,
1934 ) {
1935 Ok(registration) => registration,
1936 Err(RegistryError::DuplicateModuleId { module_id }) => {
1937 return Ok(vec![control_error_frame(
1938 &frame,
1939 "duplicate_module_id",
1940 format!(
1941 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
1942 ),
1943 )?])
1944 }
1945 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
1946 return Ok(vec![control_error_frame(
1947 &frame,
1948 "invalid_module_id",
1949 err.to_string(),
1950 )?])
1951 }
1952 Err(err) => {
1953 return Ok(vec![control_error_frame(
1954 &frame,
1955 "registry_error",
1956 err.to_string(),
1957 )?])
1958 }
1959 };
1960
1961 let reply = if let Some(sink) = sink {
1962 let concurrency = manifest_concurrency(®istration.manifest);
1978 if let Err(err) = self.forwarding.register_module_connection_acked(
1979 connection_id,
1980 registration.manifest.module_id.clone(),
1981 negotiated_ver,
1982 concurrency,
1983 sink,
1984 hello_ack,
1985 ) {
1986 if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
1990 crate::supervise::notify_registration_release();
1991 }
1992 return Ok(vec![control_error_frame(
1993 &frame,
1994 forwarding_error_code(&err),
1995 err.to_string(),
1996 )?]);
1997 }
1998 Vec::new()
1999 } else {
2000 vec![hello_ack]
2003 };
2004
2005 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2014 info!(
2015 module_id = %registration.manifest.module_id,
2016 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2017 );
2018 }
2019
2020 self.apply_registration_capabilities(®istration);
2021
2022 info!(
2023 module_id = %registration.manifest.module_id,
2024 module_version = %registration.manifest.module_version,
2025 negotiated_ver,
2026 routable_provider = manifest_provides_routable_role(®istration.manifest),
2027 connection_id = connection_id.get(),
2028 "module registered"
2029 );
2030
2031 Ok(reply)
2032 }
2033
2034 #[allow(clippy::too_many_arguments)]
2044 fn register_swap_candidate(
2045 &self,
2046 connection_id: ConnectionId,
2047 sink: Option<crate::FrameSink>,
2048 frame: &Frame,
2049 manifest: ModuleManifest,
2050 negotiated_ver: u8,
2051 control_ops: Vec<String>,
2052 hello_ack: Frame,
2053 ) -> Result<Vec<Frame>, RouterError> {
2054 let module_id = manifest.module_id.clone();
2055 let registration = match self.registry.register_candidate_with_control_ops(
2056 manifest,
2057 negotiated_ver,
2058 connection_id,
2059 control_ops,
2060 ) {
2061 Ok(registration) => registration,
2062 Err(RegistryError::DuplicateModuleId { module_id }) => {
2063 return Ok(vec![control_error_frame(
2064 frame,
2065 "duplicate_module_id",
2066 format!(
2067 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2068 ),
2069 )?])
2070 }
2071 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2072 return Ok(vec![control_error_frame(
2073 frame,
2074 "invalid_module_id",
2075 err.to_string(),
2076 )?])
2077 }
2078 Err(err) => {
2079 return Ok(vec![control_error_frame(
2080 frame,
2081 "registry_error",
2082 err.to_string(),
2083 )?])
2084 }
2085 };
2086 let reply = if let Some(sink) = sink {
2087 let concurrency = manifest_concurrency(®istration.manifest);
2091 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2092 connection_id,
2093 module_id.clone(),
2094 negotiated_ver,
2095 concurrency,
2096 sink,
2097 hello_ack,
2098 ) {
2099 if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
2100 crate::supervise::notify_registration_release();
2101 }
2102 return Ok(vec![control_error_frame(
2103 frame,
2104 forwarding_error_code(&err),
2105 err.to_string(),
2106 )?]);
2107 }
2108 Vec::new()
2109 } else {
2110 vec![hello_ack]
2111 };
2112 self.supervisor.mark_swap_candidate_admitted(&module_id);
2113 info!(
2114 module_id = %module_id,
2115 module_version = %registration.manifest.module_version,
2116 negotiated_ver,
2117 ready = registration.ready,
2118 connection_id = connection_id.get(),
2119 "swap candidate registered; not routable until cutover"
2120 );
2121 Ok(reply)
2122 }
2123
2124 fn build_hello_ack(
2125 &self,
2126 frame: &Frame,
2127 negotiated_ver: u8,
2128 module_id: &str,
2129 ) -> Result<Frame, RouterError> {
2130 let ack = ModuleHelloAckBody {
2131 negotiated_ver,
2132 subc_ops: module_subc_ops(),
2133 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2134 storage: self
2135 .storage_config
2136 .as_ref()
2137 .map(|cfg| cfg.descriptor_for(module_id)),
2138 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2139 };
2140 let body = serde_json::to_vec(&ack).map_err(|err| {
2141 RouterError::backend(
2142 0,
2143 frame.header.corr,
2144 format!("failed to encode HELLO_ACK: {err}"),
2145 )
2146 })?;
2147
2148 Frame::build_with_version(
2149 negotiated_ver,
2150 FrameType::HelloAck,
2151 control_flags(),
2152 0,
2153 0,
2154 frame.header.corr,
2155 body,
2156 )
2157 .map_err(RouterError::FrameBuild)
2158 }
2159
2160 async fn handle_client_control_request(
2161 &self,
2162 ctx: &RouteCtx,
2163 frame: Frame,
2164 request: ClientControlRequest,
2165 ) -> Result<Vec<Frame>, RouterError> {
2166 match request {
2167 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2168 ClientControlRequest::CatalogList { module_id } => {
2169 self.handle_catalog_list(frame, module_id)
2170 }
2171 ClientControlRequest::RouteOpen {
2172 target,
2173 identity,
2174 consumer_identity,
2175 consumer_capabilities,
2176 role_versions,
2177 admission_facts,
2178 scope,
2179 } => {
2180 self.handle_route_open(
2181 ctx,
2182 frame,
2183 RouteOpenRequest {
2184 target,
2185 identity,
2186 consumer_identity,
2187 consumer_capabilities,
2188 role_versions,
2189 admission_facts,
2190 scope,
2191 },
2192 )
2193 .await
2194 }
2195 ClientControlRequest::RoutePoll {
2196 route_channel,
2197 route_epoch,
2198 kind,
2199 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2200 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2201 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2202 self.handle_supervisor_spawn_snapshot(frame)
2203 }
2204 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2205 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2206 }
2207 ClientControlRequest::SupervisorRestart {
2208 module_id,
2209 drain_timeout_ms,
2210 } => {
2211 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms)
2212 .await
2213 }
2214 ClientControlRequest::SupervisorSwap {
2215 module_id,
2216 ready_timeout_ms,
2217 } => {
2218 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms)
2219 .await
2220 }
2221 ClientControlRequest::SupervisorReload { module_id } => {
2222 self.handle_supervisor_reload(frame, module_id).await
2223 }
2224 ClientControlRequest::SupervisorRescan { preview } => {
2225 self.handle_supervisor_rescan(frame, preview).await
2226 }
2227 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2228 self.handle_supervisor_release_reserved(frame, module_id)
2229 .await
2230 }
2231 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2232 self.handle_supervisor_set_enabled(frame, module_id, enabled)
2233 .await
2234 }
2235 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2236 self.handle_supervisor_health_probe(frame, module_id).await
2237 }
2238 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2239 ClientControlRequest::SupervisorRoutes { module_id } => {
2240 self.handle_supervisor_routes(frame, module_id)
2241 }
2242 ClientControlRequest::SupervisorProvenance { module_id } => {
2243 self.handle_supervisor_provenance(frame, module_id).await
2244 }
2245 ClientControlRequest::SupervisorStderrTail {
2246 module_id,
2247 max_lines,
2248 max_bytes,
2249 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2250 ClientControlRequest::SupervisorTerminals { module_id } => {
2251 self.handle_supervisor_terminals(frame, module_id).await
2252 }
2253 }
2254 }
2255
2256 fn handle_module_control_request(
2257 &self,
2258 connection_id: ConnectionId,
2259 frame: Frame,
2260 request: ModuleControlRequestFromModule,
2261 ) -> Result<Vec<Frame>, RouterError> {
2262 match request {
2263 ModuleControlRequestFromModule::CatalogUpdate {
2264 provides,
2265 capabilities,
2266 ready,
2267 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2268 ModuleControlRequestFromModule::LiveRoots {} => {
2269 let registered = self
2270 .registry
2271 .get_module_by_connection(connection_id)
2272 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2273 let Some(registration) = registered else {
2274 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2275 };
2276 let response = self
2277 .forwarding
2278 .live_roots(®istration.manifest.module_id)
2279 .map_err(RouterError::Forwarding)?;
2280 Ok(vec![control_response_body_frame(
2281 &frame,
2282 &response,
2283 "ModuleControlResponseToModule::LiveRoots",
2284 )?])
2285 }
2286 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
2287 self.handle_scope_sync(connection_id, frame, generation, scopes)
2288 }
2289 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
2290 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
2291 }
2292 }
2293 }
2294
2295 fn handle_scope_sync(
2299 &self,
2300 connection_id: ConnectionId,
2301 frame: Frame,
2302 generation: u64,
2303 scopes: Vec<ScopeRecord>,
2304 ) -> Result<Vec<Frame>, RouterError> {
2305 let Some(registration) = self
2306 .registry
2307 .get_module_by_connection(connection_id)
2308 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2309 else {
2310 return Ok(vec![control_error_frame(
2311 &frame,
2312 "not_registered",
2313 "scope.sync requires an active module registration owned by this connection",
2314 )?]);
2315 };
2316 let owner = registration.manifest.module_id;
2317 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
2318 let is_current_launch = |connection: ConnectionId| {
2319 self.hello_launch_nonces
2320 .lock()
2321 .unwrap_or_else(|poisoned| poisoned.into_inner())
2322 .presented(connection, current_nonce.as_deref())
2323 };
2324 let mut table = self
2329 .scopes
2330 .write()
2331 .unwrap_or_else(|poisoned| poisoned.into_inner());
2332 let outcome = table.sync(&owner, connection_id, is_current_launch, generation, scopes);
2333 let drained = match &outcome {
2334 Ok(applied) => self
2335 .forwarding
2336 .publish_scope_changes(&applied.tag_changes)
2337 .map_err(RouterError::Forwarding)?,
2338 Err(_) => Vec::new(),
2339 };
2340 drop(table);
2341 match outcome {
2342 Ok(applied) => {
2343 let counts = ScopeOutcomeCounts::of(&applied.results);
2344 info!(
2345 owner = %owner,
2346 generation,
2347 records = applied.results.len(),
2348 created = counts.created,
2349 replaced = counts.replaced,
2350 updated = counts.updated,
2351 unchanged = counts.unchanged,
2352 refused = counts.refused,
2353 ended = applied.ended.len(),
2354 tag_changes = applied.tag_changes.len(),
2355 routes_closed = drained.len(),
2356 "scope sync accepted"
2357 );
2358 for refused in applied
2364 .results
2365 .iter()
2366 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
2367 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
2368 {
2369 warn!(
2370 owner = %owner,
2371 generation,
2372 scope_ref = %refused.scope_ref,
2373 scope_epoch = refused.scope_epoch,
2374 code = refused.code.as_deref().unwrap_or(""),
2375 "scope record refused"
2376 );
2377 }
2378 self.close_scope_drained_routes(drained);
2379 let response = ModuleControlResponseToModule::ScopeSync {
2380 generation,
2381 results: applied.results,
2382 ended: applied.ended,
2383 };
2384 Ok(vec![control_response_body_frame(
2385 &frame,
2386 &response,
2387 "ModuleControlResponseToModule::ScopeSync",
2388 )?])
2389 }
2390 Err(refusal) => {
2391 info!(
2392 owner = %owner,
2393 generation,
2394 code = refusal.code,
2395 "scope sync refused"
2396 );
2397 Ok(vec![control_error_frame(
2398 &frame,
2399 refusal.code,
2400 refusal.message,
2401 )?])
2402 }
2403 }
2404 }
2405
2406 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
2412 if drained.is_empty() {
2413 return;
2414 }
2415 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
2416 BTreeMap::new();
2417 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
2418 for route in drained {
2419 warn!(
2420 module_id = %route.module_id,
2421 reason = ?route.reason,
2422 client_connection_id = route.client.connection_id.get(),
2423 route_channel = route.client.channel,
2424 "closing route because its scope changed"
2425 );
2426 pushes
2427 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
2428 .or_insert_with(|| (route.reason, Vec::new()))
2429 .1
2430 .push(EndpointRoute {
2431 goodbye_target: route.client.clone(),
2432 principal: Principal::Unverified,
2433 bound_at: Instant::now(),
2434 draining: false,
2435 drain_reason: None,
2436 });
2437 goodbyes.push(route.module);
2438 goodbyes.push(route.client);
2439 }
2440 for ((module_id, _), (reason, routes)) in pushes {
2441 send_route_control_pushes(
2442 &self.forwarding,
2443 routes,
2444 ClientControlPush::RouteClosed {
2445 module_id,
2446 reason,
2447 drained: false,
2448 abandoned: 0,
2449 excluded_subscriptions: 0,
2450 terminal: Some(false),
2451 },
2452 );
2453 }
2454 self.emit_route_goodbyes(goodbyes);
2455 }
2456
2457 fn handle_scope_describe(
2460 &self,
2461 connection_id: ConnectionId,
2462 frame: Frame,
2463 owner: Principal,
2464 scope_ref: String,
2465 ) -> Result<Vec<Frame>, RouterError> {
2466 let registered = self
2467 .registry
2468 .get_module_by_connection(connection_id)
2469 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2470 if registered.is_none() {
2471 return Ok(vec![control_error_frame(
2472 &frame,
2473 "not_registered",
2474 "scope.describe requires an active module registration owned by this connection",
2475 )?]);
2476 }
2477 let description = self
2478 .scopes
2479 .read()
2480 .unwrap_or_else(|poisoned| poisoned.into_inner())
2481 .describe(&owner, &scope_ref);
2482 let owner_configured = match &owner {
2483 Principal::Reserved { module_id } => self.supervisor.get(module_id).is_some(),
2484 _ => false,
2485 };
2486 let response = ModuleControlResponseToModule::ScopeDescribe {
2487 status: description.status,
2488 scope_epoch: description.scope_epoch,
2489 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
2490 owner_synced: description.owner_synced,
2491 owner_configured,
2492 scope: description.stamp,
2493 };
2494 Ok(vec![control_response_body_frame(
2495 &frame,
2496 &response,
2497 "ModuleControlResponseToModule::ScopeDescribe",
2498 )?])
2499 }
2500
2501 fn handle_catalog_update(
2502 &self,
2503 connection_id: ConnectionId,
2504 frame: Frame,
2505 provides: Vec<ProviderRole>,
2506 capabilities: Option<CapabilityDeclarations>,
2507 ready: Option<bool>,
2508 ) -> Result<Vec<Frame>, RouterError> {
2509 self.refresh_capability_requirements();
2510 let Some(registration) = self
2511 .registry
2512 .get_module_by_connection(connection_id)
2513 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2514 else {
2515 return Ok(vec![control_error_frame(
2516 &frame,
2517 "not_registered",
2518 "catalog.update requires an active module registration owned by this connection",
2519 )?]);
2520 };
2521
2522 if let Some(message) =
2523 catalog_update_frozen_field_message(®istration.manifest, &provides)
2524 {
2525 return Ok(vec![control_error_frame(
2526 &frame,
2527 "catalog_update_frozen_field",
2528 message,
2529 )?]);
2530 }
2531
2532 let mut candidate = registration.manifest.clone();
2533 candidate.provides = provides.clone();
2534 candidate.capabilities = capabilities
2535 .clone()
2536 .or_else(|| registration.manifest.capabilities.clone());
2537 if let Err(err) = candidate.validate_capability_grammar() {
2538 return Ok(vec![control_error_frame(
2539 &frame,
2540 "invalid_capability_grammar",
2541 err.to_string(),
2542 )?]);
2543 }
2544
2545 let updated = self
2546 .registry
2547 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
2548 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2549 if updated.is_none() {
2550 return Ok(vec![control_error_frame(
2551 &frame,
2552 "not_registered",
2553 "catalog.update requires an active module registration owned by this connection",
2554 )?]);
2555 }
2556 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
2557 log_duplicate_claim_events(
2558 self.capability_evaluator
2559 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
2560 );
2561 }
2562 if capability_census_trigger(
2563 registration.manifest.capabilities.as_ref(),
2564 updated
2565 .as_ref()
2566 .and_then(|entry| entry.manifest.capabilities.as_ref()),
2567 ) {
2568 self.enforce_capability_denies();
2569 }
2570 self.refresh_capability_requirements();
2571
2572 let response = ModuleControlResponseToModule::CatalogUpdate {};
2573 control_response_body_frame(
2574 &frame,
2575 &response,
2576 "ModuleControlResponseToModule::CatalogUpdate",
2577 )
2578 .map(|frame| vec![frame])
2579 }
2580
2581 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
2582 self.refresh_capability_requirements();
2583 let mut counters = self.counters.snapshot();
2589 if let (Ok((connections_with_routes, max)), Some(obj)) = (
2590 self.forwarding.client_route_concentration(),
2591 counters.as_object_mut(),
2592 ) {
2593 obj.insert(
2594 "client_connections_with_routes".into(),
2595 connections_with_routes.into(),
2596 );
2597 obj.insert("max_routes_on_one_connection".into(), max.into());
2598 }
2599 if let (Some(open_breakers), Some(obj)) = (
2605 self.route_bind_breakers.open_snapshot(),
2606 counters.as_object_mut(),
2607 ) {
2608 obj.insert("route_bind_breakers_open".into(), open_breakers);
2609 }
2610 let response = ClientControlResponse::ServerDescribe {
2611 protocol_ver: PROTOCOL_VERSION,
2612 subc_ops: subc_ops(),
2613 capabilities: self.subc_capabilities.as_ref().to_vec(),
2614 connected_clients: self.connected_clients.count(),
2615 counters: Some(counters),
2616 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
2617 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
2618 capability_requirements: self.capability_requirement_statuses(),
2619 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2620 };
2621 Ok(vec![control_response_body_frame(
2622 &frame,
2623 &response,
2624 "ClientControlResponse::ServerDescribe",
2625 )?])
2626 }
2627
2628 fn handle_catalog_list(
2629 &self,
2630 frame: Frame,
2631 module_id: Option<String>,
2632 ) -> Result<Vec<Frame>, RouterError> {
2633 let (generation, modules) = self.registry.list_modules().map_err(|err| {
2634 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
2635 })?;
2636 let entries = modules
2637 .into_iter()
2638 .filter(|registration| {
2639 module_id
2640 .as_deref()
2641 .map(|wanted| registration.manifest.module_id == wanted)
2642 .unwrap_or(true)
2643 })
2644 .map(|registration| {
2645 let not_ready = self.not_ready_reason(®istration);
2646 let roles = registration.manifest.provides;
2647 CatalogEntry {
2648 module_id: registration.manifest.module_id,
2649 ready: not_ready.is_none(),
2650 not_ready,
2651 module_version: Some(registration.manifest.module_version),
2652 roles,
2653 control_ops: registration.control_ops,
2654 capabilities: registration.manifest.capabilities,
2655 self_signals: registration.manifest.self_signals,
2656 }
2657 })
2658 .collect();
2659 let response = ClientControlResponse::CatalogList {
2660 generation,
2661 modules: entries,
2662 subc_ops: subc_ops(),
2663 };
2664 Ok(vec![control_response_body_frame(
2665 &frame,
2666 &response,
2667 "ClientControlResponse::CatalogList",
2668 )?])
2669 }
2670
2671 fn route_open_principal(
2672 &self,
2673 frame: &Frame,
2674 consumer_identity: Option<ConsumerIdentity>,
2675 ) -> Result<Result<Principal, Frame>, RouterError> {
2676 let Some(consumer_identity) = consumer_identity else {
2677 return Ok(Ok(Principal::Direct));
2678 };
2679
2680 if self.supervisor.spawned_consumer_authorized(
2681 &consumer_identity.module_id,
2682 &consumer_identity.launch_nonce,
2683 ) {
2684 return Ok(Ok(Principal::Reserved {
2685 module_id: consumer_identity.module_id,
2686 }));
2687 }
2688
2689 Ok(Err(control_error_frame(
2690 frame,
2691 "bad_consumer_identity",
2692 format!(
2693 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
2694 consumer_identity.module_id
2695 ),
2696 )?))
2697 }
2698
2699 fn route_open_refusal_frame(
2706 &self,
2707 ctx: &RouteCtx,
2708 frame: &Frame,
2709 module_id: &str,
2710 reason: &'static str,
2711 code: &'static str,
2712 message: impl Into<String>,
2713 ) -> Result<Frame, RouterError> {
2714 self.observe_route_open_refusal(ctx, module_id, reason, code);
2715 control_error_frame(frame, code, message.into())
2716 }
2717
2718 fn route_open_breaker_refusal_frame(
2736 &self,
2737 ctx: &RouteCtx,
2738 frame: &Frame,
2739 module_id: &str,
2740 consecutive_timeouts: u32,
2741 retry_in: Duration,
2742 probe_in_flight: bool,
2743 ) -> Result<Frame, RouterError> {
2744 self.counters
2745 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
2746 debug!(
2747 target: "control",
2748 code = "module_timeout",
2749 module_id = ?module_id,
2750 connection_id = ctx.connection_id.get(),
2751 consecutive_timeouts,
2752 retry_in_ms = retry_in.as_millis() as u64,
2753 probe_in_flight,
2754 "route.open refused by open bind-relay breaker"
2755 );
2756 let detail = if probe_in_flight {
2757 "one probe bind is already in flight; retry once it settles".to_string()
2758 } else {
2759 format!("not relaying for another {retry_in:?}")
2760 };
2761 control_error_frame(
2762 frame,
2763 "module_timeout",
2764 format!(
2765 "module_id '{module_id}' failed {consecutive_timeouts} consecutive route.bind \
2766 relays; {detail}"
2767 ),
2768 )
2769 }
2770
2771 fn observe_route_open_refusal(
2784 &self,
2785 ctx: &RouteCtx,
2786 module_id: &str,
2787 reason: &'static str,
2788 code: &'static str,
2789 ) {
2790 self.counters.increment_route_open_refused(code);
2791 info!(
2792 target: "control",
2793 code,
2794 reason,
2795 module_id = ?module_id,
2796 connection_id = ctx.connection_id.get(),
2797 "route.open refused"
2798 );
2799 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
2800 self.route_outages.record_not_serving(module_id, reason);
2801 }
2802 }
2803
2804 fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
2849 self.route_outages.record_accepted(module_id);
2850 self.counters.increment_route_open_accepted(principal);
2851 info!(
2852 target: "control",
2853 principal,
2854 module_id,
2855 connection_id = ctx.connection_id.get(),
2856 "route.open accepted"
2857 );
2858 }
2859
2860 fn supervised_absent_route_open_refusal_frame(
2861 &self,
2862 ctx: &RouteCtx,
2863 frame: &Frame,
2864 module_id: &str,
2865 code: &'static str,
2866 status: &crate::supervise::ModuleStatus,
2867 ) -> Result<Frame, RouterError> {
2868 self.counters.increment_route_open_refused(code);
2869 info!(
2870 target: "control",
2871 code,
2872 reason = "supervised_not_registered",
2873 module_id = ?module_id,
2874 connection_id = ctx.connection_id.get(),
2875 state = %status.state,
2876 enabled = status.enabled,
2877 live = status.live,
2878 "route.open refused"
2879 );
2880 self.route_outages
2884 .record_not_serving(module_id, "supervised_not_registered");
2885 control_error_frame(
2886 frame,
2887 code,
2888 format!(
2889 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
2890 status.state, status.enabled, status.live
2891 ),
2892 )
2893 }
2894
2895 async fn handle_route_open(
2896 &self,
2897 ctx: &RouteCtx,
2898 frame: Frame,
2899 request: RouteOpenRequest,
2900 ) -> Result<Vec<Frame>, RouterError> {
2901 let RouteOpenRequest {
2902 target,
2903 mut identity,
2904 consumer_identity,
2905 consumer_capabilities,
2906 role_versions,
2907 admission_facts,
2908 scope,
2909 } = request;
2910 let target_module_id = target_module_id(&target).to_string();
2911 debug!(
2912 connection_id = ctx.connection_id.get(),
2913 corr = frame.header.corr,
2914 module_id = %target_module_id,
2915 "handling route.open"
2916 );
2917
2918 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
2924 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
2925 self.observe_route_open_refusal(
2926 ctx,
2927 &target_module_id,
2928 "invalid_role_versions",
2929 error_codes::INVALID_REQUEST,
2930 );
2931 return Ok(vec![control_error_body_frame(
2932 &frame,
2933 ErrorBody {
2934 code: error_codes::INVALID_REQUEST.to_string(),
2935 message: error.to_string(),
2936 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
2937 },
2938 )?]);
2939 }
2940
2941 let Some(registration) = self
2960 .registry
2961 .get_module(&target_module_id)
2962 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2963 else {
2964 if let Some((status, warming)) =
2965 self.supervisor_status(&target_module_id, frame.header.corr)?
2966 {
2967 if status.protocol == ModuleProtocol::None {
2974 return Ok(vec![self.route_open_refusal_frame(
2975 ctx,
2976 &frame,
2977 &target_module_id,
2978 "protocol_none",
2979 error_codes::MODULE_NO_PROTOCOL,
2980 format!(
2981 "module_id '{target_module_id}' is declared protocol: none; \
2982 it speaks no subc wire and serves no routes"
2983 ),
2984 )?]);
2985 }
2986 let code = if warming {
2987 "module_warming"
2988 } else {
2989 "target_unavailable"
2990 };
2991 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
2992 ctx,
2993 &frame,
2994 &target_module_id,
2995 code,
2996 &status,
2997 )?]);
2998 }
2999 if let Some(removed_ago_ms) =
3000 self.supervisor.removal_tombstone_age_ms(&target_module_id)
3001 {
3002 return Ok(vec![self.route_open_refusal_frame(
3003 ctx,
3004 &frame,
3005 &target_module_id,
3006 "removed",
3007 error_codes::MODULE_REMOVED,
3008 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
3009 )?]);
3010 }
3011 return Ok(vec![self.route_open_refusal_frame(
3012 ctx,
3013 &frame,
3014 &target_module_id,
3015 "not_registered",
3016 error_codes::UNKNOWN_MODULE,
3017 format!("module_id '{target_module_id}' is not registered"),
3018 )?]);
3019 };
3020
3021 if !registration.ready {
3025 self.counters
3026 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
3027 info!(
3028 target: "control",
3029 code = error_codes::MODULE_WARMING,
3030 module_id = ?target_module_id,
3031 connection_id = ctx.connection_id.get(),
3032 reason = "declared_not_ready",
3033 "route.open refused"
3034 );
3035 self.route_outages
3038 .record_not_serving(&target_module_id, "declared_not_ready");
3039 return Ok(vec![control_error_body_frame(
3040 &frame,
3041 ErrorBody {
3042 code: error_codes::MODULE_WARMING.to_string(),
3043 message: format!(
3044 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
3045 ),
3046 detail: Some(serde_json::json!({
3047 "reason": "declared_not_ready"
3048 })),
3049 },
3050 )?]);
3051 }
3052
3053 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
3072 self.counters
3073 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
3074 info!(
3075 target: "control",
3076 code = error_codes::MODULE_WARMING,
3077 module_id = ?target_module_id,
3078 connection_id = ctx.connection_id.get(),
3079 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3080 capability = %capability,
3081 "route.open refused"
3082 );
3083 return Ok(vec![control_error_body_frame(
3084 &frame,
3085 ErrorBody {
3086 code: error_codes::MODULE_WARMING.to_string(),
3087 message: format!(
3088 "module_id '{target_module_id}' requires capability '{capability}', \
3089 which no registered module provides; retry"
3090 ),
3091 detail: Some(serde_json::json!({
3092 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3093 "capability": capability,
3094 })),
3095 },
3096 )?]);
3097 }
3098
3099 if !target_has_required_role(&target, ®istration.manifest.provides) {
3100 return Ok(vec![self.route_open_refusal_frame(
3101 ctx,
3102 &frame,
3103 &target_module_id,
3104 "role_not_provided",
3105 "target_unavailable",
3106 format!("module_id '{target_module_id}' does not provide the requested target"),
3107 )?]);
3108 }
3109
3110 if registration.state != ChannelState::Active {
3111 return Ok(vec![self.route_open_refusal_frame(
3112 ctx,
3113 &frame,
3114 &target_module_id,
3115 "registration_not_active",
3116 "target_unavailable",
3117 format!("module_id '{target_module_id}' is not active"),
3118 )?]);
3119 }
3120
3121 if self
3122 .forwarding
3123 .module_is_draining(&target_module_id)
3124 .map_err(RouterError::Forwarding)?
3125 {
3126 return Ok(vec![self.route_open_refusal_frame(
3127 ctx,
3128 &frame,
3129 &target_module_id,
3130 "reloading",
3131 "module_reloading",
3132 format!("module_id '{target_module_id}' is reloading"),
3133 )?]);
3134 }
3135
3136 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
3137 process_liveness.process_live(&target_module_id) == Some(false)
3138 }) {
3139 if process_liveness.process_replacing(&target_module_id) {
3146 return Ok(vec![self.route_open_refusal_frame(
3147 ctx,
3148 &frame,
3149 &target_module_id,
3150 "reloading",
3151 "module_reloading",
3152 format!("module_id '{target_module_id}' is reloading"),
3153 )?]);
3154 }
3155 return Ok(vec![self.route_open_refusal_frame(
3156 ctx,
3157 &frame,
3158 &target_module_id,
3159 "supervisor_not_live",
3160 "target_unavailable",
3161 format!("module_id '{target_module_id}' is not live"),
3162 )?]);
3163 }
3164
3165 if !self
3166 .forwarding
3167 .has_live_module_connection(&target_module_id)
3168 .map_err(RouterError::Forwarding)?
3169 {
3170 return Ok(vec![self.route_open_refusal_frame(
3171 ctx,
3172 &frame,
3173 &target_module_id,
3174 "no_forwarding_connection",
3175 "target_unavailable",
3176 format!("module_id '{target_module_id}' has no live forwarding connection"),
3177 )?]);
3178 }
3179
3180 if let Some(error) =
3181 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
3182 {
3183 self.observe_route_open_refusal(
3184 ctx,
3185 &target_module_id,
3186 "op_not_allowed",
3187 "op_not_allowed",
3188 );
3189 return Ok(vec![error]);
3190 }
3191
3192 let principal = match self.route_open_principal(&frame, consumer_identity)? {
3193 Ok(principal) => principal,
3194 Err(error) => {
3195 self.observe_route_open_refusal(
3196 ctx,
3197 &target_module_id,
3198 "bad_consumer_identity",
3199 "bad_consumer_identity",
3200 );
3201 return Ok(vec![error]);
3202 }
3203 };
3204
3205 if let Principal::Reserved {
3209 module_id: opening_module_id,
3210 } = &principal
3211 {
3212 if let Some(opening_registration) = self
3213 .registry
3214 .get_module(opening_module_id)
3215 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3216 {
3217 if let Some(capability) =
3218 denied_capability(&opening_registration.manifest, ®istration.manifest)
3219 {
3220 warn!(
3221 opening_module_id,
3222 target_module_id,
3223 capability,
3224 "refusing route.open because an attested capability deny edge matches"
3225 );
3226 return Ok(vec![self.route_open_refusal_frame(
3227 ctx,
3228 &frame,
3229 &target_module_id,
3230 "capability_deny_edge",
3231 "capability_forbidden",
3232 format!(
3233 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
3234 ),
3235 )?]);
3236 }
3237 }
3238 }
3239
3240 if admission_facts.is_some() {
3241 let carrier_matches = matches!(
3242 &principal,
3243 Principal::Reserved { module_id }
3244 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
3245 );
3246 if !carrier_matches {
3247 return Ok(vec![self.route_open_refusal_frame(
3248 ctx,
3249 &frame,
3250 &target_module_id,
3251 "admission_facts_carrier_not_permitted",
3252 "admission_facts_not_permitted",
3253 "admission facts may only be carried by the configured reserved module",
3254 )?]);
3255 }
3256
3257 let target_allowed = self
3258 .admission_facts_targets
3259 .as_ref()
3260 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
3261 if !target_allowed {
3262 return Ok(vec![self.route_open_refusal_frame(
3263 ctx,
3264 &frame,
3265 &target_module_id,
3266 "admission_facts_target_not_listed",
3267 "admission_facts_target_not_allowed",
3268 format!(
3269 "admission facts are not permitted for target module_id '{target_module_id}'"
3270 ),
3271 )?]);
3272 }
3273
3274 }
3278
3279 let (bound_scope, scope_stamp) = match scope {
3285 None => (None, None),
3286 Some(selector) => {
3287 let owner_configured = match &selector.owner {
3288 Principal::Reserved { module_id } => self.supervisor.get(module_id).is_some(),
3289 _ => false,
3290 };
3291 let admitted = self
3292 .scopes
3293 .read()
3294 .unwrap_or_else(|poisoned| poisoned.into_inner())
3295 .admit(&principal, &target_module_id, &selector, owner_configured);
3296 match admitted {
3297 Ok(admission) => (
3298 Some(BoundScope {
3299 owner: admission.owner,
3300 scope_ref: admission.stamp.scope_ref.clone(),
3301 tag: admission.tag,
3302 }),
3303 Some(admission.stamp),
3304 ),
3305 Err(refusal) => {
3306 return Ok(vec![self.route_open_refusal_frame(
3307 ctx,
3308 &frame,
3309 &target_module_id,
3310 refusal.code,
3311 refusal.code,
3312 refusal.message,
3313 )?]);
3314 }
3315 }
3316 }
3317 };
3318
3319 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
3341 Ok(project_root) => project_root,
3342 Err(err) => {
3343 return Ok(vec![control_error_frame(
3344 &frame,
3345 "invalid_project_root",
3346 err.to_string(),
3347 )?])
3348 }
3349 };
3350 identity.project_root = project_root.as_path().to_path_buf();
3351
3352 let _concurrency_guard = match self
3363 .route_bind_concurrency
3364 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
3365 {
3366 Ok(guard) => guard,
3367 Err(in_flight) => {
3368 return Ok(vec![self.route_open_target_capacity_refusal(
3369 ctx,
3370 &frame,
3371 &target_module_id,
3372 in_flight,
3373 )?]);
3374 }
3375 };
3376
3377 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
3380 RouteBindAdmission::Admitted { guard, probe } => {
3381 if probe {
3382 info!(
3383 module_id = %target_module_id,
3384 connection_id = ctx.connection_id.get(),
3385 "route.bind breaker half-open: admitting one probe"
3386 );
3387 }
3388 guard
3389 }
3390 RouteBindAdmission::Refused {
3391 consecutive_timeouts,
3392 retry_in,
3393 probe_in_flight,
3394 } => {
3395 return Ok(vec![self.route_open_breaker_refusal_frame(
3396 ctx,
3397 &frame,
3398 &target_module_id,
3399 consecutive_timeouts,
3400 retry_in,
3401 probe_in_flight,
3402 )?]);
3403 }
3404 };
3405
3406 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
3411 let relay_deadline = Instant::now() + route_bind_relay_timeout;
3412 let pending = match self
3413 .forwarding
3414 .begin_route_bind_relay_for(
3415 ctx.connection_id,
3416 ctx.egress.clone(),
3417 response_version(&frame),
3418 frame.header.corr,
3419 &target_module_id,
3420 principal.clone(),
3421 bound_scope,
3422 Some(project_root),
3423 relay_deadline,
3424 )
3425 .await
3426 {
3427 Ok(pending) => pending,
3428 Err(err) => {
3429 return Ok(vec![self.route_open_refusal_frame(
3430 ctx,
3431 &frame,
3432 &target_module_id,
3433 "relay_reservation_failed",
3434 forwarding_error_code(&err),
3435 err.to_string(),
3436 )?])
3437 }
3438 };
3439 let crate::forwarding::PendingRouteBindRelay {
3440 endpoint,
3441 module_sink,
3442 negotiated_ver,
3443 client_channel,
3444 client_epoch,
3445 module_channel,
3446 module_epoch,
3447 corr: relay_corr,
3448 receiver,
3449 } = pending;
3450 let mut reservation =
3451 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
3452
3453 debug!(
3454 connection_id = ctx.connection_id.get(),
3455 client_channel,
3456 client_epoch,
3457 module_channel,
3458 module_epoch,
3459 "reserved route handle pair"
3460 );
3461 let principal_label = match &principal {
3464 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
3465 Principal::Direct => "direct".to_string(),
3466 other => format!("{other:?}"),
3467 };
3468 let relay = ModuleControlRequest::RouteBind {
3469 route_channel: module_channel,
3470 epoch: module_epoch,
3471 target,
3472 identity,
3473 principal: Some(principal),
3474 consumer_capabilities,
3475 role_versions,
3476 admission_facts,
3477 scope: scope_stamp,
3478 };
3479 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
3480 RouterError::backend(
3481 0,
3482 frame.header.corr,
3483 format!("failed to encode route.bind request: {err}"),
3484 )
3485 })?;
3486 let relay_frame = Frame::build_with_version(
3487 negotiated_ver,
3488 FrameType::Request,
3489 control_flags(),
3490 0,
3491 0,
3492 relay_corr,
3493 relay_body,
3494 )
3495 .map_err(RouterError::FrameBuild)?;
3496
3497 if let Err(err) = module_sink.send(relay_frame).await {
3498 reservation.release_and_disarm();
3499 return Ok(vec![self.route_open_refusal_frame(
3500 ctx,
3501 &frame,
3502 &target_module_id,
3503 "relay_send_failed",
3504 "target_unavailable",
3505 err.to_string(),
3506 )?]);
3507 }
3508
3509 if !self
3510 .forwarding
3511 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
3512 .map_err(RouterError::Forwarding)?
3513 {
3514 self.send_abandoned_route_bind_goodbye(
3515 &module_sink,
3516 negotiated_ver,
3517 module_channel,
3518 module_epoch,
3519 );
3520 }
3521
3522 match timeout_at(relay_deadline, receiver).await {
3523 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
3524 reservation.disarm();
3525 if breaker.record_accepted() {
3526 info!(
3527 module_id = %target_module_id,
3528 "route.bind breaker closed: the probe was accepted"
3529 );
3530 }
3531 self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
3532 Ok(Vec::new())
3533 }
3534 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
3535 reservation.release_and_disarm();
3536 breaker.record_inconclusive();
3540 let scope_code = match body.code.as_str() {
3544 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
3545 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
3546 _ => None,
3547 };
3548 if let Some(code) = scope_code {
3549 self.observe_route_open_refusal(
3550 ctx,
3551 &target_module_id,
3552 "scope_changed_before_commit",
3553 code,
3554 );
3555 return Ok(vec![control_error_body_frame(&frame, body)?]);
3556 }
3557 self.counters
3558 .increment_route_open_refused("module_rejected");
3559 info!(
3560 target: "control",
3561 code = "module_rejected",
3562 module_code = ?body.code,
3563 module_id = ?target_module_id,
3564 connection_id = ctx.connection_id.get(),
3565 "route.open refused"
3566 );
3567 Ok(vec![control_error_body_frame(&frame, body)?])
3568 }
3569 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
3570 reservation.release_and_disarm();
3571 breaker.record_inconclusive();
3572 tracing::warn!(
3579 module_id = %target_module_id,
3580 "route.bind relay abandoned: {message}"
3581 );
3582 Ok(vec![self.route_open_refusal_frame(
3583 ctx,
3584 &frame,
3585 &target_module_id,
3586 "relay_abandoned",
3587 "target_unavailable",
3588 message,
3589 )?])
3590 }
3591 Ok(Err(_)) => {
3592 reservation.release_and_disarm();
3593 breaker.record_inconclusive();
3594 Ok(vec![self.route_open_refusal_frame(
3595 ctx,
3596 &frame,
3597 &target_module_id,
3598 "relay_waiter_canceled",
3599 "target_unavailable",
3600 "route.bind relay waiter was canceled before the module responded",
3601 )?])
3602 }
3603 Err(_) => {
3604 reservation.release_and_disarm();
3605 if let Some(opened) = breaker.record_timeout(
3609 self.route_bind_breaker_threshold,
3610 self.route_bind_breaker_cooldown,
3611 ) {
3612 warn!(
3613 module_id = %target_module_id,
3614 consecutive_timeouts = opened.consecutive_timeouts,
3615 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
3616 reopened_after_probe = opened.reopened_after_probe,
3617 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
3618 );
3619 }
3620 tracing::warn!(
3628 module_id = %target_module_id,
3629 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
3630 "route.bind relay timed out: module did not ack within budget"
3631 );
3632 Ok(vec![self.route_open_refusal_frame(
3633 ctx,
3634 &frame,
3635 &target_module_id,
3636 "relay_timed_out",
3637 "module_timeout",
3638 format!(
3639 "module_id '{target_module_id}' did not answer route.bind within {:?}",
3640 route_bind_relay_timeout
3641 ),
3642 )?])
3643 }
3644 }
3645 }
3646
3647 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3648 let response = ClientControlResponse::SupervisorSpawnSnapshot {
3649 snapshot: self.supervisor.spawn_snapshot(),
3650 };
3651 Ok(vec![control_response_body_frame(
3652 &frame,
3653 &response,
3654 "ClientControlResponse::SupervisorSpawnSnapshot",
3655 )?])
3656 }
3657
3658 fn handle_supervisor_spawn_subscribe(
3659 &self,
3660 ctx: &RouteCtx,
3661 frame: Frame,
3662 since: Option<SpawnCursor>,
3663 ) -> Result<Vec<Frame>, RouterError> {
3664 match self.supervisor.subscribe_spawns(
3665 ctx.connection_id,
3666 frame.header.corr,
3667 response_version(&frame),
3668 since,
3669 ctx.egress.clone(),
3670 ) {
3671 Ok(()) => Ok(Vec::new()),
3672 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
3673 Ok(vec![control_error_body_frame(
3674 &frame,
3675 ErrorBody {
3676 code: "spawn_cursor_incarnation_mismatch".to_string(),
3677 message: "spawn cursor belongs to a different daemon incarnation"
3678 .to_string(),
3679 detail: Some(serde_json::json!({
3680 "current_daemon_incarnation": current
3681 })),
3682 },
3683 )?])
3684 }
3685 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
3686 &frame,
3687 ErrorBody {
3688 code: "spawn_cursor_too_old".to_string(),
3689 message: "spawn cursor predates the retained event ring".to_string(),
3690 detail: Some(serde_json::json!({
3691 "oldest_retained_cursor": oldest
3692 })),
3693 },
3694 )?]),
3695 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
3696 0,
3697 frame.header.corr,
3698 format!("failed to open supervisor spawn subscription: {error}"),
3699 )),
3700 }
3701 }
3702
3703 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3704 let generation = self
3705 .registry
3706 .generation()
3707 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3708 let mut modules = Vec::new();
3709 for module in self.supervisor.list() {
3710 let status = module.status_for_control("list").map_err(|err| {
3711 RouterError::backend(
3712 0,
3713 frame.header.corr,
3714 format!("failed to read supervisor status: {err}"),
3715 )
3716 })?;
3717 let (configured, _) = module.configuration().map_err(|err| {
3718 RouterError::backend(
3719 0,
3720 frame.header.corr,
3721 format!("failed to read module configuration: {err}"),
3722 )
3723 })?;
3724 let image = module.running_image_agreement().await;
3726 let resources = Some(module.child_resource_usage());
3729 let pending_reload = Some(reload_verdict(
3730 &configured.program,
3731 status.spawned_from.as_deref(),
3732 image,
3733 ));
3734 modules.push(SupervisorEntry {
3735 launch_nonce_env: Some(!cfg!(unix)),
3738 module_id: status.module_id,
3739 state: status.state.to_string(),
3740 enabled: status.enabled,
3741 live: status.live,
3742 protocol: status.protocol,
3743 health: status.health.status,
3744 pending_reload,
3745 last_probe_ms: status.health.last_probe_ms,
3746 last_exit_code: status.last_exit.as_ref().and_then(|e| e.code),
3747 last_exit_signal: status.last_exit.as_ref().and_then(|e| e.signal),
3748 last_exit_ms: status.last_exit.as_ref().map(|e| e.at_ms),
3749 last_exit_kind: status.last_exit.as_ref().map(|e| e.kind.into()),
3750 restart_count: Some(status.restart_count),
3751 max_restarts: Some(status.max_restarts),
3752 lifetime_restarts: Some(status.lifetime_restarts),
3753 spawn_generation: Some(status.spawn_generation),
3754 restart_window_secs: Some(status.restart_window.as_secs()),
3755 drain_timeout_ms: Some(status.drain_timeout.as_millis() as u64),
3756 restart_backoff_ms: Some(status.restart_backoff.as_millis() as u64),
3757 restart_max_backoff_ms: Some(status.restart_max_backoff.as_millis() as u64),
3758 resources,
3759 });
3760 }
3761 let response = ClientControlResponse::SupervisorList {
3762 generation,
3763 modules,
3764 };
3765 Ok(vec![control_response_body_frame(
3766 &frame,
3767 &response,
3768 "ClientControlResponse::SupervisorList",
3769 )?])
3770 }
3771
3772 fn handle_supervisor_stderr_tail(
3773 &self,
3774 frame: Frame,
3775 module_id: String,
3776 max_lines: Option<u32>,
3777 max_bytes: Option<u32>,
3778 ) -> Result<Vec<Frame>, RouterError> {
3779 let Some(module) = self.supervisor.get(&module_id) else {
3780 return Ok(vec![control_error_frame(
3781 &frame,
3782 "unknown_module",
3783 format!("module_id '{module_id}' is not supervised"),
3784 )?]);
3785 };
3786
3787 let snapshot = module.stderr_tail(
3788 max_lines.map(|value| value as usize),
3789 max_bytes.map(|value| value as usize),
3790 );
3791
3792 let response = ClientControlResponse::SupervisorStderrTail {
3793 module_id,
3794 tail: StderrTail {
3795 capture: match snapshot.capture {
3796 CaptureState::Captured => StderrCaptureState::Captured,
3797 CaptureState::Incomplete { reason } => {
3798 StderrCaptureState::Incomplete { reason }
3799 }
3800 CaptureState::NotCaptured { reason } => {
3801 StderrCaptureState::NotCaptured { reason }
3802 }
3803 },
3804 entries: snapshot
3805 .entries
3806 .into_iter()
3807 .map(|entry| match entry {
3808 TailEntry::Line {
3809 text,
3810 truncated,
3811 at_ms,
3812 } => StderrTailEntry::Line {
3813 text,
3814 truncated,
3815 at_ms,
3816 },
3817 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
3818 })
3819 .collect(),
3820 dropped_lines: snapshot.dropped_lines,
3821 },
3822 };
3823 Ok(vec![control_response_body_frame(
3824 &frame,
3825 &response,
3826 "ClientControlResponse::SupervisorStderrTail",
3827 )?])
3828 }
3829
3830 async fn handle_supervisor_terminals(
3831 &self,
3832 frame: Frame,
3833 module_id: String,
3834 ) -> Result<Vec<Frame>, RouterError> {
3835 let Some(module) = self.supervisor.get(&module_id) else {
3836 return Ok(vec![control_error_frame(
3837 &frame,
3838 "unknown_module",
3839 format!("module_id '{module_id}' is not supervised"),
3840 )?]);
3841 };
3842
3843 let terminals = module
3846 .read_durable_terminal_history()
3847 .await
3848 .map_err(|error| {
3849 RouterError::backend(
3850 0,
3851 frame.header.corr,
3852 format!("failed to read terminal history: {error}"),
3853 )
3854 })?;
3855 let response = ClientControlResponse::SupervisorTerminals {
3856 module_id,
3857 terminals,
3858 };
3859 Ok(vec![control_response_body_frame(
3860 &frame,
3861 &response,
3862 "ClientControlResponse::SupervisorTerminals",
3863 )?])
3864 }
3865
3866 fn handle_supervisor_routes(
3867 &self,
3868 frame: Frame,
3869 module_id: Option<String>,
3870 ) -> Result<Vec<Frame>, RouterError> {
3871 let modules = self
3872 .forwarding
3873 .route_census(module_id.as_deref())
3874 .map_err(RouterError::Forwarding)?
3875 .into_iter()
3876 .map(|(module_id, routes)| SupervisorRouteModule {
3877 module_id,
3878 routes: routes
3879 .into_iter()
3880 .map(|route| SupervisorRoute {
3881 consumer: match route.principal {
3882 Principal::Reserved { module_id } => {
3883 SupervisorRouteConsumer::Reserved { module_id }
3884 }
3885 Principal::Direct | Principal::Unverified => {
3886 SupervisorRouteConsumer::Direct {
3887 connection_id: route.goodbye_target.connection_id.get(),
3888 }
3889 }
3890 },
3891 age_ms: Instant::now()
3892 .saturating_duration_since(route.bound_at)
3893 .as_millis()
3894 .try_into()
3895 .unwrap_or(u64::MAX),
3896 draining: route.draining,
3897 drain_reason: route.drain_reason,
3898 })
3899 .collect(),
3900 })
3901 .collect();
3902 let response = ClientControlResponse::SupervisorRoutes { modules };
3903 Ok(vec![control_response_body_frame(
3904 &frame,
3905 &response,
3906 "ClientControlResponse::SupervisorRoutes",
3907 )?])
3908 }
3909
3910 async fn handle_supervisor_provenance(
3911 &self,
3912 frame: Frame,
3913 module_id: Option<String>,
3914 ) -> Result<Vec<Frame>, RouterError> {
3915 let mut selected = if let Some(module_id) = module_id {
3916 let Some(module) = self.supervisor.get(&module_id) else {
3917 return Ok(vec![control_error_frame(
3918 &frame,
3919 "unknown_module",
3920 format!("module_id '{module_id}' is not supervised"),
3921 )?]);
3922 };
3923 vec![module]
3924 } else {
3925 self.supervisor.list()
3926 };
3927
3928 let mut modules = Vec::with_capacity(selected.len());
3929 for module in selected.drain(..) {
3930 let status = module.status().map_err(|err| {
3931 RouterError::backend(
3932 0,
3933 frame.header.corr,
3934 format!("failed to read supervisor status: {err}"),
3935 )
3936 })?;
3937 let module_declared = self
3938 .registry
3939 .get_module(&status.module_id)
3940 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3941 .and_then(|registration| registration.manifest.provenance)
3942 .map(|build| ModuleDeclaredProvenance::Reported { build })
3943 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
3944 #[cfg(test)]
3945 let running_image = match &self.provenance_probe_override {
3946 Some(result) => result.clone(),
3947 None => module.running_image_agreement().await,
3948 };
3949 #[cfg(not(test))]
3950 let running_image = module.running_image_agreement().await;
3951 modules.push(SupervisorModuleProvenance {
3952 module_id: status.module_id,
3953 module_declared,
3954 daemon_observed: SupervisorObservedProcess {
3955 pid: status.pid,
3956 spawned_at_ms: status.spawned_at_ms,
3957 spawned_from: status.spawned_from,
3958 running_image,
3959 },
3960 });
3961 }
3962 let daemon = SupervisorDaemonProvenance {
3963 daemon_build: self.daemon_provenance.build.clone(),
3964 daemon_observed: DaemonObservedProcess {
3965 pid: self.daemon_provenance.pid,
3966 started_at_ms: self
3967 .daemon_provenance
3968 .start_clock
3969 .map(|clock| clock.started_at_ms())
3970 .or(self.daemon_provenance.started_at_ms),
3971 running_image: self
3972 .daemon_provenance
3973 .probe
3974 .observe(
3975 self.daemon_provenance.pid,
3976 self.daemon_provenance.executable_path.as_deref(),
3977 self.daemon_provenance.executable_identity,
3978 self.daemon_provenance.process_start_time,
3979 )
3980 .await,
3981 },
3982 };
3983 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
3984 Ok(vec![control_response_body_frame(
3985 &frame,
3986 &response,
3987 "ClientControlResponse::SupervisorProvenance",
3988 )?])
3989 }
3990
3991 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3992 self.refresh_capability_requirements();
3993 let generation = self
3994 .registry
3995 .generation()
3996 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3997 let modules = self
3998 .supervisor
3999 .list()
4000 .into_iter()
4001 .map(|module| {
4002 let status = module.status_for_control("health").map_err(|err| {
4003 RouterError::backend(
4004 0,
4005 frame.header.corr,
4006 format!("failed to read supervisor health: {err}"),
4007 )
4008 })?;
4009 let module_id = status.module_id;
4010 let capability_detail = self
4011 .capability_evaluator
4012 .required_problem_detail(&module_id);
4013 Ok(SupervisorHealthEntry {
4014 module_id,
4015 status: status.health.status,
4016 detail: append_capability_problem_detail(
4017 status.health.detail,
4018 capability_detail,
4019 ),
4020 metrics: status.health.metrics,
4021 consecutive_failures: status.health.consecutive_failures,
4022 late_answer_count: status.health.late_answer_count,
4023 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
4024 last_action: status.health.last_action,
4025 last_action_ms: status.health.last_action_ms,
4026 last_probe_ms: status.health.last_probe_ms,
4027 })
4028 })
4029 .collect::<Result<Vec<_>, RouterError>>()?;
4030 let response = ClientControlResponse::SupervisorHealth {
4031 generation,
4032 modules,
4033 };
4034 Ok(vec![control_response_body_frame(
4035 &frame,
4036 &response,
4037 "ClientControlResponse::SupervisorHealth",
4038 )?])
4039 }
4040
4041 async fn handle_supervisor_restart(
4042 &self,
4043 frame: Frame,
4044 module_id: String,
4045 drain_timeout_ms: Option<u64>,
4046 ) -> Result<Vec<Frame>, RouterError> {
4047 let operation_lock = self.supervisor.operation_lock();
4048 let _operation_guard = operation_lock.lock().await;
4049 let Some(module) = self.supervisor.get(&module_id) else {
4050 return Ok(vec![control_error_frame(
4051 &frame,
4052 "unknown_module",
4053 format!("module_id '{module_id}' is not supervised"),
4054 )?]);
4055 };
4056
4057 self.route_outages.mark_operator_action(&module_id);
4058 if let Err(err) = module.restart(drain_timeout_ms).await {
4059 self.route_outages
4060 .operator_action_ended_unrefused(&module_id);
4061 let (code, message) = match err {
4062 crate::supervise::SuperviseError::Disabled { .. } => {
4063 ("module_disabled", err.to_string())
4064 }
4065 crate::supervise::SuperviseError::SwapInProgress { .. } => {
4066 ("swap_in_progress", err.to_string())
4067 }
4068 _ => (
4069 "target_unavailable",
4070 format!("failed to restart module_id '{module_id}': {err}"),
4071 ),
4072 };
4073 return Ok(vec![control_error_frame(&frame, code, message)?]);
4074 }
4075
4076 let response = ClientControlResponse::SupervisorAck {
4077 module_id,
4078 applied: true,
4079 };
4080 Ok(vec![control_response_body_frame(
4081 &frame,
4082 &response,
4083 "ClientControlResponse::SupervisorAck",
4084 )?])
4085 }
4086
4087 async fn handle_supervisor_swap(
4091 &self,
4092 frame: Frame,
4093 module_id: String,
4094 ready_timeout_ms: Option<u64>,
4095 ) -> Result<Vec<Frame>, RouterError> {
4096 let module = {
4103 let operation_lock = self.supervisor.operation_lock();
4104 let _operation_guard = operation_lock.lock().await;
4105 self.supervisor.get(&module_id)
4106 };
4107 let Some(module) = module else {
4108 return Ok(vec![control_error_frame(
4109 &frame,
4110 "unknown_module",
4111 format!("module_id '{module_id}' is not supervised"),
4112 )?]);
4113 };
4114
4115 self.route_outages.mark_operator_action(&module_id);
4116 if let Err(err) = module
4117 .swap(ready_timeout_ms.map(Duration::from_millis))
4118 .await
4119 {
4120 self.route_outages
4121 .operator_action_ended_unrefused(&module_id);
4122 use crate::supervise::SuperviseError;
4123 let message = err.to_string();
4124 let error = match err {
4125 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
4126 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
4127 code: "swap_refused".to_string(),
4128 message,
4129 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
4130 },
4131 SuperviseError::SwapFailed {
4132 arm,
4133 candidate_exit,
4134 ..
4135 } => ErrorBody {
4136 code: "swap_failed".to_string(),
4137 message,
4138 detail: Some(serde_json::json!({
4139 "arm": arm.as_str(),
4140 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
4141 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
4142 })),
4143 },
4144 _ => ErrorBody::new(
4145 "target_unavailable",
4146 format!("failed to swap module_id '{module_id}': {message}"),
4147 ),
4148 };
4149 return Ok(vec![control_error_body_frame(&frame, error)?]);
4150 }
4151 self.route_outages
4155 .operator_action_ended_unrefused(&module_id);
4156
4157 let response = ClientControlResponse::SupervisorAck {
4158 module_id,
4159 applied: true,
4160 };
4161 Ok(vec![control_response_body_frame(
4162 &frame,
4163 &response,
4164 "ClientControlResponse::SupervisorAck",
4165 )?])
4166 }
4167
4168 async fn handle_supervisor_reload(
4169 &self,
4170 frame: Frame,
4171 module_id: String,
4172 ) -> Result<Vec<Frame>, RouterError> {
4173 let operation_lock = self.supervisor.operation_lock();
4174 let _operation_guard = operation_lock.lock().await;
4175 let Some(module) = self.supervisor.get(&module_id) else {
4176 return Ok(vec![control_error_frame(
4177 &frame,
4178 "unknown_module",
4179 format!("module_id '{module_id}' is not supervised"),
4180 )?]);
4181 };
4182
4183 self.route_outages.mark_operator_action(&module_id);
4184 if let Err(err) = module.reload().await {
4185 self.route_outages
4186 .operator_action_ended_unrefused(&module_id);
4187 let (code, message) = match err {
4188 crate::supervise::SuperviseError::Disabled { .. } => {
4189 ("module_disabled", err.to_string())
4190 }
4191 crate::supervise::SuperviseError::SwapInProgress { .. } => {
4192 ("swap_in_progress", err.to_string())
4193 }
4194 _ => (
4195 "reload_failed",
4196 format!("failed to reload module_id '{module_id}': {err}"),
4197 ),
4198 };
4199 return Ok(vec![control_error_frame(&frame, code, message)?]);
4200 }
4201
4202 let response = ClientControlResponse::SupervisorAck {
4203 module_id,
4204 applied: true,
4205 };
4206 Ok(vec![control_response_body_frame(
4207 &frame,
4208 &response,
4209 "ClientControlResponse::SupervisorAck",
4210 )?])
4211 }
4212
4213 async fn handle_supervisor_rescan(
4214 &self,
4215 frame: Frame,
4216 preview: bool,
4217 ) -> Result<Vec<Frame>, RouterError> {
4218 let Some(context) = self.rescan.clone() else {
4219 return Ok(vec![control_error_frame(
4220 &frame,
4221 "rescan_unavailable",
4222 "the daemon was not started with a reloadable config path".to_string(),
4223 )?]);
4224 };
4225
4226 let operation_lock = self.supervisor.operation_lock();
4227 let _operation_guard = operation_lock.lock().await;
4228 let loaded = match crate::daemon_config::load(&context.config_path) {
4229 Ok(config) => config,
4230 Err(err) => {
4231 return Ok(vec![control_error_frame(
4232 &frame,
4233 "invalid_daemon_config",
4234 format!("supervisor rescan rejected daemon config: {err}"),
4235 )?])
4236 }
4237 };
4238 let Some(config) = loaded else {
4246 return Ok(vec![control_error_frame(
4247 &frame,
4248 "invalid_daemon_config",
4249 format!(
4250 "daemon config not found at {}; refusing to rescan (an absent config would \
4251 retire every supervised module)",
4252 context.config_path.display()
4253 ),
4254 )?]);
4255 };
4256 let (
4257 configured_port,
4258 storage_config,
4259 admission_facts_carrier_module_id,
4260 admission_facts_targets,
4261 scope_authority_owners,
4262 modules,
4263 reserved_capabilities,
4264 ) = (
4265 config.port,
4266 config.storage,
4267 config.admission_facts_carrier_module_id,
4268 config.admission_facts_targets,
4269 config.scope_authority_owners,
4270 config.modules,
4271 config.reserved_capabilities,
4272 );
4273
4274 let mut restart_required = Vec::new();
4283 for section in RestartRequiredSection::ALL {
4284 let changed = match section {
4285 RestartRequiredSection::Port => configured_port != context.configured_port,
4286 RestartRequiredSection::Storage => storage_config != context.storage_config,
4287 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
4288 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
4289 }
4290 RestartRequiredSection::AdmissionFactsTargets => {
4291 admission_facts_targets != context.admission_facts_targets
4292 }
4293 RestartRequiredSection::ScopeAuthorityOwners => {
4294 scope_authority_owners != context.scope_authority_owners
4295 }
4296 };
4297 if changed {
4298 restart_required.push(section.label().to_string());
4299 }
4300 }
4301 if !restart_required.is_empty() {
4302 warn!(
4303 config_path = %context.config_path.display(),
4304 sections = %restart_required.join(", "),
4305 "daemon config changed outside the modules section; restart the daemon to apply those changes"
4306 );
4307 }
4308
4309 for configured in &modules {
4310 if let Err(err) = validate_spec(&configured.module_spec()) {
4311 return Ok(vec![control_error_frame(
4312 &frame,
4313 "invalid_daemon_config",
4314 format!("supervisor rescan rejected daemon config: {err}"),
4315 )?]);
4316 }
4317 }
4318
4319 let configured_capabilities = modules
4320 .iter()
4321 .map(|module| (module.module_id.clone(), module.enabled))
4322 .collect::<Vec<_>>();
4323 let preview_capability_warnings = if preview {
4324 let (_, registrations) = self.runtime_capability_snapshot()?;
4325 let current_modules = self
4326 .supervisor
4327 .list()
4328 .into_iter()
4329 .map(|module| module.module_id().to_string())
4330 .collect::<BTreeSet<_>>();
4331 let resulting_modules = configured_capabilities.clone();
4332 let removed = current_modules
4333 .into_iter()
4334 .filter(|module_id| {
4335 !resulting_modules
4336 .iter()
4337 .any(|(configured_id, _)| configured_id == module_id)
4338 })
4339 .collect::<Vec<_>>();
4340 self.capability_evaluator.preview_removal_warnings(
4341 resulting_modules,
4342 &removed,
4343 ®istrations,
4344 )
4345 } else {
4346 Vec::new()
4347 };
4348 let result = match self
4349 .reconcile_supervised_modules(&context.supervisor, modules, preview)
4350 .await
4351 {
4352 Ok(result) => result,
4353 Err(message) => {
4354 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
4355 }
4356 };
4357 if !preview {
4358 self.capability_evaluator
4359 .configure(configured_capabilities, reserved_capabilities);
4360 self.capability_evaluator.wake_deadline_loop();
4361 self.refresh_capability_requirements();
4362 }
4363 let mut result = result;
4364 result.restart_required = restart_required;
4365 result.capability_warnings = preview_capability_warnings;
4366 let response = ClientControlResponse::SupervisorRescan { result };
4367 Ok(vec![control_response_body_frame(
4368 &frame,
4369 &response,
4370 "ClientControlResponse::SupervisorRescan",
4371 )?])
4372 }
4373
4374 async fn handle_supervisor_release_reserved(
4375 &self,
4376 frame: Frame,
4377 module_id: String,
4378 ) -> Result<Vec<Frame>, RouterError> {
4379 let Some(context) = self.rescan.clone() else {
4380 return Ok(vec![control_error_frame(
4381 &frame,
4382 "release_unavailable",
4383 "reserved-id release requires a daemon started with a reloadable config path",
4384 )?]);
4385 };
4386 let operation_lock = self.supervisor.operation_lock();
4387 let _operation_guard = operation_lock.lock().await;
4388 let loaded = match crate::daemon_config::load(&context.config_path) {
4389 Ok(Some(config)) => config,
4390 Ok(None) => {
4391 return Ok(vec![control_error_frame(
4392 &frame,
4393 "invalid_daemon_config",
4394 format!(
4395 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
4396 context.config_path.display()
4397 ),
4398 )?])
4399 }
4400 Err(err) => {
4401 return Ok(vec![control_error_frame(
4402 &frame,
4403 "invalid_daemon_config",
4404 format!("unable to verify reserved-id release against daemon config: {err}"),
4405 )?])
4406 }
4407 };
4408 if loaded
4409 .modules
4410 .iter()
4411 .any(|configured| configured.module_id == module_id)
4412 {
4413 return Ok(vec![control_error_frame(
4414 &frame,
4415 "reserved_module_configured",
4416 format!(
4417 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
4418 ),
4419 )?]);
4420 }
4421 if !self.supervisor.release_retained_reserved_gate(&module_id) {
4422 return Ok(vec![control_error_frame(
4423 &frame,
4424 "reserved_gate_not_retained",
4425 format!(
4426 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
4427 ),
4428 )?]);
4429 }
4430
4431 let response = ClientControlResponse::SupervisorAck {
4432 module_id,
4433 applied: true,
4434 };
4435 Ok(vec![control_response_body_frame(
4436 &frame,
4437 &response,
4438 "ClientControlResponse::SupervisorAck",
4439 )?])
4440 }
4441
4442 async fn reconcile_supervised_modules(
4451 &self,
4452 supervisor: &Supervisor,
4453 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
4454 preview: bool,
4455 ) -> Result<SupervisorRescanResult, String> {
4456 let mut current = BTreeMap::new();
4457 for module in self.supervisor.list() {
4458 let (spec, health) = module.configuration().map_err(|err| {
4459 format!(
4460 "failed to read configuration for module_id '{}': {err}",
4461 module.module_id()
4462 )
4463 })?;
4464 let enabled = module
4465 .status()
4466 .map_err(|err| {
4467 format!(
4468 "failed to read status for module_id '{}': {err}",
4469 module.module_id()
4470 )
4471 })?
4472 .enabled;
4473 current.insert(
4474 module.module_id().to_string(),
4475 (module, spec, health, enabled),
4476 );
4477 }
4478 let configured = configured_modules
4479 .into_iter()
4480 .map(|module| (module.module_id.clone(), module))
4481 .collect::<BTreeMap<_, _>>();
4482
4483 let added = configured
4484 .keys()
4485 .filter(|module_id| !current.contains_key(*module_id))
4486 .cloned()
4487 .collect::<Vec<_>>();
4488 let removed = current
4489 .keys()
4490 .filter(|module_id| !configured.contains_key(*module_id))
4491 .cloned()
4492 .collect::<Vec<_>>();
4493 let mut changed_pending_reload = Vec::new();
4494 let mut configuration_changes = BTreeSet::new();
4495 let mut enabled_changes = BTreeSet::new();
4496 let mut unchanged = 0_u32;
4497
4498 for (module_id, configured_module) in &configured {
4499 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
4500 else {
4501 continue;
4502 };
4503 let configuration_changed = *current_spec != configured_module.module_spec()
4504 || *current_health != configured_module.health;
4505 let enabled_changed = *current_enabled != configured_module.enabled;
4506 if configuration_changed {
4507 configuration_changes.insert(module_id.clone());
4508 changed_pending_reload.push(module_id.clone());
4509 }
4510 if enabled_changed {
4511 enabled_changes.insert(module_id.clone());
4512 }
4513 if !configuration_changed && !enabled_changed {
4514 unchanged = unchanged.saturating_add(1);
4515 }
4516 }
4517
4518 if preview {
4524 return Ok(SupervisorRescanResult {
4525 added,
4526 removed,
4527 changed_pending_reload,
4528 enabled_changes: enabled_changes.iter().cloned().collect(),
4529 unchanged,
4530 preview: true,
4531 restart_required: Vec::new(),
4535 capability_warnings: Vec::new(),
4536 });
4537 }
4538
4539 for module_id in &removed {
4540 let module = ¤t
4541 .get(module_id)
4542 .expect("removed module came from current supervisor state")
4543 .0;
4544 module.retire().await.map_err(|err| {
4545 format!("failed to retire module_id '{module_id}' during rescan: {err}")
4546 })?;
4547 self.supervisor.record_rescan_removal(module_id);
4581 self.supervisor.retire(module_id);
4582 self.route_outages.forget(module_id);
4583 }
4584
4585 for module_id in configured.keys() {
4586 let Some((module, _, _, _)) = current.get(module_id) else {
4587 continue;
4588 };
4589 let configured_module = configured
4590 .get(module_id)
4591 .expect("configured module id came from configured map");
4592 if configuration_changes.contains(module_id) {
4593 module
4594 .update_configuration(
4595 configured_module.module_spec(),
4596 configured_module.health,
4597 configured_module.drain_timeout_ms,
4598 )
4599 .await
4600 .map_err(|err| {
4601 format!(
4602 "failed to update module_id '{module_id}' configuration during rescan: {err}"
4603 )
4604 })?;
4605 }
4606 if enabled_changes.contains(module_id) {
4607 self.route_outages.mark_operator_action(module_id);
4610 module
4611 .set_enabled(configured_module.enabled)
4612 .await
4613 .map_err(|err| {
4614 self.route_outages.operator_action_ended_unrefused(module_id);
4615 format!(
4616 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
4617 configured_module.enabled
4618 )
4619 })?;
4620 }
4621 }
4622
4623 for module_id in &added {
4624 let configured_module = configured
4625 .get(module_id)
4626 .expect("added module id came from configured map");
4627 supervisor
4628 .supervise_configured_with_health(
4629 configured_module.module_spec(),
4630 configured_module.enabled,
4631 configured_module.health,
4632 configured_module.drain_timeout_ms,
4633 configured_module.restart,
4634 )
4635 .map_err(|err| {
4636 format!("failed to add module_id '{module_id}' during rescan: {err}")
4637 })?;
4638 }
4639
4640 Ok(SupervisorRescanResult {
4641 added,
4642 removed,
4643 changed_pending_reload,
4644 enabled_changes: enabled_changes.iter().cloned().collect(),
4645 unchanged,
4646 preview: false,
4647 restart_required: Vec::new(),
4650 capability_warnings: Vec::new(),
4651 })
4652 }
4653
4654 async fn handle_supervisor_set_enabled(
4655 &self,
4656 frame: Frame,
4657 module_id: String,
4658 enabled: bool,
4659 ) -> Result<Vec<Frame>, RouterError> {
4660 let operation_lock = self.supervisor.operation_lock();
4661 let _operation_guard = operation_lock.lock().await;
4662 let Some(module) = self.supervisor.get(&module_id) else {
4663 return Ok(vec![control_error_frame(
4664 &frame,
4665 "unknown_module",
4666 format!("module_id '{module_id}' is not supervised"),
4667 )?]);
4668 };
4669
4670 self.route_outages.mark_operator_action(&module_id);
4673 let applied = match module.set_enabled(enabled).await {
4674 Ok(applied) => applied,
4675 Err(err) => {
4676 self.route_outages
4677 .operator_action_ended_unrefused(&module_id);
4678 return Ok(vec![control_error_frame(
4679 &frame,
4680 "target_unavailable",
4681 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
4682 )?]);
4683 }
4684 };
4685 if !applied {
4686 self.route_outages
4689 .operator_action_ended_unrefused(&module_id);
4690 }
4691
4692 self.capability_evaluator.wake_deadline_loop();
4693 self.refresh_capability_requirements();
4694 let response = ClientControlResponse::SupervisorAck { module_id, applied };
4695 Ok(vec![control_response_body_frame(
4696 &frame,
4697 &response,
4698 "ClientControlResponse::SupervisorAck",
4699 )?])
4700 }
4701
4702 async fn handle_supervisor_health_probe(
4703 &self,
4704 frame: Frame,
4705 module_id: String,
4706 ) -> Result<Vec<Frame>, RouterError> {
4707 self.refresh_capability_requirements();
4708 let Some(registration) = self
4709 .registry
4710 .get_module(&module_id)
4711 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4712 else {
4713 return Ok(vec![control_error_frame(
4714 &frame,
4715 "unknown_module",
4716 format!("module_id '{module_id}' is not registered"),
4717 )?]);
4718 };
4719
4720 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
4733 {
4734 return Ok(vec![control_error_frame(
4735 &frame,
4736 "health_not_advertised",
4737 format!("module_id '{module_id}' did not advertise health.check"),
4738 )?]);
4739 }
4740
4741 let deadline = Instant::now() + self.health_probe_timeout;
4742 let pending = match self.forwarding.begin_module_control_rpc_for(
4743 &module_id,
4744 MODULE_CONTROL_OP_HEALTH_CHECK,
4745 deadline,
4746 ) {
4747 Ok(pending) => pending,
4748 Err(err) => {
4749 return Ok(vec![control_error_frame(
4750 &frame,
4751 forwarding_error_code(&err),
4752 err.to_string(),
4753 )?])
4754 }
4755 };
4756
4757 let PendingModuleControlRpc {
4758 endpoint,
4759 module_sink,
4760 negotiated_ver,
4761 corr: probe_corr,
4762 receiver,
4763 } = pending;
4764 let mut guard =
4765 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
4766 let probe_body =
4767 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
4768 RouterError::backend(
4769 0,
4770 frame.header.corr,
4771 format!("failed to encode health.check request: {err}"),
4772 )
4773 })?;
4774 let probe_frame = Frame::build_with_version(
4775 negotiated_ver,
4776 FrameType::Request,
4777 control_flags(),
4778 0,
4779 0,
4780 probe_corr,
4781 probe_body,
4782 )
4783 .map_err(RouterError::FrameBuild)?;
4784
4785 if let Err(err) = module_sink.send(probe_frame).await {
4786 return Ok(vec![control_error_frame(
4787 &frame,
4788 "target_unavailable",
4789 err.to_string(),
4790 )?]);
4791 }
4792
4793 match timeout_at(deadline, receiver).await {
4794 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
4795 guard.disarm();
4796 let Some(report) = response.health_report() else {
4797 return Ok(vec![control_error_frame(
4798 &frame,
4799 "invalid_control_body",
4800 "health.check RPC returned a non-health response",
4801 )?]);
4802 };
4803 let HealthReport {
4808 status,
4809 detail,
4810 metrics,
4811 } = report;
4812 let capability_detail = self
4813 .capability_evaluator
4814 .required_problem_detail(&module_id);
4815 let response = ClientControlResponse::SupervisorHealthProbe {
4816 module_id,
4817 status,
4818 detail: append_capability_problem_detail(detail, capability_detail),
4819 metrics,
4820 };
4821 Ok(vec![control_response_body_frame(
4822 &frame,
4823 &response,
4824 "ClientControlResponse::SupervisorHealthProbe",
4825 )?])
4826 }
4827 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
4828 guard.disarm();
4829 Ok(vec![control_error_body_frame(&frame, body)?])
4830 }
4831 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
4832 guard.disarm();
4833 Ok(vec![control_error_frame(
4834 &frame,
4835 "target_unavailable",
4836 message,
4837 )?])
4838 }
4839 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
4840 guard.disarm();
4841 Ok(vec![control_error_frame(
4842 &frame,
4843 "invalid_control_body",
4844 message,
4845 )?])
4846 }
4847 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
4848 guard.disarm();
4849 Ok(vec![control_error_frame(
4850 &frame,
4851 "invalid_control_body",
4852 format!("expected module-control op '{expected}', got '{actual}'"),
4853 )?])
4854 }
4855 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
4856 guard.disarm();
4857 Ok(vec![control_error_frame(
4858 &frame,
4859 "module_timeout",
4860 format!(
4861 "module_id '{module_id}' answered health.check after {:?}",
4862 self.health_probe_timeout
4863 ),
4864 )?])
4865 }
4866 Ok(Err(_)) => Ok(vec![control_error_frame(
4867 &frame,
4868 "target_unavailable",
4869 "health.check waiter was canceled before the module responded",
4870 )?]),
4871 Err(_) => Ok(vec![control_error_frame(
4872 &frame,
4873 "module_timeout",
4874 format!(
4875 "module_id '{module_id}' did not answer health.check within {:?}",
4876 self.health_probe_timeout
4877 ),
4878 )?]),
4879 }
4880 }
4881
4882 fn supervisor_status(
4883 &self,
4884 module_id: &str,
4885 corr: u64,
4886 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
4887 self.supervisor
4888 .get(module_id)
4889 .map(|module| {
4890 let warming = module.is_warming_for_control("status").map_err(|err| {
4891 RouterError::backend(
4892 0,
4893 corr,
4894 format!(
4895 "failed to read supervisor warming state for module_id '{module_id}': {err}"
4896 ),
4897 )
4898 })?;
4899 module.status_for_control("status").map_err(|err| {
4900 RouterError::backend(
4901 0,
4902 corr,
4903 format!(
4904 "failed to read supervisor status for module_id '{module_id}': {err}"
4905 ),
4906 )
4907 }).map(|status| (status, warming))
4908 })
4909 .transpose()
4910 }
4911
4912 fn guard_module_control_op(
4913 &self,
4914 frame: &Frame,
4915 module_id: &str,
4916 op: &str,
4917 ) -> Result<Option<Frame>, RouterError> {
4918 if self.module_grants_op(module_id, op, frame.header.corr)? {
4919 return Ok(None);
4920 }
4921
4922 Ok(Some(control_error_frame(
4923 frame,
4924 "op_not_allowed",
4925 format!("module_id '{module_id}' did not grant control op '{op}'"),
4926 )?))
4927 }
4928
4929 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
4930 let Some(registration) = self
4931 .registry
4932 .get_module(module_id)
4933 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
4934 else {
4935 return Ok(false);
4936 };
4937 Ok(module_registration_grants_op(®istration.control_ops, op))
4938 }
4939
4940 fn handle_status_update(
4941 &self,
4942 endpoint: ModuleEndpointId,
4943 frame: Frame,
4944 ) -> Result<Vec<Frame>, RouterError> {
4945 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
4946 Ok(update) => update,
4947 Err(err) => {
4948 if is_known_module_push_op(&frame.body) {
4953 return Ok(vec![control_error_frame(
4954 &frame,
4955 "invalid_control_body",
4956 format!("malformed module control push body: {err}"),
4957 )?]);
4958 }
4959 return Ok(Vec::new());
4960 }
4961 };
4962
4963 match update {
4964 ModuleControlPush::RouteStatus {
4965 route_channel,
4966 route_epoch,
4967 status,
4968 } => {
4969 self.forwarding
4970 .cache_status(endpoint, route_channel, route_epoch, status)
4971 .map_err(RouterError::Forwarding)?;
4972 }
4973 }
4974 Ok(Vec::new())
4975 }
4976
4977 fn handle_route_poll(
4978 &self,
4979 ctx: &RouteCtx,
4980 frame: Frame,
4981 route_channel: u16,
4982 route_epoch: u32,
4983 kind: PollKind,
4984 ) -> Result<Vec<Frame>, RouterError> {
4985 let snapshot = self
4986 .forwarding
4987 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
4988 .map_err(RouterError::Forwarding)?;
4989 let response = match (kind, snapshot) {
4990 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
4991 ClientControlResponse::RoutePoll {
4992 route_channel,
4993 route_epoch,
4994 status,
4995 live: None,
4996 }
4997 }
4998 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
4999 route_channel,
5000 route_epoch,
5001 status: None,
5002 live: None,
5003 },
5004 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
5005 let live = self
5031 .process_liveness
5032 .as_ref()
5033 .and_then(|source| source.process_live(&module_id))
5034 .unwrap_or(true);
5035 ClientControlResponse::RoutePoll {
5036 route_channel,
5037 route_epoch,
5038 status: None,
5039 live: Some(live),
5040 }
5041 }
5042 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
5043 route_channel,
5044 route_epoch,
5045 status: None,
5046 live: Some(false),
5047 },
5048 };
5049
5050 Ok(vec![control_response_body_frame(
5051 &frame,
5052 &response,
5053 "ClientControlResponse::RoutePoll",
5054 )?])
5055 }
5056
5057 pub(crate) fn observe_module_control_completion(
5058 &self,
5059 completion: ModuleControlRpcCompletion,
5060 ) -> bool {
5061 match completion {
5062 ModuleControlRpcCompletion::Unknown => false,
5063 ModuleControlRpcCompletion::Settled => true,
5064 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
5065 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
5066 info!(
5067 module_id = %module_id,
5068 latency_ms,
5069 "late health.check answer proves the module is alive"
5070 );
5071 match self
5072 .supervisor
5073 .record_late_health_answer(&module_id, latency_ms)
5074 {
5075 Ok(true) => {}
5076 Ok(false) => debug!(
5077 module_id = %module_id,
5078 latency_ms,
5079 "late health.check answer has no active supervisor snapshot"
5080 ),
5081 Err(err) => warn!(
5082 module_id = %module_id,
5083 latency_ms,
5084 error = %err,
5085 "failed to record late health.check answer"
5086 ),
5087 }
5088 true
5089 }
5090 }
5091 }
5092
5093 fn refuse_to_end_module_connection_for_a_client(
5114 &self,
5115 module_connection_id: ConnectionId,
5116 corr: u64,
5117 err: ForwardingError,
5118 ) -> Result<(), RouterError> {
5119 if let ForwardingError::ConnectionClosing { connection_id } = err {
5120 if connection_id != module_connection_id {
5121 warn!(
5122 module_connection_id = module_connection_id.get(),
5123 client_connection_id = connection_id.get(),
5124 corr,
5125 "dropping a route.bind response for a closing client; the module connection keeps serving"
5126 );
5127 return Ok(());
5128 }
5129 }
5130 Err(RouterError::Forwarding(err))
5131 }
5132
5133 fn handle_module_relay_response(
5134 &self,
5135 connection_id: ConnectionId,
5136 frame: Frame,
5137 ) -> Result<Vec<Frame>, RouterError> {
5138 let mut secondary_error = None;
5139 let outcome = match frame.header.ty {
5140 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
5141 Ok(probe) if probe.op == "route.bind" => {
5142 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
5143 Ok(ModuleControlResponse::RouteBindAck {}) => {
5144 RouteBindRelayOutcome::Accepted
5145 }
5146 Ok(other) => {
5147 let message =
5148 format!("route.bind response carried unexpected body: {other:?}");
5149 secondary_error = Some(control_error_frame(
5150 &frame,
5151 "invalid_control_body",
5152 message.clone(),
5153 )?);
5154 RouteBindRelayOutcome::ModuleGone(message)
5155 }
5156 Err(err) => {
5157 let message = format!("malformed route.bind response body: {err}");
5158 secondary_error = Some(control_error_frame(
5159 &frame,
5160 "invalid_control_body",
5161 message.clone(),
5162 )?);
5163 RouteBindRelayOutcome::ModuleGone(message)
5164 }
5165 }
5166 }
5167 Ok(probe) => {
5168 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
5169 {
5170 Ok(response) => ModuleControlRpcOutcome::Response(response),
5171 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
5172 "malformed {} response body: {err}",
5173 probe.op
5174 )),
5175 };
5176 let completion = self
5177 .forwarding
5178 .complete_module_control_rpc(
5179 connection_id,
5180 frame.header.corr,
5181 Some(&probe.op),
5182 outcome,
5183 )
5184 .map_err(RouterError::Forwarding)?;
5185 if !self.observe_module_control_completion(completion) {
5186 debug!(
5187 connection_id = connection_id.get(),
5188 corr = frame.header.corr,
5189 op = %probe.op,
5190 "dropping late or unknown module-control RPC response"
5191 );
5192 }
5193 return Ok(Vec::new());
5194 }
5195 Err(err) => {
5196 if let Some(expected_op) = self
5197 .forwarding
5198 .pending_module_control_op(connection_id, frame.header.corr)
5199 .map_err(RouterError::Forwarding)?
5200 {
5201 let completion = self
5202 .forwarding
5203 .complete_module_control_rpc(
5204 connection_id,
5205 frame.header.corr,
5206 None,
5207 ModuleControlRpcOutcome::MalformedResponse(format!(
5208 "malformed {expected_op} response body: {err}"
5209 )),
5210 )
5211 .map_err(RouterError::Forwarding)?;
5212 if !self.observe_module_control_completion(completion) {
5213 debug!(
5214 connection_id = connection_id.get(),
5215 corr = frame.header.corr,
5216 "dropping late malformed module-control RPC response"
5217 );
5218 }
5219 return Ok(Vec::new());
5220 }
5221 let message = format!("malformed route.bind response body: {err}");
5222 secondary_error = Some(control_error_frame(
5223 &frame,
5224 "invalid_control_body",
5225 message.clone(),
5226 )?);
5227 RouteBindRelayOutcome::ModuleGone(message)
5228 }
5229 },
5230 FrameType::Error => {
5231 if self
5232 .forwarding
5233 .pending_module_control_op(connection_id, frame.header.corr)
5234 .map_err(RouterError::Forwarding)?
5235 .is_some()
5236 {
5237 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
5238 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
5239 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
5240 "malformed module-control ERROR body: {err}"
5241 )),
5242 };
5243 let completion = self
5244 .forwarding
5245 .complete_module_control_rpc(
5246 connection_id,
5247 frame.header.corr,
5248 None,
5249 outcome,
5250 )
5251 .map_err(RouterError::Forwarding)?;
5252 if !self.observe_module_control_completion(completion) {
5253 debug!(
5254 connection_id = connection_id.get(),
5255 corr = frame.header.corr,
5256 "dropping late or unknown module-control RPC error"
5257 );
5258 }
5259 return Ok(Vec::new());
5260 }
5261 match serde_json::from_slice::<ErrorBody>(&frame.body) {
5262 Ok(body) => RouteBindRelayOutcome::Rejected(body),
5263 Err(err) => {
5264 let message = format!("malformed route.bind ERROR body: {err}");
5265 secondary_error = Some(control_error_frame(
5266 &frame,
5267 "invalid_control_body",
5268 message.clone(),
5269 )?);
5270 RouteBindRelayOutcome::ModuleGone(message)
5271 }
5272 }
5273 }
5274 ty => {
5275 return Ok(vec![control_error_frame(
5276 &frame,
5277 "unsupported_control_frame",
5278 format!("unsupported module channel-0 frame {ty:?}"),
5279 )?])
5280 }
5281 };
5282
5283 let settled =
5284 self.forwarding
5285 .complete_pending_relay(connection_id, frame.header.corr, outcome);
5286 let completion = match settled {
5287 Ok(completion) => completion,
5288 Err(err) => {
5289 self.refuse_to_end_module_connection_for_a_client(
5290 connection_id,
5291 frame.header.corr,
5292 err,
5293 )?;
5294 return Ok(secondary_error.into_iter().collect());
5295 }
5296 };
5297 if let Some(target) = completion.abandoned.as_ref() {
5298 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
5299 }
5300 if !completion.settled {
5301 debug!(
5302 connection_id = connection_id.get(),
5303 corr = frame.header.corr,
5304 frame_type = ?frame.header.ty,
5305 "dropping late or unknown route.bind relay response"
5306 );
5307 }
5308 Ok(secondary_error.into_iter().collect())
5309 }
5310
5311 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
5312 debug!(connection_id = connection_id.get(), "handling GOODBYE");
5313 let registrations = self
5314 .deregister_connection(connection_id)
5315 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
5316 let released_routes = self
5317 .forwarding
5318 .cleanup_connection(connection_id)
5319 .map_err(RouterError::Forwarding)?;
5320 self.emit_route_goodbyes(released_routes);
5321 if !registrations.is_empty() {
5323 crate::supervise::notify_registration_release();
5324 }
5325 Ok(Vec::new())
5326 }
5327}
5328
5329impl Default for ControlHandler {
5330 fn default() -> Self {
5331 Self::new(Arc::new(Registry::default()))
5332 }
5333}
5334
5335impl crate::supervise::SwapPromotionObserver for ControlHandler {
5336 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
5337 self.apply_registration_capabilities(registration);
5338 }
5339}
5340
5341fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
5342 CapabilityRequirementStatus {
5343 consumer: status.consumer,
5344 capability: status.capability,
5345 need: match status.need {
5346 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
5347 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
5348 },
5349 verdict: status.verdict.as_str().to_string(),
5350 episode_seq: status.episode_seq,
5351 config_satisfiable: status.config_satisfiable,
5352 runtime_available: status.runtime_available,
5353 detail: status.detail,
5354 }
5355}
5356
5357fn append_capability_problem_detail(
5358 detail: Option<String>,
5359 capability_detail: Option<String>,
5360) -> Option<String> {
5361 match (detail, capability_detail) {
5362 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
5363 (Some(detail), None) => Some(detail),
5364 (None, Some(capability_detail)) => Some(capability_detail),
5365 (None, None) => None,
5366 }
5367}
5368
5369fn subc_ops() -> Vec<String> {
5370 SUBC_CONTROL_OPS
5371 .iter()
5372 .map(|op| (*op).to_string())
5373 .collect()
5374}
5375
5376fn module_subc_ops() -> Vec<String> {
5377 SUBC_CONTROL_OPS
5378 .iter()
5379 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
5380 .map(|op| (*op).to_string())
5381 .collect()
5382}
5383
5384#[cfg(test)]
5385fn module_baseline_control_ops() -> Vec<String> {
5386 MODULE_BASELINE_CONTROL_OPS
5387 .iter()
5388 .map(|op| (*op).to_string())
5389 .collect()
5390}
5391
5392fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
5393 let mut seen = HashSet::new();
5394 let mut effective = Vec::new();
5395 for op in MODULE_BASELINE_CONTROL_OPS {
5396 if seen.insert((*op).to_string()) {
5397 effective.push((*op).to_string());
5398 }
5399 }
5400 for op in declared.unwrap_or_default() {
5401 if seen.insert(op.clone()) {
5402 effective.push(op);
5403 }
5404 }
5405 effective
5406}
5407
5408fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
5409 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
5410}
5411
5412fn target_module_id(target: &RouteTarget) -> &str {
5413 match target {
5414 RouteTarget::ToolProvider { module_id }
5415 | RouteTarget::ManagementSurface { module_id }
5416 | RouteTarget::InternalService { module_id, .. } => module_id,
5417 }
5418}
5419
5420fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
5421 roles.iter().any(|role| match (target, role) {
5422 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
5423 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
5424 (
5425 RouteTarget::InternalService { service_id, .. },
5426 ProviderRole::InternalService {
5427 service_id: provided,
5428 ..
5429 },
5430 ) => service_id == provided,
5431 _ => false,
5432 })
5433}
5434
5435fn is_routable_role(role: &ProviderRole) -> bool {
5436 matches!(
5437 role,
5438 ProviderRole::ToolProvider { .. }
5439 | ProviderRole::ManagementSurface { .. }
5440 | ProviderRole::InternalService { .. }
5441 )
5442}
5443
5444#[derive(Debug, Clone, Copy, PartialEq, Eq)]
5445enum ControlRequestBodyError {
5446 UnknownOp,
5447 InvalidBody,
5448}
5449
5450#[derive(Debug, Deserialize)]
5451struct ControlOpProbe {
5452 op: String,
5453}
5454
5455const MODULE_PUSH_OPS: &[&str] = &["route.status"];
5458
5459fn is_known_module_push_op(body: &[u8]) -> bool {
5460 serde_json::from_slice::<ControlOpProbe>(body)
5461 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
5462 .unwrap_or(false)
5463}
5464
5465fn is_known_module_request_op(body: &[u8]) -> bool {
5466 serde_json::from_slice::<ControlOpProbe>(body)
5467 .map(|probe| is_module_to_subc_op(&probe.op))
5468 .unwrap_or(false)
5469}
5470
5471fn is_module_to_subc_op(op: &str) -> bool {
5472 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
5473}
5474
5475fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
5476 debug!(
5477 op = %op,
5478 connection_id = connection_id.get(),
5479 corr,
5480 "control dispatch"
5481 );
5482}
5483
5484fn log_slow_control_dispatch(
5485 dispatch_started_at: Option<StdInstant>,
5486 op: &'static str,
5487 connection_id: ConnectionId,
5488 corr: u64,
5489) {
5490 let Some(dispatch_started_at) = dispatch_started_at else {
5491 return;
5492 };
5493 let elapsed = dispatch_started_at.elapsed();
5494 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
5495 warn!(
5496 op = %op,
5497 connection_id = connection_id.get(),
5498 corr,
5499 elapsed_ms = elapsed.as_millis() as u64,
5500 "slow control dispatch"
5501 );
5502 }
5503}
5504
5505fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
5506 match request {
5507 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
5508 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
5509 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
5510 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
5511 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
5512 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
5513 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
5514 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
5515 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
5516 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
5517 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
5518 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
5519 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
5520 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
5521 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
5522 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
5523 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
5524 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
5525 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
5526 }
5527}
5528
5529fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
5530 match request {
5531 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
5532 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
5533 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
5534 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
5535 }
5536}
5537
5538fn parse_client_control_request(
5539 body: &[u8],
5540) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
5541 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
5542 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5543 Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
5544 ControlRequestBodyError::InvalidBody
5545 }
5546 Ok(_) => ControlRequestBodyError::UnknownOp,
5547 Err(_) => ControlRequestBodyError::InvalidBody,
5548 };
5549 (err, classification)
5550 })
5551}
5552
5553fn parse_module_control_request_from_module(
5554 body: &[u8],
5555) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
5556 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
5557 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5558 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
5559 Ok(_) => ControlRequestBodyError::UnknownOp,
5560 Err(_) => ControlRequestBodyError::InvalidBody,
5561 };
5562 (err, classification)
5563 })
5564}
5565
5566#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
5567enum ProviderRoleKind {
5568 ToolProvider,
5569 PipelineStage,
5570 ManagementSurface,
5571 InternalService,
5572}
5573
5574fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
5575 match role {
5576 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
5577 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
5578 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
5579 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
5580 }
5581}
5582
5583fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
5584 roles.iter().map(provider_role_kind).collect()
5585}
5586
5587const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
5590
5591#[derive(Debug, Default, PartialEq, Eq)]
5593struct ScopeOutcomeCounts {
5594 created: usize,
5595 replaced: usize,
5596 updated: usize,
5597 unchanged: usize,
5598 refused: usize,
5599}
5600
5601impl ScopeOutcomeCounts {
5602 fn of(results: &[ScopeRecordResult]) -> Self {
5603 let mut counts = Self::default();
5604 for result in results {
5605 let slot = match result.outcome {
5606 ScopeRecordOutcome::Created => &mut counts.created,
5607 ScopeRecordOutcome::Replaced => &mut counts.replaced,
5608 ScopeRecordOutcome::Updated => &mut counts.updated,
5609 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
5610 ScopeRecordOutcome::Refused => &mut counts.refused,
5611 };
5612 *slot += 1;
5613 }
5614 counts
5615 }
5616}
5617
5618#[cfg(test)]
5619mod scope_outcome_count_tests {
5620 use super::*;
5621
5622 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
5623 ScopeRecordResult {
5624 scope_ref: "r".to_string(),
5625 scope_epoch: 1,
5626 outcome,
5627 code: None,
5628 message: None,
5629 version: None,
5630 parent_state: None,
5631 }
5632 }
5633
5634 #[test]
5637 fn every_outcome_is_counted_in_its_own_field() {
5638 let results = [
5639 result(ScopeRecordOutcome::Created),
5640 result(ScopeRecordOutcome::Created),
5641 result(ScopeRecordOutcome::Replaced),
5642 result(ScopeRecordOutcome::Updated),
5643 result(ScopeRecordOutcome::Unchanged),
5644 result(ScopeRecordOutcome::Refused),
5645 result(ScopeRecordOutcome::Refused),
5646 result(ScopeRecordOutcome::Refused),
5647 ];
5648 assert_eq!(
5649 ScopeOutcomeCounts::of(&results),
5650 ScopeOutcomeCounts {
5651 created: 2,
5652 replaced: 1,
5653 updated: 1,
5654 unchanged: 1,
5655 refused: 3,
5656 }
5657 );
5658 }
5659}
5660
5661fn capability_census_trigger(
5665 old: Option<&CapabilityDeclarations>,
5666 new: Option<&CapabilityDeclarations>,
5667) -> bool {
5668 let old_provides = old
5669 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
5670 .unwrap_or_default();
5671 let old_denies = old
5672 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
5673 .unwrap_or_default();
5674 let new = new.cloned().unwrap_or(CapabilityDeclarations {
5675 provides: Vec::new(),
5676 requires: Vec::new(),
5677 must_never_reach: Vec::new(),
5678 });
5679
5680 new.provides
5681 .iter()
5682 .any(|capability| !old_provides.contains(capability))
5683 || new
5684 .must_never_reach
5685 .iter()
5686 .any(|capability| !old_denies.contains(capability))
5687}
5688
5689fn denied_capability<'a>(
5692 opening_manifest: &'a ModuleManifest,
5693 target_manifest: &ModuleManifest,
5694) -> Option<&'a str> {
5695 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
5696 let target_capabilities = target_manifest.capabilities.as_ref()?;
5697 opening_capabilities
5698 .must_never_reach
5699 .iter()
5700 .find(|denied| {
5701 target_capabilities
5702 .provides
5703 .iter()
5704 .any(|provided| provided == *denied)
5705 })
5706 .map(String::as_str)
5707}
5708
5709fn catalog_update_frozen_field_message(
5710 registered: &ModuleManifest,
5711 provides: &[ProviderRole],
5712) -> Option<String> {
5713 let old_has_provides = !registered.provides.is_empty();
5714 let new_has_provides = !provides.is_empty();
5715 if old_has_provides != new_has_provides {
5716 return Some(format!(
5717 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
5718 registered.module_id
5719 ));
5720 }
5721
5722 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
5723 return Some(format!(
5724 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
5725 registered.module_id
5726 ));
5727 }
5728
5729 let registered_concurrency = manifest_concurrency(registered);
5730 let mut candidate = registered.clone();
5731 candidate.provides = provides.to_vec();
5732 let candidate_concurrency = manifest_concurrency(&candidate);
5733 if candidate_concurrency != registered_concurrency {
5734 return Some(format!(
5735 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
5736 registered.module_id, registered_concurrency, candidate_concurrency
5737 ));
5738 }
5739
5740 None
5743}
5744
5745fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
5746 manifest.provides.iter().any(is_routable_role)
5747}
5748
5749fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
5755 manifest
5756 .provides
5757 .iter()
5758 .find_map(|provider| match provider {
5759 ProviderRole::ToolProvider { concurrency, .. }
5760 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
5761 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
5762 })
5763 .unwrap_or(Concurrency::ModuleManaged)
5764}
5765
5766fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
5773 let has_management_surface = manifest
5774 .provides
5775 .iter()
5776 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
5777 if !has_management_surface {
5778 return false;
5779 }
5780 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
5781 return false;
5782 };
5783 let Some(provides) = raw
5784 .get("manifest")
5785 .and_then(|manifest| manifest.get("provides"))
5786 .and_then(serde_json::Value::as_array)
5787 else {
5788 return false;
5789 };
5790 provides.iter().any(|role| {
5795 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
5796 && role.get("concurrency").is_none()
5797 })
5798}
5799
5800fn negotiate_version(peer_version: u8) -> Result<u8, String> {
5801 if peer_version != PROTOCOL_VERSION {
5802 return Err(format!(
5803 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
5804 ));
5805 }
5806 Ok(PROTOCOL_VERSION)
5807}
5808
5809fn pong(frame: &Frame) -> Result<Frame, RouterError> {
5810 Frame::build_with_version(
5811 response_version(frame),
5812 FrameType::Pong,
5813 frame.header.flags,
5814 0,
5815 0,
5816 frame.header.corr,
5817 Vec::new(),
5818 )
5819 .map_err(RouterError::FrameBuild)
5820}
5821
5822fn control_error_frame(
5823 frame: &Frame,
5824 code: &'static str,
5825 message: impl Into<String>,
5826) -> Result<Frame, RouterError> {
5827 control_error_body_frame(
5828 frame,
5829 ErrorBody {
5830 code: code.to_string(),
5831 message: message.into(),
5832 detail: None,
5833 },
5834 )
5835}
5836
5837fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
5838 let body = serde_json::to_vec(&error).map_err(|err| {
5839 RouterError::backend(
5840 0,
5841 frame.header.corr,
5842 format!("failed to encode control ERROR: {err}"),
5843 )
5844 })?;
5845
5846 Frame::build_with_version(
5847 response_version(frame),
5848 FrameType::Error,
5849 control_flags(),
5850 0,
5851 0,
5852 frame.header.corr,
5853 body,
5854 )
5855 .map_err(RouterError::FrameBuild)
5856}
5857
5858fn control_response_body_frame<T: Serialize>(
5859 frame: &Frame,
5860 reply: &T,
5861 label: &'static str,
5862) -> Result<Frame, RouterError> {
5863 let body = serde_json::to_vec(reply).map_err(|err| {
5864 RouterError::backend(
5865 0,
5866 frame.header.corr,
5867 format!("failed to encode {label}: {err}"),
5868 )
5869 })?;
5870
5871 Frame::build_with_version(
5872 response_version(frame),
5873 FrameType::Response,
5874 control_flags(),
5875 0,
5876 0,
5877 frame.header.corr,
5878 body,
5879 )
5880 .map_err(RouterError::FrameBuild)
5881}
5882
5883fn forwarding_error_code(err: &ForwardingError) -> &'static str {
5913 match err {
5914 ForwardingError::NoModuleConnection => "target_unavailable",
5915 ForwardingError::ModuleReloading { .. } => "module_reloading",
5916 ForwardingError::ClientRouteChannelExhausted { .. }
5917 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
5918 ForwardingError::StaleModuleEndpoint
5919 | ForwardingError::UnknownReservation { .. }
5920 | ForwardingError::ConnectionClosing { .. }
5921 | ForwardingError::ClientEgressClosed { .. }
5922 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
5923 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
5926 ForwardingError::RelayCorrelationExhausted
5927 | ForwardingError::RouteOpenBuild(_)
5928 | ForwardingError::Poisoned => "forwarding_error",
5929 }
5930}
5931
5932fn response_version(frame: &Frame) -> u8 {
5933 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
5934 frame.header.ver
5935 } else {
5936 PROTOCOL_VERSION
5937 }
5938}
5939
5940fn control_flags() -> Flags {
5941 Flags::new(false, Priority::Passive, false)
5942}
5943
5944fn send_goodbye_target_best_effort(
5949 counters: &DaemonCounters,
5950 target: &GoodbyeTarget,
5951 context: &'static str,
5952) {
5953 let Ok(frame) = Frame::build_with_version(
5954 target.negotiated_ver,
5955 FrameType::Goodbye,
5956 control_flags(),
5957 target.channel,
5958 target.epoch,
5959 0,
5960 Vec::new(),
5961 ) else {
5962 return;
5963 };
5964 crate::forwarding::send_module_route_goodbye(
5965 counters,
5966 &target.sink,
5967 frame,
5968 target.module_id.as_deref(),
5969 context,
5970 );
5971}
5972
5973pub(crate) fn send_route_control_pushes(
5974 forwarding: &ForwardingTable,
5975 routes: Vec<EndpointRoute>,
5976 push: ClientControlPush,
5977) {
5978 let body = match serde_json::to_vec(&push) {
5979 Ok(body) => body,
5980 Err(err) => {
5981 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
5982 return;
5983 }
5984 };
5985 let mut targets = Vec::new();
5986 for route in routes {
5987 let target = route.goodbye_target;
5988 if let Some(existing) = targets
5989 .iter()
5990 .find(|existing: &&GoodbyeTarget| existing.connection_id == target.connection_id)
5991 {
5992 debug_assert_eq!(
5993 existing.negotiated_ver, target.negotiated_ver,
5994 "one connection cannot negotiate multiple frame versions"
5995 );
5996 continue;
5997 }
5998 targets.push(target);
5999 }
6000 for target in targets {
6001 let frame = match Frame::build_with_version(
6002 target.negotiated_ver,
6003 FrameType::Push,
6004 control_flags(),
6005 0,
6006 0,
6007 0,
6008 body.clone(),
6009 ) {
6010 Ok(frame) => frame,
6011 Err(err) => {
6012 warn!(
6013 route_channel = target.channel,
6014 error = %err,
6015 "failed to build route lifecycle control PUSH frame"
6016 );
6017 continue;
6018 }
6019 };
6020 if let Err(err) = target.sink.try_send(frame) {
6021 if target.close_on_delivery_failure() {
6022 warn!(
6023 target_connection_id = target.connection_id.get(),
6024 route_channel = target.channel,
6025 error = %err,
6026 "route lifecycle control PUSH was not delivered to client; closing target connection"
6027 );
6028 let _ = forwarding.escalate_client_delivery_failure(
6029 target.connection_id,
6030 target.channel,
6031 target.epoch,
6032 CloseReason::new(
6033 "route_lifecycle_push_delivery_failed",
6034 format!(
6035 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
6036 target.channel
6037 ),
6038 ),
6039 crate::forwarding::UndeliveredFrame {
6040 module_id: target.module_id.as_deref(),
6041 sink: &target.sink,
6042 },
6043 );
6044 }
6045 }
6046 }
6047}
6048
6049#[cfg(test)]
6050mod tests {
6051 use std::{
6052 collections::BTreeMap,
6053 fmt,
6054 path::PathBuf,
6055 sync::{Arc, Mutex},
6056 time::Duration,
6057 };
6058 use subc_test_support::TestTempDir;
6059
6060 use serde_json::{json, Value};
6061 use subc_protocol::{
6062 manifest::{
6063 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
6064 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
6065 },
6066 session::HealthStatus,
6067 FrameType,
6068 };
6069
6070 use super::*;
6071 use crate::{
6072 forwarding::{DataRoute, DataRouteState},
6073 registry::ChannelState,
6074 router::FrameSink,
6075 stderr_tail::DEFAULT_MAX_LINE_BYTES,
6076 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
6077 RouteCtx, Router,
6078 };
6079 use tokio::{
6080 sync::mpsc,
6081 time::{sleep, Instant},
6082 };
6083 use tracing::{
6084 field::{Field, Visit},
6085 Event, Subscriber,
6086 };
6087 use tracing_subscriber::{layer::Context, prelude::*, Layer};
6088
6089 fn fake_aft_stub_path() -> PathBuf {
6108 let mut path = std::env::current_exe().expect("current_exe available in tests");
6109 path.pop(); path.pop(); path.push(if cfg!(windows) {
6112 "fake-aft-stub.exe"
6113 } else {
6114 "fake-aft-stub"
6115 });
6116 assert!(
6117 path.exists(),
6118 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
6119 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
6120 path.display()
6121 );
6122 path
6123 }
6124
6125 fn client_retries(code: &str) -> bool {
6130 subc_protocol::error_codes::is_retryable_route_open(code)
6131 }
6132
6133 #[test]
6144 fn retryability_of_forwarding_codes_matches_the_failure() {
6145 let transient = [
6148 ForwardingError::NoModuleConnection,
6149 ForwardingError::ModuleReloading {
6150 module_id: "m".into(),
6151 },
6152 ForwardingError::StaleModuleEndpoint,
6153 ForwardingError::UnknownReservation {
6154 client_channel: 1,
6155 module_channel: 1,
6156 },
6157 ForwardingError::ConnectionClosing {
6158 connection_id: ConnectionId::new(1),
6159 },
6160 ForwardingError::ClientEgressClosed {
6161 connection_id: ConnectionId::new(1),
6162 },
6163 ForwardingError::ModuleEgressUnavailable {
6164 connection_id: ConnectionId::new(1),
6165 },
6166 ];
6167 for err in transient {
6168 let code = forwarding_error_code(&err);
6169 assert!(
6170 client_retries(code),
6171 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
6172 );
6173 }
6174
6175 let permanent = [
6180 ForwardingError::ClientRouteChannelExhausted {
6181 connection_id: ConnectionId::new(1),
6182 },
6183 ForwardingError::ModuleRouteChannelExhausted {
6184 endpoint: ModuleEndpointId {
6185 connection_id: ConnectionId::new(1),
6186 generation: 1,
6187 },
6188 },
6189 ForwardingError::RelayCorrelationExhausted,
6190 ForwardingError::RouteOpenBuild("x".into()),
6191 ForwardingError::Poisoned,
6192 ];
6193 for err in permanent {
6194 let code = forwarding_error_code(&err);
6195 assert!(
6196 !client_retries(code),
6197 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
6198 );
6199 }
6200 }
6201
6202 #[tokio::test]
6230 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
6231 let handler = ControlHandler::default();
6232 let frame =
6233 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
6234
6235 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
6239 assert_eq!(
6240 stamped,
6241 Principal::Direct,
6242 "a caller that proved nothing must not be stamped as a supervised module"
6243 );
6244
6245 let forged = handler
6250 .route_open_principal(
6251 &frame,
6252 Some(ConsumerIdentity {
6253 module_id: "aft".to_string(),
6254 launch_nonce: "not-a-real-nonce".to_string(),
6255 }),
6256 )
6257 .unwrap();
6258 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
6259 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
6260 }
6261
6262 #[test]
6279 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
6280 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
6281 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
6282 let ClientControlRequest::RouteOpen {
6283 consumer_identity, ..
6284 } = parsed
6285 else {
6286 panic!("route.open body must parse as RouteOpen");
6287 };
6288 assert_eq!(
6289 consumer_identity,
6290 Some(ConsumerIdentity {
6291 module_id: "aft".to_string(),
6292 launch_nonce: "n".to_string(),
6293 }),
6294 "the wire field name must reach the value route_open_principal reads"
6295 );
6296 }
6297
6298 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
6299 ModuleManifest::builder(module_id, "0.1.0")
6300 .protocol_ver(protocol_ver)
6301 .provides(vec![ProviderRole::ToolProvider {
6302 tools: vec![Tool {
6303 name: "read".to_string(),
6304 description: None,
6305 execution_mode: ExecutionMode::Pure,
6306 schema: json!({"type": "object"}),
6307 }],
6308 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
6309 concurrency: Concurrency::ModuleManaged,
6310 emits_push: true,
6311 sub_supervises: true,
6312 }])
6313 .build()
6314 }
6315
6316 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
6317 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
6318 }
6319
6320 fn hello_frame_with_control_ops(
6321 module_id: &str,
6322 protocol_ver: u8,
6323 corr: u64,
6324 control_ops: Option<Vec<String>>,
6325 ) -> Frame {
6326 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
6327 }
6328
6329 fn hello_frame_with_nonce(
6330 module_id: &str,
6331 protocol_ver: u8,
6332 corr: u64,
6333 launch_nonce: Option<&str>,
6334 ) -> Frame {
6335 hello_frame_full(
6336 module_id,
6337 protocol_ver,
6338 corr,
6339 None,
6340 launch_nonce.map(ToOwned::to_owned),
6341 )
6342 }
6343
6344 fn hello_frame_full(
6345 module_id: &str,
6346 protocol_ver: u8,
6347 corr: u64,
6348 control_ops: Option<Vec<String>>,
6349 launch_nonce: Option<String>,
6350 ) -> Frame {
6351 let body = serde_json::to_vec(&ModuleHelloBody {
6352 manifest: manifest(module_id, protocol_ver),
6353 protocol_ver,
6354 control_ops,
6355 launch_nonce,
6356 })
6357 .unwrap();
6358 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
6359 }
6360
6361 fn non_routable_hello_frame_with_control_ops(
6362 module_id: &str,
6363 corr: u64,
6364 control_ops: Option<Vec<String>>,
6365 ) -> Frame {
6366 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
6367 manifest.provides.clear();
6368 let body = serde_json::to_vec(&ModuleHelloBody {
6369 manifest,
6370 protocol_ver: PROTOCOL_VERSION,
6371 control_ops,
6372 launch_nonce: None,
6373 })
6374 .unwrap();
6375 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
6376 }
6377
6378 fn capability_grammar_hello_frame(
6379 capabilities: Value,
6380 runtime_computed: Option<Value>,
6381 corr: u64,
6382 ) -> Frame {
6383 let mut body = serde_json::to_value(ModuleHelloBody {
6384 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
6385 protocol_ver: PROTOCOL_VERSION,
6386 control_ops: None,
6387 launch_nonce: None,
6388 })
6389 .expect("HELLO body serializes");
6390 body["manifest"]["capabilities"] = capabilities;
6391 if let Some(runtime_computed) = runtime_computed {
6392 body["runtime_computed"] = runtime_computed;
6393 }
6394 Frame::build(
6395 FrameType::Hello,
6396 control_flags(),
6397 0,
6398 0,
6399 corr,
6400 serde_json::to_vec(&body).expect("HELLO body reserializes"),
6401 )
6402 .expect("HELLO frame builds")
6403 }
6404
6405 fn channel_request(channel: u16, corr: u64) -> Frame {
6406 Frame::build(
6407 FrameType::Request,
6408 Flags::new(true, Priority::Interactive, false),
6409 channel,
6410 0,
6411 corr,
6412 b"opaque".to_vec(),
6413 )
6414 .unwrap()
6415 }
6416
6417 fn route_ctx(
6418 connection_id: ConnectionId,
6419 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
6420 let (tx, rx) = mpsc::channel(8);
6421 (
6422 RouteCtx {
6423 connection_id,
6424 egress: FrameSink::new(tx),
6425 },
6426 rx,
6427 )
6428 }
6429
6430 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
6431 serde_json::from_slice(&frame.body).unwrap()
6432 }
6433
6434 async fn hello_via_sink(
6439 handler: &ControlHandler,
6440 ctx: &RouteCtx,
6441 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
6442 hello: Frame,
6443 ) -> Frame {
6444 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
6445 assert!(
6446 replies.is_empty(),
6447 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
6448 );
6449 let ack = rx
6450 .try_recv()
6451 .expect("HELLO_ACK is queued on the module sink")
6452 .frame;
6453 assert_eq!(ack.header.ty, FrameType::HelloAck);
6454 ack
6455 }
6456
6457 fn parse_error(frame: &Frame) -> Value {
6458 serde_json::from_slice(&frame.body).unwrap()
6459 }
6460
6461 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
6462 serde_json::from_slice(&frame.body).unwrap()
6463 }
6464
6465 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
6466 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
6467 route_channel,
6468 route_epoch: 0,
6469 kind,
6470 })
6471 .unwrap();
6472 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6473 }
6474
6475 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
6476 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
6477 module_id: module_id.to_string(),
6478 })
6479 .unwrap();
6480 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6481 }
6482
6483 fn route_open_frame(corr: u64, module_id: &str, project_root: TestTempDir) -> Frame {
6484 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
6485 }
6486
6487 fn route_open_frame_with_consumer_capabilities(
6488 corr: u64,
6489 module_id: &str,
6490 project_root: TestTempDir,
6491 consumer_capabilities: Option<Vec<String>>,
6492 ) -> Frame {
6493 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6494 target: RouteTarget::ToolProvider {
6495 module_id: module_id.to_string(),
6496 },
6497 identity: BindIdentity::new(
6498 project_root.path().to_path_buf(),
6499 "unit".to_string(),
6500 "session".to_string(),
6501 ),
6502 consumer_identity: None,
6503 consumer_capabilities,
6504 role_versions: None,
6505 admission_facts: None,
6506 scope: None,
6507 })
6508 .unwrap();
6509 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6510 }
6511
6512 fn route_open_frame_with_role_versions(
6513 corr: u64,
6514 module_id: &str,
6515 project_root: TestTempDir,
6516 role_versions: Option<BTreeMap<String, String>>,
6517 ) -> Frame {
6518 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6519 target: RouteTarget::ToolProvider {
6520 module_id: module_id.to_string(),
6521 },
6522 identity: BindIdentity::new(
6523 project_root.path().to_path_buf(),
6524 "unit".to_string(),
6525 format!("session-{corr}"),
6526 ),
6527 consumer_identity: None,
6528 consumer_capabilities: None,
6529 role_versions,
6530 admission_facts: None,
6531 scope: None,
6532 })
6533 .unwrap();
6534 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6535 }
6536
6537 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
6538 entries
6539 .iter()
6540 .map(|(role, version)| (role.to_string(), version.to_string()))
6541 .collect()
6542 }
6543
6544 fn route_open_frame_with_admission_facts(
6545 corr: u64,
6546 module_id: &str,
6547 project_root: TestTempDir,
6548 consumer_identity: Option<subc_control::ConsumerIdentity>,
6549 facts: Option<Value>,
6550 ) -> Frame {
6551 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6552 target: RouteTarget::ToolProvider {
6553 module_id: module_id.to_string(),
6554 },
6555 identity: BindIdentity::new(
6556 project_root.path().to_path_buf(),
6557 "unit".to_string(),
6558 format!("session-{corr}"),
6559 ),
6560 consumer_identity,
6561 consumer_capabilities: None,
6562 role_versions: None,
6563 admission_facts: facts,
6564 scope: None,
6565 })
6566 .unwrap();
6567 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6568 }
6569
6570 #[derive(Clone, Default)]
6571 struct EventCapture {
6572 events: Arc<Mutex<Vec<CapturedEvent>>>,
6573 }
6574
6575 #[derive(Clone, Debug)]
6576 struct CapturedEvent {
6577 target: String,
6578 level: tracing::Level,
6579 fields: BTreeMap<String, String>,
6580 }
6581
6582 impl EventCapture {
6583 fn events(&self) -> Vec<CapturedEvent> {
6584 self.events.lock().unwrap().clone()
6585 }
6586 }
6587
6588 impl<S> Layer<S> for EventCapture
6589 where
6590 S: Subscriber,
6591 {
6592 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
6593 let mut visitor = EventFieldVisitor::default();
6594 event.record(&mut visitor);
6595 self.events.lock().unwrap().push(CapturedEvent {
6596 target: event.metadata().target().to_string(),
6597 level: *event.metadata().level(),
6598 fields: visitor.fields,
6599 });
6600 }
6601 }
6602
6603 #[derive(Default)]
6604 struct EventFieldVisitor {
6605 fields: BTreeMap<String, String>,
6606 }
6607
6608 impl Visit for EventFieldVisitor {
6609 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
6610 self.fields
6611 .insert(field.name().to_string(), format!("{value:?}"));
6612 }
6613 }
6614
6615 fn health_response(corr: u64, status: HealthStatus) -> Frame {
6616 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
6617 status,
6618 detail: Some("warming".to_string()),
6619 metrics: Some(json!({"queue_depth": 3})),
6620 })
6621 .unwrap();
6622 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
6623 }
6624
6625 fn route_bind_ack(corr: u64) -> Frame {
6626 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
6627 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
6628 }
6629
6630 fn unique_project_root(label: &str) -> TestTempDir {
6631 TestTempDir::new(label)
6632 }
6633
6634 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
6635 match parse_route_poll(frame) {
6636 ClientControlResponse::RoutePoll {
6637 status: None,
6638 live: Some(live),
6639 ..
6640 } => assert_eq!(live, expected_live),
6641 other => panic!("unexpected route.poll response: {other:?}"),
6642 }
6643 }
6644
6645 fn bind_liveness_route(
6646 registry: &Registry,
6647 forwarding: &ForwardingTable,
6648 module_id: &str,
6649 ) -> (RouteCtx, u16, u32) {
6650 let module_connection = ConnectionId::new(101);
6651 let client_connection = ConnectionId::new(202);
6652 let registration = registry
6653 .register_with_control_ops(
6654 manifest(module_id, PROTOCOL_VERSION),
6655 PROTOCOL_VERSION,
6656 module_connection,
6657 module_baseline_control_ops(),
6658 )
6659 .unwrap();
6660 let (module_tx, _module_rx) = mpsc::channel(8);
6661 let endpoint = forwarding
6662 .register_module_connection(
6663 module_connection,
6664 module_id.to_string(),
6665 PROTOCOL_VERSION,
6666 manifest_concurrency(®istration.manifest),
6667 FrameSink::new(module_tx),
6668 )
6669 .unwrap();
6670 let (client_ctx, _client_rx) = route_ctx(client_connection);
6671 let pending = forwarding
6672 .begin_route_bind_relay_for_test(
6673 client_connection,
6674 client_ctx.egress.clone(),
6675 1,
6676 module_id,
6677 )
6678 .unwrap();
6679 assert_eq!(pending.endpoint, endpoint);
6680 let route_channel = pending.client_channel;
6681 let route_epoch = pending.client_epoch;
6682 forwarding
6683 .complete_pending_relay(
6684 module_connection,
6685 pending.corr,
6686 RouteBindRelayOutcome::Accepted,
6687 )
6688 .unwrap();
6689 (client_ctx, route_channel, route_epoch)
6690 }
6691
6692 struct FakeProcessLiveness {
6693 live: Option<bool>,
6694 }
6695
6696 impl ModuleProcessLiveness for FakeProcessLiveness {
6697 fn process_live(&self, _module_id: &str) -> Option<bool> {
6698 self.live
6699 }
6700 }
6701
6702 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6703 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
6704 {
6705 let registry = Arc::new(Registry::default());
6706 let supervisor_handle = SupervisorHandle::new();
6707 let supervisor = Supervisor::new(
6708 Arc::clone(®istry),
6709 RestartPolicy::new(1, Duration::from_millis(10)),
6710 )
6711 .with_handle(supervisor_handle.clone());
6712 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
6713 let module = supervisor
6714 .spawn(ModuleSpec {
6715 module_id: "stderr-tail-wire".to_string(),
6716 program: fake_aft_stub_path(),
6717 args: Vec::new(),
6718 env: vec![
6719 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
6720 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
6721 ],
6722 reserved: false,
6723 reserved_prefixes: Vec::new(),
6724 protocol: ModuleProtocol::Subc,
6725 overlap: Default::default(),
6726 })
6727 .unwrap();
6728
6729 let deadline = Instant::now() + Duration::from_secs(5);
6730 loop {
6731 let tail = module.stderr_tail(None, None);
6732 if tail
6733 .entries
6734 .iter()
6735 .any(|entry| matches!(entry, TailEntry::ProcessStart))
6736 && tail.entries.iter().any(|entry| {
6737 matches!(
6738 entry,
6739 TailEntry::Line {
6740 truncated: true,
6741 ..
6742 }
6743 )
6744 })
6745 {
6746 break;
6747 }
6748 assert!(
6749 Instant::now() < deadline,
6750 "module did not produce a truncated line and restart boundary: {tail:?}"
6751 );
6752 sleep(Duration::from_millis(10)).await;
6753 }
6754
6755 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
6756 let request = ClientControlRequest::SupervisorStderrTail {
6757 module_id: "stderr-tail-wire".to_string(),
6758 max_lines: None,
6759 max_bytes: None,
6760 };
6761 let frame = Frame::build(
6762 FrameType::Request,
6763 control_flags(),
6764 0,
6765 0,
6766 1,
6767 serde_json::to_vec(&request).unwrap(),
6768 )
6769 .unwrap();
6770 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6771 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
6772 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
6773 serde_json::from_slice(&responses[0].body).unwrap()
6774 else {
6775 panic!("expected supervisor.stderr_tail response");
6776 };
6777
6778 assert!(
6779 tail.entries
6780 .iter()
6781 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
6782 "the control response lost the restart boundary"
6783 );
6784 let Some(StderrTailEntry::Line {
6785 text,
6786 truncated,
6787 at_ms,
6788 }) = tail.entries.iter().find(|entry| {
6789 matches!(
6790 entry,
6791 StderrTailEntry::Line {
6792 truncated: true,
6793 ..
6794 }
6795 )
6796 })
6797 else {
6798 panic!("the control response lost the truncated line");
6799 };
6800 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
6801 assert!(*truncated);
6802 assert!(
6803 at_ms.is_some(),
6804 "the control response lost the line's capture time"
6805 );
6806 }
6807
6808 #[tokio::test(flavor = "current_thread")]
6813 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
6814 let dir = TestTempDir::new("terminals-off-worker");
6815 let journal_path = dir.join("terminals.jsonl");
6816 let registry = Arc::new(Registry::default());
6817 let supervisor_handle = SupervisorHandle::new();
6818 let supervisor =
6819 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
6820 .with_handle(supervisor_handle.clone())
6821 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
6822 let module = supervisor
6823 .spawn(ModuleSpec {
6824 module_id: "terminal-off-worker".to_string(),
6825 program: fake_aft_stub_path(),
6826 args: Vec::new(),
6827 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
6828 reserved: false,
6829 reserved_prefixes: Vec::new(),
6830 protocol: ModuleProtocol::Subc,
6831 overlap: Default::default(),
6832 })
6833 .unwrap();
6834 let deadline = Instant::now() + Duration::from_secs(5);
6835 while module.terminal_history().entries.len() != 2 {
6836 assert!(Instant::now() < deadline, "module did not record two exits");
6837 sleep(Duration::from_millis(10)).await;
6838 }
6839
6840 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
6841 let handler =
6842 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
6843 let frame = Frame::build(
6844 FrameType::Request,
6845 control_flags(),
6846 0,
6847 0,
6848 1,
6849 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
6850 module_id: "terminal-off-worker".to_string(),
6851 })
6852 .unwrap(),
6853 )
6854 .unwrap();
6855 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6856 let spawned_at = std::time::Instant::now();
6857 let read = tokio::spawn({
6858 let handler = Arc::clone(&handler);
6859 async move { handler.handle_control_frame(&ctx, frame).await }
6860 });
6861 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
6864 .await
6865 .unwrap()
6866 .expect("the history read reached its pause");
6867 let elapsed = spawned_at.elapsed();
6868 assert!(
6869 elapsed < Duration::from_secs(2) && !read.is_finished(),
6870 "this task could not run while the history read was paused \
6871 (resumed after {elapsed:?}, read finished: {})",
6872 read.is_finished()
6873 );
6874
6875 drop(release);
6876 let responses = read.await.unwrap().unwrap();
6877 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
6878 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
6879 panic!("expected supervisor.terminals response");
6880 };
6881 assert_eq!(terminals.entries.len(), 2);
6882 assert_eq!(terminals.journal_skipped_lines, 0);
6883 assert_eq!(terminals.journal_read_errors, 0);
6884 }
6885
6886 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6887 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
6888 let registry = Arc::new(Registry::default());
6889 let supervisor_handle = SupervisorHandle::new();
6890 let supervisor =
6891 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
6892 .with_handle(supervisor_handle.clone());
6893 let module = supervisor
6894 .spawn(ModuleSpec {
6895 module_id: "terminal-golden".to_string(),
6896 program: fake_aft_stub_path(),
6897 args: Vec::new(),
6898 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
6899 reserved: false,
6900 reserved_prefixes: Vec::new(),
6901 protocol: ModuleProtocol::Subc,
6902 overlap: Default::default(),
6903 })
6904 .unwrap();
6905
6906 let deadline = Instant::now() + Duration::from_secs(5);
6907 while module.terminal_history().entries.len() != 2 {
6908 assert!(
6909 Instant::now() < deadline,
6910 "module did not retain two terminal exits: {:?}",
6911 module.terminal_history()
6912 );
6913 sleep(Duration::from_millis(10)).await;
6914 }
6915
6916 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
6917 let request = ClientControlRequest::SupervisorTerminals {
6918 module_id: "terminal-golden".to_string(),
6919 };
6920 let frame = Frame::build(
6921 FrameType::Request,
6922 control_flags(),
6923 0,
6924 0,
6925 1,
6926 serde_json::to_vec(&request).unwrap(),
6927 )
6928 .unwrap();
6929 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6930 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
6931 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
6932 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
6933 panic!("expected supervisor.terminals response");
6934 };
6935 assert_eq!(terminals.entries.len(), 2);
6936 assert_eq!(terminals.dropped, 0);
6937
6938 let mut rendered = serde_json::to_value(response).unwrap();
6939 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
6942 for (index, entry) in rendered["entries"]
6943 .as_array_mut()
6944 .expect("terminal response entries array")
6945 .iter_mut()
6946 .enumerate()
6947 {
6948 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
6949 }
6950
6951 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
6952 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
6953 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
6954 if std::env::var_os("UPDATE_GOLDEN").is_some() {
6955 std::fs::write(&golden_path, &serialized).unwrap();
6956 }
6957 let expected: Value =
6958 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
6959 assert_eq!(rendered, expected);
6960 }
6961
6962 #[test]
6963 fn hello_registers_manifest_and_returns_ack() {
6964 let registry = Arc::new(Registry::default());
6965 let handler = ControlHandler::new(Arc::clone(®istry));
6966 let conn = ConnectionId::new(1);
6967
6968 let responses = handler
6969 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
6970 .unwrap();
6971
6972 assert_eq!(responses.len(), 1);
6973 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
6974 assert_eq!(responses[0].header.channel, 0);
6975 assert_eq!(responses[0].header.corr, 7);
6976 let ack = parse_ack(&responses[0]);
6977 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
6978 assert!(ack
6979 .subc_capabilities
6980 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
6981 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
6982 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
6983 assert!(ack
6984 .subc_ops
6985 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
6986 assert!(ack
6987 .subc_ops
6988 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
6989
6990 let registration = registry.get_module("aft").unwrap().unwrap();
6991 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
6992 assert_eq!(registration.state, ChannelState::Active);
6993 assert_eq!(registration.connection_id, conn);
6994 assert_eq!(registration.control_ops, module_baseline_control_ops());
6995 }
6996
6997 #[test]
6998 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
6999 let invalid_identifiers = [
7000 ("case_change", "credentials-Provider/v1"),
7001 ("leading_zero", "credentials-provider/v01"),
7002 ("trailing_hyphen", "credentials-provider-/v1"),
7003 ("consecutive_hyphens", "credentials--provider/v1"),
7004 ("uppercase", "Credentials-provider/v1"),
7005 ("missing_v", "credentials-provider/1"),
7006 ("whitespace", "credentials provider/v1"),
7007 ("zero_version", "credentials-provider/v0"),
7008 ("out_of_range_version", "credentials-provider/v4294967296"),
7009 (
7010 "overlength_name",
7011 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
7012 ),
7013 ];
7014 let mut cases = invalid_identifiers
7015 .into_iter()
7016 .map(|(name, identifier)| {
7017 (
7018 format!("identifier_{name}"),
7019 "capabilities.provides[0]".to_string(),
7020 identifier.to_string(),
7021 json!({ "provides": [identifier] }),
7022 None,
7023 )
7024 })
7025 .collect::<Vec<_>>();
7026 cases.extend([
7027 (
7028 "unknown_need".to_string(),
7029 "capabilities.requires[0].need".to_string(),
7030 "deferred".to_string(),
7031 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
7032 None,
7033 ),
7034 (
7035 "duplicate_provides".to_string(),
7036 "capabilities.provides[1]".to_string(),
7037 "credentials-provider/v1".to_string(),
7038 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
7039 None,
7040 ),
7041 (
7042 "duplicate_must_never_reach".to_string(),
7043 "capabilities.must_never_reach[1]".to_string(),
7044 "credentials-provider/v1".to_string(),
7045 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
7046 None,
7047 ),
7048 (
7049 "duplicate_requires_same_need".to_string(),
7050 "capabilities.requires[1]".to_string(),
7051 "credentials-provider/v1".to_string(),
7052 json!({ "requires": [
7053 { "capability": "credentials-provider/v1", "need": "required" },
7054 { "capability": "credentials-provider/v1", "need": "required" }
7055 ] }),
7056 None,
7057 ),
7058 (
7059 "duplicate_requires_conflicting_need".to_string(),
7060 "capabilities.requires[1]".to_string(),
7061 "credentials-provider/v1".to_string(),
7062 json!({ "requires": [
7063 { "capability": "credentials-provider/v1", "need": "required" },
7064 { "capability": "credentials-provider/v1", "need": "optional" }
7065 ] }),
7066 None,
7067 ),
7068 (
7069 "capabilities_root_pointer".to_string(),
7070 "runtime_computed[0]".to_string(),
7071 "/capabilities".to_string(),
7072 json!({}),
7073 Some(json!(["/capabilities"])),
7074 ),
7075 (
7076 "capabilities_descendant_pointer".to_string(),
7077 "runtime_computed[0]".to_string(),
7078 "/capabilities/provides".to_string(),
7079 json!({}),
7080 Some(json!(["/capabilities/provides"])),
7081 ),
7082 (
7083 "malformed_pointer_without_leading_slash".to_string(),
7084 "runtime_computed[0]".to_string(),
7085 "capabilities".to_string(),
7086 json!({}),
7087 Some(json!(["capabilities"])),
7088 ),
7089 (
7090 "malformed_pointer_escape".to_string(),
7091 "runtime_computed[0]".to_string(),
7092 "/roles/~2/tools".to_string(),
7093 json!({}),
7094 Some(json!(["/roles/~2/tools"])),
7095 ),
7096 (
7097 "unknown_capabilities_field".to_string(),
7098 "capabilities.future".to_string(),
7099 "<array>".to_string(),
7100 json!({ "future": [] }),
7101 None,
7102 ),
7103 ]);
7104
7105 for (index, (name, field, value, capabilities, runtime_computed)) in
7106 cases.into_iter().enumerate()
7107 {
7108 let registry = Arc::new(Registry::default());
7109 let handler = ControlHandler::new(Arc::clone(®istry));
7110 let response = handler
7111 .handle_control(
7112 ConnectionId::new((index + 1) as u64),
7113 capability_grammar_hello_frame(
7114 capabilities,
7115 runtime_computed,
7116 index as u64 + 1,
7117 ),
7118 )
7119 .expect("invalid HELLO returns a refusal");
7120
7121 assert_eq!(response.len(), 1, "{name} must emit one refusal");
7122 let error = parse_error(&response[0]);
7123 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
7124 let message = error["message"]
7125 .as_str()
7126 .expect("error message is a string");
7127 assert!(
7128 message.contains(&field),
7129 "{name}: field missing from {message}"
7130 );
7131 assert!(
7132 message.contains(&value),
7133 "{name}: value missing from {message}"
7134 );
7135 assert_eq!(
7136 registry
7137 .active_registration_count()
7138 .expect("registry reads"),
7139 0,
7140 "{name}: refused HELLO must not create a catalog entry"
7141 );
7142 }
7143 }
7144
7145 #[test]
7146 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
7147 let registry = Arc::new(Registry::default());
7148 let handler = ControlHandler::new(Arc::clone(®istry));
7149 let capabilities = json!({
7150 "provides": ["credentials-provider/v1"],
7151 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
7152 "must_never_reach": ["federation-transport/v1"]
7153 });
7154 let response = handler
7155 .handle_control(
7156 ConnectionId::new(99),
7157 capability_grammar_hello_frame(
7158 capabilities.clone(),
7159 Some(json!(["/roles/0/tools"])),
7160 99,
7161 ),
7162 )
7163 .expect("valid HELLO registers");
7164 assert_eq!(response[0].header.ty, FrameType::HelloAck);
7165
7166 let request = Frame::build(
7167 FrameType::Request,
7168 control_flags(),
7169 0,
7170 0,
7171 100,
7172 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7173 .expect("catalog request serializes"),
7174 )
7175 .expect("catalog request frame builds");
7176 let response = handler
7177 .handle_catalog_list(request, None)
7178 .expect("catalog list succeeds");
7179 let ClientControlResponse::CatalogList { modules, .. } =
7180 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
7181 else {
7182 panic!("catalog request must return catalog.list");
7183 };
7184 assert_eq!(modules.len(), 1);
7185 assert_eq!(
7186 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
7187 capabilities
7188 );
7189 }
7190
7191 #[test]
7192 fn catalog_list_mirrors_management_operation_description() {
7193 let registry = Arc::new(Registry::default());
7194 let handler = ControlHandler::new(Arc::clone(®istry));
7195 let description = "List managed records and return their identifiers and metadata.";
7196 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
7197 manifest.provides = vec![ProviderRole::ManagementSurface {
7198 operations: vec![ManagementOperation {
7199 name: "records.list".to_string(),
7200 kind: ManagementOperationKind::Query,
7201 description: Some(description.to_string()),
7202 }],
7203 config_schema: json!({"type": "object"}),
7204 observability: vec![ObservabilitySurface {
7205 name: "records.stats".to_string(),
7206 kind: ObservabilityKind::Snapshot,
7207 }],
7208 identity_scope: vec![IdentityScope::Project],
7209 concurrency: Concurrency::ModuleManaged,
7210 }];
7211 registry
7212 .register_with_control_ops(
7213 manifest,
7214 PROTOCOL_VERSION,
7215 ConnectionId::new(99),
7216 Vec::new(),
7217 )
7218 .expect("described management manifest registers");
7219
7220 let request = Frame::build(
7221 FrameType::Request,
7222 control_flags(),
7223 0,
7224 0,
7225 100,
7226 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7227 .expect("catalog request serializes"),
7228 )
7229 .expect("catalog request frame builds");
7230 let response = handler
7231 .handle_catalog_list(request, None)
7232 .expect("catalog list succeeds");
7233 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
7234 assert_eq!(
7235 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
7236 "catalog.list must preserve the declared operation description verbatim"
7237 );
7238 }
7239
7240 #[test]
7241 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
7242 let registry = Arc::new(Registry::default());
7243 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
7244 [("vault".to_string(), true), ("squatter".to_string(), true)],
7245 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
7246 );
7247 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
7248 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
7249 provides: vec!["credentials-provider/v1".to_string()],
7250 requires: Vec::new(),
7251 must_never_reach: Vec::new(),
7252 });
7253 let frame = Frame::build(
7254 FrameType::Hello,
7255 control_flags(),
7256 0,
7257 0,
7258 77,
7259 serde_json::to_vec(&ModuleHelloBody {
7260 manifest: squatter,
7261 protocol_ver: PROTOCOL_VERSION,
7262 control_ops: None,
7263 launch_nonce: None,
7264 })
7265 .expect("HELLO serializes"),
7266 )
7267 .expect("HELLO frame builds");
7268 let response = handler
7269 .handle_control(ConnectionId::new(77), frame)
7270 .expect("reserved claim receives a typed refusal");
7271 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
7272 assert_eq!(
7273 registry
7274 .active_registration_count()
7275 .expect("registry reads"),
7276 0,
7277 "a reserved capability refusal must not leave a catalog entry"
7278 );
7279 }
7280
7281 #[test]
7282 fn server_describe_surfaces_required_capability_verdict_fields() {
7283 let registry = Arc::new(Registry::default());
7284 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
7285 [
7286 ("consumer".to_string(), true),
7287 ("provider".to_string(), false),
7288 ],
7289 BTreeMap::new(),
7290 );
7291 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
7292 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
7293 provides: Vec::new(),
7294 requires: vec![subc_protocol::manifest::CapabilityRequirement {
7295 capability: "credentials-provider/v1".to_string(),
7296 need: subc_protocol::manifest::CapabilityNeed::Required,
7297 }],
7298 must_never_reach: Vec::new(),
7299 });
7300 let hello = Frame::build(
7301 FrameType::Hello,
7302 control_flags(),
7303 0,
7304 0,
7305 78,
7306 serde_json::to_vec(&ModuleHelloBody {
7307 manifest: consumer,
7308 protocol_ver: PROTOCOL_VERSION,
7309 control_ops: None,
7310 launch_nonce: None,
7311 })
7312 .expect("HELLO serializes"),
7313 )
7314 .expect("HELLO frame builds");
7315 handler
7316 .handle_control(ConnectionId::new(78), hello)
7317 .expect("consumer registers");
7318 let describe = Frame::build(
7319 FrameType::Request,
7320 control_flags(),
7321 0,
7322 0,
7323 79,
7324 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
7325 .expect("request serializes"),
7326 )
7327 .expect("describe frame builds");
7328 let response = handler
7329 .handle_server_describe(describe)
7330 .expect("server.describe succeeds");
7331 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
7332 let requirement = &rendered["capability_requirements"][0];
7333 assert_eq!(requirement["consumer"], "consumer");
7334 assert_eq!(requirement["verdict"], "never_provided");
7335 assert_eq!(requirement["episode_seq"], 1);
7336 assert_eq!(requirement["config_satisfiable"], false);
7337 assert_eq!(requirement["runtime_available"], false);
7338 assert!(requirement["detail"]
7339 .as_str()
7340 .expect("detail string")
7341 .contains("credentials-provider/v1"));
7342 }
7343
7344 #[test]
7345 fn catalog_list_omits_capabilities_for_legacy_manifest() {
7346 let registry = Arc::new(Registry::default());
7347 let handler = ControlHandler::new(Arc::clone(®istry));
7348 let hello = handler
7349 .handle_control(
7350 ConnectionId::new(101),
7351 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
7352 )
7353 .expect("legacy HELLO registers");
7354 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
7355
7356 let request = Frame::build(
7357 FrameType::Request,
7358 control_flags(),
7359 0,
7360 0,
7361 102,
7362 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7363 .expect("catalog request serializes"),
7364 )
7365 .expect("catalog request frame builds");
7366 let response = handler
7367 .handle_catalog_list(request, None)
7368 .expect("catalog list succeeds");
7369 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
7370 assert!(
7371 body["modules"][0].get("capabilities").is_none(),
7372 "legacy manifest must retain an absent capabilities field on catalog.list"
7373 );
7374 }
7375
7376 #[test]
7377 fn hello_ack_omits_storage_when_no_storage_config() {
7378 let registry = Arc::new(Registry::default());
7379 let handler = ControlHandler::new(Arc::clone(®istry));
7380 let responses = handler
7381 .handle_control(
7382 ConnectionId::new(1),
7383 hello_frame("aft", PROTOCOL_VERSION, 7),
7384 )
7385 .unwrap();
7386 let ack = parse_ack(&responses[0]);
7387 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
7388 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
7389 }
7390
7391 #[tokio::test]
7392 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
7393 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
7394 let registry = Arc::new(Registry::default());
7395 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
7396 let responses = handler
7397 .handle_control(
7398 ConnectionId::new(1),
7399 hello_frame("aft", PROTOCOL_VERSION, 7),
7400 )
7401 .unwrap();
7402 let ack = parse_ack(&responses[0]);
7403 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
7404
7405 let described = handler
7406 .handle_control_frame(
7407 &route_ctx(ConnectionId::new(2)).0,
7408 Frame::build(
7409 FrameType::Request,
7410 control_flags(),
7411 0,
7412 0,
7413 9,
7414 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
7415 )
7416 .unwrap(),
7417 )
7418 .await
7419 .unwrap();
7420 let ClientControlResponse::ServerDescribe { machine_id, .. } =
7421 serde_json::from_slice(&described[0].body).unwrap()
7422 else {
7423 panic!("server.describe answered with another shape");
7424 };
7425 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
7426 }
7427
7428 #[test]
7429 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
7430 let registry = Arc::new(Registry::default());
7433 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
7434 crate::daemon_config::StorageConfig::Sqlite {
7435 data_home: std::path::PathBuf::from("/data"),
7436 },
7437 ));
7438
7439 let responses = handler
7440 .handle_control(
7441 ConnectionId::new(1),
7442 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
7443 )
7444 .unwrap();
7445 let ack = parse_ack(&responses[0]);
7446 assert_eq!(
7447 ack.storage,
7448 Some(serde_json::json!({
7449 "module_id": "alfonso-routing",
7450 "storage_namespace": "default",
7451 "isolation": { "kind": "module" },
7452 "backend": {
7453 "backend": "sqlite",
7454 "path": "/data/cortexkit/alfonso-routing/store.db"
7455 }
7456 })),
7457 "the delivered descriptor is the module's own sqlite store path"
7458 );
7459 }
7460
7461 #[test]
7462 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
7463 let registry = Arc::new(Registry::default());
7464 let handler = ControlHandler::new(Arc::clone(®istry));
7465 let conn = ConnectionId::new(1);
7466 let responses = handler
7467 .handle_control(
7468 conn,
7469 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
7470 )
7471 .unwrap();
7472 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
7473 let registration = registry.get_module("aft").unwrap().unwrap();
7474 assert_eq!(registration.control_ops, module_baseline_control_ops());
7475
7476 let frame =
7477 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
7478 assert!(handler
7479 .guard_module_control_op(&frame, "aft", "route.bind")
7480 .unwrap()
7481 .is_none());
7482 let error = handler
7483 .guard_module_control_op(&frame, "aft", "test.synthetic")
7484 .unwrap()
7485 .expect("synthetic ungranted op should be rejected");
7486 assert_eq!(error.header.ty, FrameType::Error);
7487 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
7488 }
7489
7490 #[test]
7491 fn hello_control_ops_some_adds_optional_grants() {
7492 let registry = Arc::new(Registry::default());
7493 let handler = ControlHandler::new(Arc::clone(®istry));
7494 handler
7495 .handle_control(
7496 ConnectionId::new(1),
7497 hello_frame_with_control_ops(
7498 "aft",
7499 PROTOCOL_VERSION,
7500 7,
7501 Some(vec![
7502 "future.synthetic".to_string(),
7503 "route.bind".to_string(),
7504 ]),
7505 ),
7506 )
7507 .unwrap();
7508 let registration = registry.get_module("aft").unwrap().unwrap();
7509 assert_eq!(
7510 registration.control_ops,
7511 vec![
7512 "route.bind".to_string(),
7513 "route.status".to_string(),
7514 "future.synthetic".to_string(),
7515 ]
7516 );
7517 let frame =
7518 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
7519 assert!(handler
7520 .guard_module_control_op(&frame, "aft", "future.synthetic")
7521 .unwrap()
7522 .is_none());
7523 }
7524
7525 #[tokio::test]
7526 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
7527 let registry = Arc::new(Registry::default());
7528 let forwarding = Arc::new(ForwardingTable::default());
7529 let handler =
7530 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
7531 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
7532 hello_via_sink(
7533 &handler,
7534 &module_ctx,
7535 &mut module_rx,
7536 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
7537 )
7538 .await;
7539
7540 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
7541 let responses = handler
7542 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
7543 .await
7544 .unwrap();
7545 assert_eq!(responses.len(), 1);
7546 assert_eq!(responses[0].header.ty, FrameType::Error);
7547 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
7548 assert!(module_rx.try_recv().is_err());
7549 }
7550
7551 #[tokio::test]
7552 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
7553 let registry = Arc::new(Registry::default());
7554 let forwarding = Arc::new(ForwardingTable::default());
7555 let handler =
7556 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
7557 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
7558 hello_via_sink(
7559 &handler,
7560 &module_ctx,
7561 &mut module_rx,
7562 hello_frame_with_control_ops(
7563 "aft",
7564 PROTOCOL_VERSION,
7565 7,
7566 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
7567 ),
7568 )
7569 .await;
7570
7571 let project_root = unique_project_root("demux");
7572 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
7573 let route_handler = handler.clone();
7574 let route_task = tokio::spawn(async move {
7575 route_handler
7576 .handle_control_frame(
7577 &route_client_ctx,
7578 route_open_frame(100, "aft", project_root),
7579 )
7580 .await
7581 .unwrap()
7582 });
7583 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7584 .await
7585 .unwrap()
7586 .unwrap();
7587 assert!(matches!(
7588 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
7589 ModuleControlRequest::RouteBind { .. }
7590 ));
7591
7592 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
7593 let health_handler = handler.clone();
7594 let health_task = tokio::spawn(async move {
7595 health_handler
7596 .handle_control_frame(
7597 &health_client_ctx,
7598 supervisor_health_probe_frame(101, "aft"),
7599 )
7600 .await
7601 .unwrap()
7602 });
7603 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7604 .await
7605 .unwrap()
7606 .unwrap();
7607 assert_eq!(
7608 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
7609 ModuleControlRequest::HealthCheck {}
7610 );
7611
7612 handler
7613 .handle_control_frame(
7614 &module_ctx,
7615 health_response(health_frame.header.corr, HealthStatus::Degraded),
7616 )
7617 .await
7618 .unwrap();
7619 let health_response = health_task.await.unwrap();
7620 assert_eq!(health_response.len(), 1);
7621 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
7622 ClientControlResponse::SupervisorHealthProbe {
7623 module_id,
7624 status,
7625 detail,
7626 metrics,
7627 } => {
7628 assert_eq!(module_id, "aft");
7629 assert_eq!(status, HealthStatus::Degraded);
7630 assert_eq!(detail.as_deref(), Some("warming"));
7631 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
7632 }
7633 other => panic!("unexpected health response: {other:?}"),
7634 }
7635
7636 handler
7637 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
7638 .await
7639 .unwrap();
7640 let route_response = route_task.await.unwrap();
7641 assert!(route_response.is_empty());
7642 let published = route_client_rx.recv().await.unwrap();
7643 assert!(matches!(
7644 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
7645 ClientControlResponse::RouteOpen { .. }
7646 ));
7647 }
7648
7649 async fn relay_route_open(
7654 handler: &ControlHandler,
7655 client_connection: ConnectionId,
7656 client_egress: &FrameSink,
7657 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
7658 corr: u64,
7659 module_id: &str,
7660 project_root_label: &str,
7661 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
7662 let ctx = RouteCtx {
7663 connection_id: client_connection,
7664 egress: client_egress.clone(),
7665 };
7666 let handler = handler.clone();
7667 let project_root = unique_project_root(project_root_label);
7668 let module_id = module_id.to_string();
7669 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
7670 let task = tokio::spawn(async move {
7671 let _guard = tracing::dispatcher::set_default(&dispatch);
7672 handler
7673 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
7674 .await
7675 .unwrap()
7676 });
7677 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
7678 .await
7679 .expect("module receives the relayed route.bind")
7680 .expect("module egress is open");
7681 (task, bind.frame)
7682 }
7683
7684 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
7685 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
7686 ModuleControlRequest::RouteBind {
7687 route_channel,
7688 epoch,
7689 ..
7690 } => (route_channel, epoch),
7691 other => panic!("expected a route.bind request, got {other:?}"),
7692 }
7693 }
7694
7695 fn published_route(frame: &Frame) -> (u16, u32) {
7696 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
7697 ClientControlResponse::RouteOpen {
7698 route_channel,
7699 route_epoch,
7700 } => (route_channel, route_epoch),
7701 other => panic!("expected a route.open response, got {other:?}"),
7702 }
7703 }
7704
7705 #[tokio::test]
7725 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
7726 let registry = Arc::new(Registry::default());
7727 let forwarding = Arc::new(ForwardingTable::default());
7728 let handler =
7729 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
7730
7731 let module_connection = ConnectionId::new(30);
7732 let (module_ctx, mut module_rx) = route_ctx(module_connection);
7733 hello_via_sink(
7734 &handler,
7735 &module_ctx,
7736 &mut module_rx,
7737 hello_frame("aft", PROTOCOL_VERSION, 7),
7738 )
7739 .await;
7740
7741 let dying_client = ConnectionId::new(31);
7742 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
7743
7744 let (first_task, first_bind) = relay_route_open(
7747 &handler,
7748 dying_client,
7749 &dying_ctx.egress,
7750 &mut module_rx,
7751 100,
7752 "aft",
7753 "closing-first",
7754 )
7755 .await;
7756 handler
7757 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
7758 .await
7759 .unwrap();
7760 assert!(first_task.await.unwrap().is_empty());
7761 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
7762
7763 let (second_task, second_bind) = relay_route_open(
7765 &handler,
7766 dying_client,
7767 &dying_ctx.egress,
7768 &mut module_rx,
7769 101,
7770 "aft",
7771 "closing-second",
7772 )
7773 .await;
7774 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
7775
7776 assert!(forwarding
7779 .escalate_client_delivery_failure(
7780 dying_client,
7781 first_channel,
7782 first_epoch,
7783 CloseReason::new(
7784 "module_to_client_delivery_failed",
7785 "client egress refused a module frame",
7786 ),
7787 crate::forwarding::UndeliveredFrame {
7788 module_id: None,
7789 sink: &dying_ctx.egress,
7790 },
7791 )
7792 .unwrap());
7793 assert!(!dying_ctx.egress.is_closed());
7794
7795 let ack = handler
7797 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
7798 .await;
7799 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
7800 if module_loop_error.is_some() {
7801 handler.cleanup_connection(module_connection).unwrap();
7805 }
7806 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7810 .await
7811 .ok()
7812 .flatten();
7813
7814 assert!(
7816 registry
7817 .get_module_by_connection(module_connection)
7818 .unwrap()
7819 .is_some(),
7820 "one client's closing connection ended the shared module connection: \
7821 {module_loop_error:?}"
7822 );
7823 let cotenant = ConnectionId::new(32);
7825 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
7826 let (cotenant_task, cotenant_bind) = relay_route_open(
7827 &handler,
7828 cotenant,
7829 &cotenant_ctx.egress,
7830 &mut module_rx,
7831 102,
7832 "aft",
7833 "closing-cotenant",
7834 )
7835 .await;
7836 handler
7837 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
7838 .await
7839 .unwrap();
7840 assert!(cotenant_task.await.unwrap().is_empty());
7841 let (cotenant_channel, cotenant_epoch) =
7842 published_route(&cotenant_rx.recv().await.unwrap());
7843 assert!(matches!(
7844 forwarding
7845 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
7846 .unwrap(),
7847 DataRoute::Client(DataRouteState::Bound(_))
7848 ));
7849
7850 let goodbye = post_ack_module_frame
7853 .expect("module receives a GOODBYE for the abandoned route channel");
7854 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
7855 assert_eq!(goodbye.header.channel, abandoned_channel);
7856 assert_eq!(goodbye.header.epoch, abandoned_epoch);
7857
7858 assert!(dying_rx.try_recv().is_err());
7862 let second_response = second_task.await.unwrap();
7863 assert_eq!(second_response.len(), 1);
7864 assert_eq!(
7865 parse_error(&second_response[0])["code"],
7866 "target_unavailable"
7867 );
7868 }
7869
7870 #[test]
7877 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
7878 let handler = ControlHandler::default();
7879 let module_connection = ConnectionId::new(30);
7880 let client_connection = ConnectionId::new(31);
7881
7882 handler
7883 .refuse_to_end_module_connection_for_a_client(
7884 module_connection,
7885 77,
7886 ForwardingError::ConnectionClosing {
7887 connection_id: client_connection,
7888 },
7889 )
7890 .expect("a closing client must never end the module connection");
7891
7892 assert!(matches!(
7893 handler.refuse_to_end_module_connection_for_a_client(
7894 module_connection,
7895 78,
7896 ForwardingError::ConnectionClosing {
7897 connection_id: module_connection,
7898 },
7899 ),
7900 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
7901 connection_id
7902 })) if connection_id == module_connection
7903 ));
7904 assert!(matches!(
7905 handler.refuse_to_end_module_connection_for_a_client(
7906 module_connection,
7907 79,
7908 ForwardingError::Poisoned,
7909 ),
7910 Err(RouterError::Forwarding(ForwardingError::Poisoned))
7911 ));
7912 assert!(matches!(
7913 handler.refuse_to_end_module_connection_for_a_client(
7914 module_connection,
7915 80,
7916 ForwardingError::StaleModuleEndpoint,
7917 ),
7918 Err(RouterError::Forwarding(
7919 ForwardingError::StaleModuleEndpoint
7920 ))
7921 ));
7922 }
7923
7924 #[tokio::test]
7937 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
7938 let registry = Arc::new(Registry::default());
7939 let forwarding = Arc::new(ForwardingTable::default());
7940 let supervisor = SupervisorHandle::new();
7941 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7942 let handler =
7943 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
7944 .with_supervisor(supervisor);
7945
7946 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
7947 hello_via_sink(
7948 &handler,
7949 &target_ctx,
7950 &mut target_rx,
7951 hello_frame("target", PROTOCOL_VERSION, 1),
7952 )
7953 .await;
7954
7955 let wrong_nonce = handler
7959 .handle_control_frame(
7960 &route_ctx(ConnectionId::new(91)).0,
7961 route_open_frame_with_admission_facts(
7962 20,
7963 "target",
7964 unique_project_root("admission-facts"),
7965 Some(subc_control::ConsumerIdentity {
7966 module_id: "fed".to_string(),
7967 launch_nonce: "not-the-real-nonce".to_string(),
7968 }),
7969 None,
7970 ),
7971 )
7972 .await
7973 .unwrap();
7974 assert_eq!(
7975 parse_error(&wrong_nonce[0])["code"],
7976 "bad_consumer_identity",
7977 "a mismatched launch nonce must be refused, not stamped Reserved"
7978 );
7979
7980 let never_spawned = handler
7984 .handle_control_frame(
7985 &route_ctx(ConnectionId::new(92)).0,
7986 route_open_frame_with_admission_facts(
7987 21,
7988 "target",
7989 unique_project_root("admission-facts"),
7990 Some(subc_control::ConsumerIdentity {
7991 module_id: "never-spawned".to_string(),
7992 launch_nonce: "any-nonce".to_string(),
7993 }),
7994 None,
7995 ),
7996 )
7997 .await
7998 .unwrap();
7999 assert_eq!(
8000 parse_error(&never_spawned[0])["code"],
8001 "bad_consumer_identity",
8002 "an unspawned module_id must be refused rather than accepted for lack of a record"
8003 );
8004 }
8005
8006 #[tokio::test]
8029 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
8030 let registry = Arc::new(Registry::default());
8031 let forwarding = Arc::new(ForwardingTable::default());
8032 let supervisor = SupervisorHandle::new();
8033 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8034 let handler =
8035 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8036 .with_supervisor(supervisor);
8037
8038 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
8039 hello_via_sink(
8040 &handler,
8041 &target_ctx,
8042 &mut target_rx,
8043 hello_frame("target", PROTOCOL_VERSION, 1),
8044 )
8045 .await;
8046
8047 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
8048 let route_handler = handler.clone();
8049 let route_task = tokio::spawn(async move {
8050 route_handler
8051 .handle_control_frame(
8052 &client_ctx,
8053 route_open_frame_with_admission_facts(
8054 30,
8055 "target",
8056 unique_project_root("admission-facts"),
8057 Some(subc_control::ConsumerIdentity {
8058 module_id: "fed".to_string(),
8059 launch_nonce: "fed-nonce".to_string(),
8060 }),
8061 None,
8062 ),
8063 )
8064 .await
8065 .unwrap()
8066 });
8067
8068 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8075 .await
8076 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
8077 .expect("module control channel closed before route.bind");
8078 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8079 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
8080 panic!("expected route.bind")
8081 };
8082 assert_eq!(
8083 principal,
8084 Some(Principal::Reserved {
8085 module_id: "fed".to_string()
8086 }),
8087 "a correctly attested consumer must be stamped Reserved for its own id"
8088 );
8089
8090 handler
8091 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8092 .await
8093 .unwrap();
8094 assert!(route_task.await.unwrap().is_empty());
8095 assert!(
8096 matches!(
8097 serde_json::from_slice::<ClientControlResponse>(
8098 &client_rx.recv().await.unwrap().body
8099 )
8100 .unwrap(),
8101 ClientControlResponse::RouteOpen { .. }
8102 ),
8103 "the route must actually open, not merely avoid an error"
8104 );
8105 }
8106
8107 #[tokio::test(start_paused = true)]
8108 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
8109 let registry = Arc::new(Registry::default());
8110 let forwarding = Arc::new(ForwardingTable::default());
8111 let supervisor = SupervisorHandle::new();
8112 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8113 let handler =
8114 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8115 .with_supervisor(supervisor);
8116
8117 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
8118 hello_via_sink(
8119 &handler,
8120 &target_ctx,
8121 &mut target_rx,
8122 hello_frame("target", PROTOCOL_VERSION, 1),
8123 )
8124 .await;
8125
8126 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
8127 let direct_handler = handler.clone();
8128 let direct_open = tokio::spawn(async move {
8129 direct_handler
8130 .handle_control_frame(
8131 &direct_ctx,
8132 route_open_frame(2, "target", unique_project_root("route-census-direct")),
8133 )
8134 .await
8135 .unwrap()
8136 });
8137 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8138 .await
8139 .expect("no direct route.bind within 5s")
8140 .expect("target control channel closed before direct route.bind");
8141 handler
8142 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
8143 .await
8144 .unwrap();
8145 assert!(direct_open.await.unwrap().is_empty());
8146 let _ = direct_rx.recv().await.unwrap();
8147
8148 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
8149 let reserved_handler = handler.clone();
8150 let reserved_open = tokio::spawn(async move {
8151 reserved_handler
8152 .handle_control_frame(
8153 &reserved_ctx,
8154 route_open_frame_with_admission_facts(
8155 3,
8156 "target",
8157 unique_project_root("admission-facts"),
8158 Some(ConsumerIdentity {
8159 module_id: "fed".to_string(),
8160 launch_nonce: "fed-nonce".to_string(),
8161 }),
8162 None,
8163 ),
8164 )
8165 .await
8166 .unwrap()
8167 });
8168 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8169 .await
8170 .expect("no reserved route.bind within 5s")
8171 .expect("target control channel closed before reserved route.bind");
8172 handler
8173 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
8174 .await
8175 .unwrap();
8176 assert!(reserved_open.await.unwrap().is_empty());
8177 let _ = reserved_rx.recv().await.unwrap();
8178
8179 forwarding
8180 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
8181 .unwrap();
8182 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
8183 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
8184 module_id: Some("target".to_string()),
8185 })
8186 .unwrap();
8187 let census_frame =
8188 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
8189 let response = handler
8190 .handle_control_frame(&census_ctx, census_frame)
8191 .await
8192 .unwrap()
8193 .pop()
8194 .unwrap();
8195 let actual: Value = serde_json::from_slice(&response.body).unwrap();
8196 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
8197 assert!(matches!(
8198 decoded,
8199 ClientControlResponse::SupervisorRoutes { .. }
8200 ));
8201 let routes = actual["modules"][0]["routes"].as_array().unwrap();
8202 assert_eq!(routes.len(), 2);
8203 assert!(routes.iter().all(|route| route["draining"] == true));
8204 assert!(
8207 routes.iter().all(|route| route["drain_reason"] == "reload"),
8208 "draining routes must name the drain's reason: {routes:?}"
8209 );
8210 assert!(routes.iter().any(|route| {
8211 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
8212 }));
8213 assert!(routes.iter().any(|route| {
8214 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
8215 }));
8216
8217 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8218 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
8219 if std::env::var_os("UPDATE_GOLDEN").is_some() {
8220 std::fs::write(
8221 &golden_path,
8222 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
8223 )
8224 .unwrap();
8225 }
8226 let expected: Value =
8227 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
8228 assert_eq!(actual, expected);
8229 }
8230
8231 async fn query_live_roots(
8232 handler: &ControlHandler,
8233 module_ctx: &RouteCtx,
8234 ) -> ModuleControlResponseToModule {
8235 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
8236 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
8237 let response = handler
8238 .handle_control_frame(module_ctx, frame)
8239 .await
8240 .unwrap()
8241 .pop()
8242 .unwrap();
8243 serde_json::from_slice(&response.body).unwrap()
8244 }
8245
8246 #[tokio::test(start_paused = true)]
8247 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
8248 let registry = Arc::new(Registry::default());
8249 let forwarding = Arc::new(ForwardingTable::default());
8250 let handler = ControlHandler::with_forwarding(registry, forwarding);
8251 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
8252 hello_via_sink(
8253 &handler,
8254 &target_ctx,
8255 &mut target_rx,
8256 hello_frame("target", PROTOCOL_VERSION, 1),
8257 )
8258 .await;
8259 let root = unique_project_root("live-roots-known");
8260 let path = ProjectRootId::from_path_allowing_missing(root.path())
8261 .unwrap()
8262 .as_path()
8263 .to_path_buf();
8264 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
8265 let open_handler = handler.clone();
8266 let opened = tokio::spawn(async move {
8267 open_handler
8268 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
8269 .await
8270 .unwrap()
8271 });
8272 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8273 .await
8274 .unwrap()
8275 .unwrap();
8276 handler
8277 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
8278 .await
8279 .unwrap();
8280 assert!(opened.await.unwrap().is_empty());
8281 let _ = client_rx.recv().await.unwrap();
8282
8283 let root = unique_project_root("live-roots-pending");
8284 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
8285 .unwrap()
8286 .as_path()
8287 .to_path_buf();
8288 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
8289 let open_handler = handler.clone();
8290 let pending = tokio::spawn(async move {
8291 open_handler
8292 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
8293 .await
8294 .unwrap()
8295 });
8296 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8297 .await
8298 .unwrap()
8299 .unwrap();
8300 let actual = query_live_roots(&handler, &target_ctx).await;
8301 let ModuleControlResponseToModule::LiveRoots {
8302 roots,
8303 unknown_root_bindings,
8304 total_bindings,
8305 } = actual
8306 else {
8307 panic!("expected live roots")
8308 };
8309 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
8310 assert_eq!(unknown_root_bindings, 0);
8311 assert_eq!(
8312 roots.len(),
8313 2,
8314 "root-known arm must retain each canonical root"
8315 );
8316 assert_eq!(
8317 total_bindings,
8318 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
8319 );
8320 let counts = roots
8321 .iter()
8322 .map(|root| (root.project_root.clone(), root.bound, root.pending))
8323 .collect::<Vec<_>>();
8324 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
8325 expected.sort_by(|a, b| a.0.cmp(&b.0));
8326 assert_eq!(
8327 counts, expected,
8328 "roots must sort by path and count pending separately"
8329 );
8330 handler
8331 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
8332 .await
8333 .unwrap();
8334 assert!(pending.await.unwrap().is_empty());
8335 }
8336
8337 #[tokio::test(start_paused = true)]
8338 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
8339 let forwarding = Arc::new(ForwardingTable::default());
8340 let handler =
8341 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
8342 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
8343 hello_via_sink(
8344 &handler,
8345 &target_ctx,
8346 &mut target_rx,
8347 hello_frame("target", PROTOCOL_VERSION, 1),
8348 )
8349 .await;
8350 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
8351 let pending = forwarding
8352 .begin_route_bind_relay_for_test(
8353 client_ctx.connection_id,
8354 client_ctx.egress.clone(),
8355 2,
8356 "target",
8357 )
8358 .unwrap();
8359 forwarding
8360 .complete_pending_relay(
8361 target_ctx.connection_id,
8362 pending.corr,
8363 RouteBindRelayOutcome::Accepted,
8364 )
8365 .unwrap();
8366 let actual = query_live_roots(&handler, &target_ctx).await;
8367 let ModuleControlResponseToModule::LiveRoots {
8368 roots,
8369 unknown_root_bindings,
8370 total_bindings,
8371 } = actual
8372 else {
8373 panic!("expected live roots")
8374 };
8375 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
8376 assert_eq!(
8377 unknown_root_bindings, 1,
8378 "unknown-root arm must not read as no bindings"
8379 );
8380 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
8381 assert_eq!(
8382 total_bindings,
8383 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
8384 );
8385 }
8386
8387 #[tokio::test(start_paused = true)]
8395 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
8396 let forwarding = Arc::new(ForwardingTable::default());
8397 let handler =
8398 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
8399 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
8400 let replies = handler
8401 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
8402 .await
8403 .unwrap();
8404 let queued_by_hello = module_rx.len();
8405
8406 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
8407 let open_handler = handler.clone();
8408 let open = tokio::spawn(async move {
8409 open_handler
8410 .handle_control_frame(
8411 &client_ctx,
8412 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
8413 )
8414 .await
8415 .unwrap()
8416 });
8417 let mut spins = 0;
8419 while module_rx.len() == queued_by_hello {
8420 spins += 1;
8421 assert!(spins < 10_000, "route.open never queued a route.bind");
8422 tokio::task::yield_now().await;
8423 }
8424
8425 for reply in replies {
8427 module_ctx.egress.send(reply).await.unwrap();
8428 }
8429
8430 let first = module_rx.recv().await.unwrap().frame;
8431 assert_eq!(
8432 first.header.ty,
8433 FrameType::HelloAck,
8434 "the first frame a registering module reads must be its HELLO_ACK"
8435 );
8436 assert_eq!(first.header.corr, 7);
8437 let second = module_rx.recv().await.unwrap().frame;
8438 assert_eq!(second.header.ty, FrameType::Request);
8439 assert!(
8440 matches!(
8441 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
8442 ModuleControlRequest::RouteBind { .. }
8443 ),
8444 "the route.bind follows the ack"
8445 );
8446 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
8447
8448 handler
8449 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
8450 .await
8451 .unwrap();
8452 assert!(open.await.unwrap().is_empty());
8453 let _ = client_rx.recv().await.unwrap();
8454 }
8455
8456 #[tokio::test(start_paused = true)]
8457 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
8458 let handler = ControlHandler::with_forwarding(
8459 Arc::new(Registry::default()),
8460 Arc::new(ForwardingTable::default()),
8461 );
8462 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
8463 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
8464 hello_via_sink(
8465 &handler,
8466 &first_ctx,
8467 &mut first_rx,
8468 hello_frame("first", PROTOCOL_VERSION, 1),
8469 )
8470 .await;
8471 hello_via_sink(
8472 &handler,
8473 &second_ctx,
8474 &mut second_rx,
8475 hello_frame("second", PROTOCOL_VERSION, 2),
8476 )
8477 .await;
8478 let root = unique_project_root("second-only");
8479 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
8480 let cloned = handler.clone();
8481 let open = tokio::spawn(async move {
8482 cloned
8483 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
8484 .await
8485 .unwrap()
8486 });
8487 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
8488 .await
8489 .unwrap()
8490 .unwrap();
8491 let first = query_live_roots(&handler, &first_ctx).await;
8492 let second = query_live_roots(&handler, &second_ctx).await;
8493 assert!(
8494 matches!(
8495 first,
8496 ModuleControlResponseToModule::LiveRoots {
8497 total_bindings: 0,
8498 ..
8499 }
8500 ),
8501 "cross-module scope must not expose another module's roots"
8502 );
8503 assert!(
8504 matches!(
8505 second,
8506 ModuleControlResponseToModule::LiveRoots {
8507 total_bindings: 1,
8508 ..
8509 }
8510 ),
8511 "second module must see its pending route"
8512 );
8513 handler
8514 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
8515 .await
8516 .unwrap();
8517 assert!(open.await.unwrap().is_empty());
8518 }
8519
8520 #[tokio::test(start_paused = true)]
8521 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
8522 let handler = ControlHandler::with_forwarding(
8523 Arc::new(Registry::default()),
8524 Arc::new(ForwardingTable::default()),
8525 );
8526 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
8527 hello_via_sink(
8528 &handler,
8529 &target_ctx,
8530 &mut target_rx,
8531 hello_frame("target", PROTOCOL_VERSION, 1),
8532 )
8533 .await;
8534 let actual = query_live_roots(&handler, &target_ctx).await;
8535 let ModuleControlResponseToModule::LiveRoots {
8536 roots,
8537 unknown_root_bindings,
8538 total_bindings,
8539 } = actual
8540 else {
8541 panic!("expected live roots")
8542 };
8543 assert!(roots.is_empty());
8544 assert_eq!(unknown_root_bindings, 0);
8545 assert_eq!(total_bindings, 0);
8546 assert_eq!(
8547 total_bindings,
8548 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
8549 );
8550 }
8551
8552 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
8559 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8560 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
8561 let text = std::fs::read_to_string(&path)
8562 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
8563 let vectors: Vec<(String, Value)> = text
8564 .lines()
8565 .filter(|line| !line.trim().is_empty())
8566 .map(|line| {
8567 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
8568 let id = entry["corpus_id"]
8569 .as_str()
8570 .expect("every vector carries a corpus_id")
8571 .to_string();
8572 (id, entry["package"].clone())
8573 })
8574 .collect();
8575 assert_eq!(
8579 vectors.len(),
8580 3,
8581 "vendored fed corpus changed size; re-sync from subc-federation"
8582 );
8583
8584 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
8599 let richest = vectors
8600 .iter()
8601 .filter_map(|(_, package)| package.as_object())
8602 .map(|object| {
8603 object
8604 .keys()
8605 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
8606 .count()
8607 })
8608 .max()
8609 .unwrap_or(0);
8610 assert!(
8611 richest >= 2,
8612 "vendored corpus no longer carries a package with unmodelled fields, \
8613 so the relay test can no longer distinguish a verbatim relay from a lossy one"
8614 );
8615
8616 vectors
8617 }
8618
8619 #[tokio::test]
8627 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
8628 for (corpus_id, package) in fed_admission_facts_vectors() {
8629 let registry = Arc::new(Registry::default());
8630 let forwarding = Arc::new(ForwardingTable::default());
8631 let supervisor = SupervisorHandle::new();
8632 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8633 let handler =
8634 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8635 .with_supervisor(supervisor)
8636 .with_admission_facts_config(
8637 Some("fed".to_string()),
8638 Some(vec!["target".to_string()]),
8639 );
8640
8641 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
8642 hello_via_sink(
8643 &handler,
8644 &target_ctx,
8645 &mut target_rx,
8646 hello_frame("target", PROTOCOL_VERSION, 1),
8647 )
8648 .await;
8649
8650 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
8651 let route_handler = handler.clone();
8652 let expected = package.clone();
8653 let route_task = tokio::spawn(async move {
8654 route_handler
8655 .handle_control_frame(
8656 &client_ctx,
8657 route_open_frame_with_admission_facts(
8658 20,
8659 "target",
8660 unique_project_root("admission-facts"),
8661 Some(subc_control::ConsumerIdentity {
8662 module_id: "fed".to_string(),
8663 launch_nonce: "fed-nonce".to_string(),
8664 }),
8665 Some(package),
8666 ),
8667 )
8668 .await
8669 .unwrap()
8670 });
8671
8672 let bind_frame = target_rx.recv().await.unwrap();
8673 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8674 let ModuleControlRequest::RouteBind {
8675 admission_facts, ..
8676 } = bind
8677 else {
8678 panic!("{corpus_id}: expected route.bind")
8679 };
8680 assert_eq!(
8681 admission_facts,
8682 Some(expected),
8683 "{corpus_id}: relay must not add, drop or reshape any field"
8684 );
8685
8686 handler
8687 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8688 .await
8689 .unwrap();
8690 route_task.await.unwrap();
8691 }
8692 }
8693
8694 #[tokio::test]
8695 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
8696 let registry = Arc::new(Registry::default());
8697 let forwarding = Arc::new(ForwardingTable::default());
8698 let supervisor = SupervisorHandle::new();
8699 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8700 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
8701 let handler =
8702 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8703 .with_supervisor(supervisor)
8704 .with_admission_facts_config(
8705 Some("fed".to_string()),
8706 Some(vec!["target".to_string()]),
8707 );
8708
8709 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
8710 hello_via_sink(
8711 &handler,
8712 &target_ctx,
8713 &mut target_rx,
8714 hello_frame("target", PROTOCOL_VERSION, 1),
8715 )
8716 .await;
8717 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
8718 hello_via_sink(
8719 &handler,
8720 &other_ctx,
8721 &mut other_rx,
8722 hello_frame("other", PROTOCOL_VERSION, 2),
8723 )
8724 .await;
8725
8726 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
8727 let expected_facts = facts.clone();
8728 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
8729 let route_handler = handler.clone();
8730 let route_task = tokio::spawn(async move {
8731 route_handler
8732 .handle_control_frame(
8733 &client_ctx,
8734 route_open_frame_with_admission_facts(
8735 10,
8736 "target",
8737 unique_project_root("admission-facts"),
8738 Some(subc_control::ConsumerIdentity {
8739 module_id: "fed".to_string(),
8740 launch_nonce: "fed-nonce".to_string(),
8741 }),
8742 Some(facts.clone()),
8743 ),
8744 )
8745 .await
8746 .unwrap()
8747 });
8748 let bind_frame = target_rx.recv().await.unwrap();
8749 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8750 let ModuleControlRequest::RouteBind {
8751 admission_facts, ..
8752 } = bind
8753 else {
8754 panic!("expected route.bind")
8755 };
8756 assert_eq!(admission_facts, Some(expected_facts));
8757 handler
8758 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8759 .await
8760 .unwrap();
8761 assert!(route_task.await.unwrap().is_empty());
8762 assert!(matches!(
8763 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
8764 .unwrap(),
8765 ClientControlResponse::RouteOpen { .. }
8766 ));
8767
8768 let direct = handler
8769 .handle_control_frame(
8770 &route_ctx(ConnectionId::new(73)).0,
8771 route_open_frame_with_admission_facts(
8772 11,
8773 "target",
8774 unique_project_root("admission-facts"),
8775 None,
8776 Some(json!({"x": 1})),
8777 ),
8778 )
8779 .await
8780 .unwrap();
8781 assert_eq!(
8782 parse_error(&direct[0])["code"],
8783 "admission_facts_not_permitted"
8784 );
8785
8786 let different_reserved = handler
8787 .handle_control_frame(
8788 &route_ctx(ConnectionId::new(77)).0,
8789 route_open_frame_with_admission_facts(
8790 15,
8791 "target",
8792 unique_project_root("admission-facts"),
8793 Some(subc_control::ConsumerIdentity {
8794 module_id: "other".to_string(),
8795 launch_nonce: "other-nonce".to_string(),
8796 }),
8797 Some(json!({"x": 1})),
8798 ),
8799 )
8800 .await
8801 .unwrap();
8802 assert_eq!(
8803 parse_error(&different_reserved[0])["code"],
8804 "admission_facts_not_permitted"
8805 );
8806
8807 let other_target = handler
8808 .handle_control_frame(
8809 &route_ctx(ConnectionId::new(74)).0,
8810 route_open_frame_with_admission_facts(
8811 12,
8812 "other",
8813 unique_project_root("admission-facts"),
8814 Some(subc_control::ConsumerIdentity {
8815 module_id: "fed".to_string(),
8816 launch_nonce: "fed-nonce".to_string(),
8817 }),
8818 Some(json!({"x": 1})),
8819 ),
8820 )
8821 .await
8822 .unwrap();
8823 assert_eq!(
8824 parse_error(&other_target[0])["code"],
8825 "admission_facts_target_not_allowed"
8826 );
8827
8828 let nonexistent = handler
8829 .handle_control_frame(
8830 &route_ctx(ConnectionId::new(75)).0,
8831 route_open_frame_with_admission_facts(
8832 13,
8833 "missing",
8834 unique_project_root("admission-facts"),
8835 None,
8836 Some(json!({"x": 1})),
8837 ),
8838 )
8839 .await
8840 .unwrap();
8841 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
8842
8843 let described = handler
8844 .handle_control_frame(
8845 &route_ctx(ConnectionId::new(76)).0,
8846 Frame::build(
8847 FrameType::Request,
8848 control_flags(),
8849 0,
8850 0,
8851 14,
8852 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
8853 )
8854 .unwrap(),
8855 )
8856 .await
8857 .unwrap();
8858 let ClientControlResponse::ServerDescribe { capabilities, .. } =
8859 serde_json::from_slice(&described[0].body).unwrap()
8860 else {
8861 panic!("expected server.describe response")
8862 };
8863 assert!(capabilities
8864 .iter()
8865 .any(|cap| cap == "admission_facts_relay_v1"));
8866 }
8867
8868 #[tokio::test]
8869 async fn admission_facts_without_configured_carrier_are_rejected() {
8870 let registry = Arc::new(Registry::default());
8871 let forwarding = Arc::new(ForwardingTable::default());
8872 let handler = ControlHandler::with_forwarding(registry, forwarding);
8873 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
8874 hello_via_sink(
8875 &handler,
8876 &target_ctx,
8877 &mut target_rx,
8878 hello_frame("target", PROTOCOL_VERSION, 1),
8879 )
8880 .await;
8881
8882 let responses = handler
8883 .handle_control_frame(
8884 &route_ctx(ConnectionId::new(79)).0,
8885 route_open_frame_with_admission_facts(
8886 16,
8887 "target",
8888 unique_project_root("admission-facts"),
8889 None,
8890 Some(json!({"x": 1})),
8891 ),
8892 )
8893 .await
8894 .unwrap();
8895 assert_eq!(
8896 parse_error(&responses[0])["code"],
8897 "admission_facts_not_permitted"
8898 );
8899 }
8900
8901 #[tokio::test]
8902 async fn route_open_relays_consumer_capabilities_verbatim() {
8903 let registry = Arc::new(Registry::default());
8904 let forwarding = Arc::new(ForwardingTable::default());
8905 let handler =
8906 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8907 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
8908 hello_via_sink(
8909 &handler,
8910 &module_ctx,
8911 &mut module_rx,
8912 hello_frame("aft", PROTOCOL_VERSION, 7),
8913 )
8914 .await;
8915
8916 let expected = vec!["elicitation".to_string(), "roots".to_string()];
8917 let expected_for_request = expected.clone();
8918 let project_root = unique_project_root("consumer-capabilities-present");
8919 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
8920 let route_handler = handler.clone();
8921 let route_task = tokio::spawn(async move {
8922 route_handler
8923 .handle_control_frame(
8924 &client_ctx,
8925 route_open_frame_with_consumer_capabilities(
8926 401,
8927 "aft",
8928 project_root,
8929 Some(expected_for_request),
8930 ),
8931 )
8932 .await
8933 .unwrap()
8934 });
8935 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8936 .await
8937 .unwrap()
8938 .unwrap();
8939 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8940 let ModuleControlRequest::RouteBind {
8941 consumer_capabilities,
8942 ..
8943 } = bind
8944 else {
8945 panic!("expected route.bind request, got {bind:?}");
8946 };
8947 assert_eq!(consumer_capabilities, Some(expected.clone()));
8948
8949 handler
8950 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8951 .await
8952 .unwrap();
8953 let route_response = route_task.await.unwrap();
8954 assert!(route_response.is_empty());
8955 let published = client_rx.recv().await.unwrap();
8956 assert!(matches!(
8957 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8958 ClientControlResponse::RouteOpen { .. }
8959 ));
8960 }
8961
8962 #[tokio::test]
8963 async fn route_open_without_consumer_capabilities_relays_none() {
8964 let registry = Arc::new(Registry::default());
8965 let forwarding = Arc::new(ForwardingTable::default());
8966 let handler =
8967 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8968 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
8969 hello_via_sink(
8970 &handler,
8971 &module_ctx,
8972 &mut module_rx,
8973 hello_frame("aft", PROTOCOL_VERSION, 7),
8974 )
8975 .await;
8976
8977 let project_root = unique_project_root("consumer-capabilities-absent");
8978 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
8979 let route_handler = handler.clone();
8980 let route_task = tokio::spawn(async move {
8981 route_handler
8982 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
8983 .await
8984 .unwrap()
8985 });
8986 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8987 .await
8988 .unwrap()
8989 .unwrap();
8990 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8991 let ModuleControlRequest::RouteBind {
8992 consumer_capabilities,
8993 ..
8994 } = bind
8995 else {
8996 panic!("expected route.bind request, got {bind:?}");
8997 };
8998 assert_eq!(consumer_capabilities, None);
8999
9000 handler
9001 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9002 .await
9003 .unwrap();
9004 let route_response = route_task.await.unwrap();
9005 assert!(route_response.is_empty());
9006 let published = client_rx.recv().await.unwrap();
9007 assert!(matches!(
9008 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9009 ClientControlResponse::RouteOpen { .. }
9010 ));
9011 }
9012
9013 async fn bind_role_versions_for(
9017 sent: Option<BTreeMap<String, String>>,
9018 connection: u64,
9019 ) -> Option<BTreeMap<String, String>> {
9020 let registry = Arc::new(Registry::default());
9021 let forwarding = Arc::new(ForwardingTable::default());
9022 let handler =
9023 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9024 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
9025 hello_via_sink(
9026 &handler,
9027 &module_ctx,
9028 &mut module_rx,
9029 hello_frame("aft", PROTOCOL_VERSION, 7),
9030 )
9031 .await;
9032 let project_root = unique_project_root("role-versions");
9033 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
9034 let route_handler = handler.clone();
9035 let route_task = tokio::spawn(async move {
9036 route_handler
9037 .handle_control_frame(
9038 &client_ctx,
9039 route_open_frame_with_role_versions(403, "aft", project_root, sent),
9040 )
9041 .await
9042 .unwrap()
9043 });
9044 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9045 .await
9046 .expect("a well-formed route.open reaches the module as a bind")
9047 .unwrap();
9048 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9049 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
9050 panic!("expected route.bind request, got {bind:?}");
9051 };
9052 handler
9053 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9054 .await
9055 .unwrap();
9056 assert!(route_task.await.unwrap().is_empty());
9057 let published = client_rx.recv().await.unwrap();
9058 assert!(matches!(
9059 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9060 ClientControlResponse::RouteOpen { .. }
9061 ));
9062 role_versions
9063 }
9064
9065 #[tokio::test]
9066 async fn route_open_relays_role_versions_verbatim() {
9067 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
9068 assert_eq!(
9069 bind_role_versions_for(Some(sent.clone()), 141).await,
9070 Some(sent)
9071 );
9072 }
9073
9074 #[tokio::test]
9077 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
9078 assert_eq!(bind_role_versions_for(None, 143).await, None);
9079 assert_eq!(
9080 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
9081 None
9082 );
9083 }
9084
9085 #[tokio::test]
9086 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
9087 let registry = Arc::new(Registry::default());
9088 let forwarding = Arc::new(ForwardingTable::default());
9089 let handler =
9090 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9091 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
9092 hello_via_sink(
9093 &handler,
9094 &module_ctx,
9095 &mut module_rx,
9096 hello_frame("aft", PROTOCOL_VERSION, 7),
9097 )
9098 .await;
9099
9100 let nine: BTreeMap<String, String> = (0..9)
9101 .map(|index| (format!("role-{index}"), "v1".to_string()))
9102 .collect();
9103 for (label, malformed) in [
9104 (
9105 "invalid role name",
9106 role_versions(&[("Tool_Provider", "v1")]),
9107 ),
9108 ("invalid version", role_versions(&[("tool-provider", "v0")])),
9109 ("nine entries", nine),
9110 ] {
9111 let responses = tokio::time::timeout(
9114 Duration::from_secs(1),
9115 handler.handle_control_frame(
9116 &route_ctx(ConnectionId::new(148)).0,
9117 route_open_frame_with_role_versions(
9118 404,
9119 "aft",
9120 unique_project_root("role-versions-malformed"),
9121 Some(malformed),
9122 ),
9123 ),
9124 )
9125 .await
9126 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
9127 .unwrap();
9128 assert_eq!(responses.len(), 1, "{label}");
9129 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
9130 let error = parse_error(&responses[0]);
9131 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
9132 assert_eq!(
9133 error["detail"]["field"], "role_versions",
9134 "{label}: {error}"
9135 );
9136 assert!(
9137 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
9138 "{label}: a malformed declaration is terminal"
9139 );
9140 assert!(
9141 module_rx.try_recv().is_err(),
9142 "{label}: the module must never see a bind"
9143 );
9144 }
9145 }
9146
9147 #[tokio::test]
9151 async fn route_role_versions_capability_is_advertised() {
9152 let handler = ControlHandler::new(Arc::new(Registry::default()));
9153 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
9154 let ack = hello_via_sink(
9155 &handler,
9156 &ctx,
9157 &mut rx,
9158 hello_frame("m", PROTOCOL_VERSION, 1),
9159 )
9160 .await;
9161 let ack = parse_ack(&ack);
9162 assert!(
9163 ack.subc_capabilities
9164 .iter()
9165 .any(|c| c == "route-role-versions/v1"),
9166 "{:?}",
9167 ack.subc_capabilities
9168 );
9169
9170 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
9171 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
9172 let reply = handler
9173 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
9174 .await
9175 .unwrap()
9176 .pop()
9177 .unwrap();
9178 let ClientControlResponse::ServerDescribe { capabilities, .. } =
9179 serde_json::from_slice(&reply.body).unwrap()
9180 else {
9181 panic!("not a server.describe reply");
9182 };
9183 assert!(
9184 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
9185 "{capabilities:?}"
9186 );
9187 }
9188
9189 #[tokio::test]
9190 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
9191 let registry = Arc::new(Registry::default());
9192 let forwarding = Arc::new(ForwardingTable::default());
9193 let handler =
9194 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9195 .with_health_probe_timeout(Duration::from_secs(5));
9196 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
9197 hello_via_sink(
9198 &handler,
9199 &module_ctx,
9200 &mut module_rx,
9201 non_routable_hello_frame_with_control_ops(
9202 "mcp",
9203 300,
9204 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9205 ),
9206 )
9207 .await;
9208 assert!(registry
9209 .get_module("mcp")
9210 .unwrap()
9211 .unwrap()
9212 .manifest
9213 .provides
9214 .is_empty());
9215
9216 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
9217 let route_response = handler
9218 .handle_control_frame(
9219 &route_client_ctx,
9220 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
9221 )
9222 .await
9223 .unwrap();
9224 assert_eq!(route_response[0].header.ty, FrameType::Error);
9225 assert_eq!(
9226 parse_error(&route_response[0])["code"],
9227 "target_unavailable"
9228 );
9229 assert!(parse_error(&route_response[0])["message"]
9230 .as_str()
9231 .unwrap()
9232 .contains("does not provide the requested target"));
9233 assert!(module_rx.try_recv().is_err());
9234
9235 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
9236 let health_handler = handler.clone();
9237 let health_task = tokio::spawn(async move {
9238 health_handler
9239 .handle_control_frame(
9240 &health_client_ctx,
9241 supervisor_health_probe_frame(302, "mcp"),
9242 )
9243 .await
9244 .unwrap()
9245 });
9246 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9247 .await
9248 .unwrap()
9249 .unwrap();
9250 assert_eq!(
9251 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
9252 ModuleControlRequest::HealthCheck {}
9253 );
9254 handler
9255 .handle_control_frame(
9256 &module_ctx,
9257 health_response(health_frame.header.corr, HealthStatus::Ok),
9258 )
9259 .await
9260 .unwrap();
9261 let health_response = health_task.await.unwrap();
9262 assert_eq!(health_response[0].header.ty, FrameType::Response);
9263 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
9264 ClientControlResponse::SupervisorHealthProbe {
9265 module_id, status, ..
9266 } => {
9267 assert_eq!(module_id, "mcp");
9268 assert_eq!(status, HealthStatus::Ok);
9269 }
9270 other => panic!("unexpected health response: {other:?}"),
9271 }
9272
9273 forwarding
9278 .cleanup_connection(module_ctx.connection_id)
9279 .unwrap();
9280 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
9281 let cleanup_response = tokio::time::timeout(
9282 Duration::from_millis(200),
9283 handler.handle_control_frame(
9284 &cleanup_probe_ctx,
9285 supervisor_health_probe_frame(303, "mcp"),
9286 ),
9287 )
9288 .await
9289 .expect("probe should fail immediately when the control lane is gone")
9290 .unwrap();
9291 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
9292 assert_eq!(
9293 parse_error(&cleanup_response[0])["code"],
9294 "target_unavailable"
9295 );
9296 assert!(parse_error(&cleanup_response[0])["message"]
9297 .as_str()
9298 .unwrap()
9299 .contains("no module connection"));
9300
9301 handler
9302 .cleanup_connection(module_ctx.connection_id)
9303 .unwrap();
9304 }
9305
9306 #[tokio::test]
9307 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
9308 let registry = Arc::new(Registry::default());
9309 let supervisor_handle = SupervisorHandle::new();
9310 let supervisor =
9311 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
9312 .with_handle(supervisor_handle.clone())
9313 .with_connection_file_path(
9314 std::env::temp_dir()
9315 .join(format!("subc-route-open-warming-{}", std::process::id())),
9316 );
9317 let module = supervisor
9318 .supervise_configured(
9319 ModuleSpec {
9320 module_id: "warming".to_string(),
9321 program: fake_aft_stub_path(),
9322 args: Vec::new(),
9323 env: Vec::new(),
9324 reserved: false,
9325 reserved_prefixes: Vec::new(),
9326 protocol: ModuleProtocol::Subc,
9327 overlap: Default::default(),
9328 },
9329 true,
9330 )
9331 .unwrap();
9332 assert_eq!(module.state().unwrap(), ModuleState::Running);
9333
9334 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9335 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
9336 let response = handler
9337 .handle_control_frame(
9338 &ctx,
9339 route_open_frame(304, "warming", unique_project_root("warming")),
9340 )
9341 .await
9342 .unwrap();
9343 module.stop().await.unwrap();
9344
9345 assert_eq!(response[0].header.ty, FrameType::Error);
9346 let error = parse_error(&response[0]);
9347 assert_eq!(error["code"], "module_warming");
9348 assert!(error["message"]
9349 .as_str()
9350 .unwrap()
9351 .contains("state=running, enabled=true, live=false"));
9352 }
9353
9354 #[test]
9355 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
9356 let handler = ControlHandler::new(Arc::new(Registry::default()));
9357 let capture = EventCapture::default();
9358 let _subscriber =
9359 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9360 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
9361 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
9362 let pending = (0..limit).collect::<Vec<_>>();
9363 let response = handler
9364 .route_open_capacity_refusal(
9365 &ctx,
9366 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
9367 "busy",
9368 pending.len(),
9369 limit,
9370 )
9371 .unwrap();
9372 assert_eq!(parse_error(&response)["code"], "target_unavailable");
9373 let event = capture
9374 .events()
9375 .into_iter()
9376 .find(|event| {
9377 event.target == "control"
9378 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
9379 })
9380 .expect("connection admission refusal event");
9381 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
9382 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
9383 }
9384
9385 #[test]
9386 fn route_open_target_cap_logs_admission_reason_and_capacity() {
9387 let handler = ControlHandler::new(Arc::new(Registry::default()));
9388 let capture = EventCapture::default();
9389 let _subscriber =
9390 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9391 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
9392 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
9393 let guards = (0..limit)
9394 .map(|_| {
9395 handler
9396 .route_bind_concurrency
9397 .try_admit("busy", limit)
9398 .unwrap()
9399 })
9400 .collect::<Vec<_>>();
9401 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
9402 Err(in_flight) => in_flight,
9403 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
9404 };
9405 let response = handler
9406 .route_open_target_capacity_refusal(
9407 &ctx,
9408 &route_open_frame(397, "busy", unique_project_root("target-cap")),
9409 "busy",
9410 in_flight,
9411 )
9412 .unwrap();
9413 assert_eq!(parse_error(&response)["code"], "target_unavailable");
9414 let event = capture
9415 .events()
9416 .into_iter()
9417 .find(|event| {
9418 event.target == "control"
9419 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
9420 })
9421 .expect("target admission refusal event");
9422 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
9423 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
9424 drop(guards);
9425 }
9426
9427 #[tokio::test]
9432 async fn route_open_refusal_names_the_check_that_refused() {
9433 let handler = ControlHandler::new(Arc::new(Registry::default()));
9434 let capture = EventCapture::default();
9435 let _subscriber =
9436 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9437 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
9438 let response = handler
9439 .handle_control_frame(
9440 &ctx,
9441 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
9442 )
9443 .await
9444 .unwrap();
9445
9446 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
9447 let event = capture
9448 .events()
9449 .into_iter()
9450 .find(|event| {
9451 event.target == "control"
9452 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
9453 })
9454 .expect("route.open refusal event");
9455 assert_eq!(
9456 event.fields.get("reason"),
9457 Some(&"\"not_registered\"".to_string())
9458 );
9459 }
9460
9461 #[tokio::test]
9462 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
9463 let registry = Arc::new(Registry::default());
9464 let supervisor_handle = SupervisorHandle::new();
9465 let supervisor =
9466 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
9467 .with_handle(supervisor_handle.clone())
9468 .with_connection_file_path(std::env::temp_dir().join(format!(
9469 "subc-route-open-refusal-info-{}",
9470 std::process::id()
9471 )));
9472 let module = supervisor
9473 .supervise_configured(
9474 ModuleSpec {
9475 module_id: "warming".to_string(),
9476 program: fake_aft_stub_path(),
9477 args: Vec::new(),
9478 env: Vec::new(),
9479 reserved: false,
9480 reserved_prefixes: Vec::new(),
9481 protocol: ModuleProtocol::Subc,
9482 overlap: Default::default(),
9483 },
9484 true,
9485 )
9486 .unwrap();
9487 assert_eq!(module.state().unwrap(), ModuleState::Running);
9488
9489 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9490 assert!(handler
9491 .counters()
9492 .snapshot()
9493 .get("route_open_refused_by_code")
9494 .is_none());
9495 let capture = EventCapture::default();
9496 let _subscriber =
9497 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9498 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
9499 let response = handler
9500 .handle_control_frame(
9501 &ctx,
9502 route_open_frame(394, "warming", unique_project_root("refusal-info")),
9503 )
9504 .await
9505 .unwrap();
9506 module.stop().await.unwrap();
9507
9508 assert_eq!(parse_error(&response[0])["code"], "module_warming");
9509 let event = capture
9510 .events()
9511 .into_iter()
9512 .find(|event| {
9513 event.target == "control"
9514 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
9515 })
9516 .expect("route.open refusal event");
9517 assert_eq!(
9518 event.fields.get("module_id"),
9519 Some(&"\"warming\"".to_string())
9520 );
9521 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
9522 assert_eq!(
9523 event.fields.get("reason"),
9524 Some(&"\"supervised_not_registered\"".to_string())
9525 );
9526 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
9527 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
9528 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
9529 assert_eq!(
9530 handler.counters().snapshot()["route_open_refused_by_code"],
9531 json!({ "module_warming": 1 })
9532 );
9533 }
9534
9535 const OUTAGE_START: &str = "route.open refusing module: not serving";
9536 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
9537
9538 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
9539 capture
9540 .events()
9541 .into_iter()
9542 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
9543 .collect()
9544 }
9545
9546 fn supervise_stub(
9547 registry: &Arc<Registry>,
9548 module_id: &str,
9549 enabled: bool,
9550 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
9551 let supervisor_handle = SupervisorHandle::new();
9552 let supervisor =
9553 Supervisor::new(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
9554 .with_handle(supervisor_handle.clone())
9555 .with_connection_file_path(std::env::temp_dir().join(format!(
9556 "subc-route-outage-{module_id}-{}",
9557 std::process::id()
9558 )));
9559 let module = supervisor
9560 .supervise_configured(
9561 ModuleSpec {
9562 module_id: module_id.to_string(),
9563 program: fake_aft_stub_path(),
9564 args: Vec::new(),
9565 env: Vec::new(),
9566 reserved: false,
9567 reserved_prefixes: Vec::new(),
9568 protocol: ModuleProtocol::Subc,
9569 overlap: Default::default(),
9570 },
9571 enabled,
9572 )
9573 .unwrap();
9574 (supervisor_handle, module)
9575 }
9576
9577 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
9578 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
9579 module_id: module_id.to_string(),
9580 drain_timeout_ms: Some(50),
9581 })
9582 .unwrap();
9583 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9584 }
9585
9586 #[test]
9590 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
9591 let registry = Arc::new(Registry::default());
9592 let forwarding = Arc::new(ForwardingTable::default());
9593 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9594 let second = ControlHandler::with_forwarding(registry, forwarding);
9595 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
9596 }
9597
9598 #[tokio::test(flavor = "current_thread")]
9602 async fn route_open_unknown_module_refusals_add_no_outage_state() {
9603 let handler = ControlHandler::new(Arc::new(Registry::default()));
9604 let capture = EventCapture::default();
9605 let _subscriber =
9606 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9607 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
9608 for corr in 0..8 {
9609 let response = handler
9610 .handle_control_frame(
9611 &ctx,
9612 route_open_frame(
9613 380 + corr,
9614 &format!("nobody-{corr}"),
9615 unique_project_root("outage-unknown"),
9616 ),
9617 )
9618 .await
9619 .unwrap();
9620 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
9621 }
9622
9623 assert_eq!(handler.route_outages.tracked_module_count(), 0);
9624 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
9625 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
9626 }
9627
9628 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9633 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
9634 let registry = Arc::new(Registry::default());
9635 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
9636 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9637 let capture = EventCapture::default();
9638 let _subscriber =
9639 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9640 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
9641 handler.route_outages.record_accepted("outage-restart");
9644
9645 let response = handler
9646 .handle_control_frame(
9647 &ctx,
9648 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
9649 )
9650 .await
9651 .unwrap();
9652 assert_eq!(response[0].header.ty, FrameType::Error);
9653 let starts = outage_lines(&capture, OUTAGE_START);
9654 assert_eq!(starts.len(), 1, "{starts:?}");
9655 assert_eq!(starts[0].level, tracing::Level::WARN);
9656 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
9657 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
9658 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
9659 handler.route_outages.record_accepted("outage-restart");
9660 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
9661
9662 let restart = handler
9663 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
9664 .await
9665 .unwrap();
9666 assert_eq!(
9667 restart[0].header.ty,
9668 FrameType::Response,
9669 "{:?}",
9670 parse_error(&restart[0])
9671 );
9672 handler
9673 .handle_control_frame(
9674 &ctx,
9675 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
9676 )
9677 .await
9678 .unwrap();
9679 module.stop().await.unwrap();
9680
9681 let starts = outage_lines(&capture, OUTAGE_START);
9682 assert_eq!(starts.len(), 2, "{starts:?}");
9683 assert_eq!(starts[1].level, tracing::Level::INFO);
9684 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
9685 }
9686
9687 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9691 async fn failed_operator_restart_leaves_no_operator_mark() {
9692 let registry = Arc::new(Registry::default());
9693 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
9694 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9695 let capture = EventCapture::default();
9696 let _subscriber =
9697 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9698 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
9699 handler.route_outages.record_accepted("outage-disabled");
9700
9701 let restart = handler
9702 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
9703 .await
9704 .unwrap();
9705 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
9706 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
9707
9708 handler
9709 .handle_control_frame(
9710 &ctx,
9711 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
9712 )
9713 .await
9714 .unwrap();
9715 let starts = outage_lines(&capture, OUTAGE_START);
9716 assert_eq!(starts.len(), 1, "{starts:?}");
9717 assert_eq!(starts[0].level, tracing::Level::WARN);
9718 }
9719
9720 #[tokio::test(flavor = "current_thread")]
9721 async fn route_open_unknown_module_escapes_target_module_id() {
9722 let handler = ControlHandler::new(Arc::new(Registry::default()));
9723 let capture = EventCapture::default();
9724 let _subscriber =
9725 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9726 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
9727 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
9728 let response = handler
9729 .handle_control_frame(
9730 &ctx,
9731 route_open_frame(
9732 395,
9733 hostile_module_id,
9734 unique_project_root("hostile-target-module-id"),
9735 ),
9736 )
9737 .await
9738 .unwrap();
9739
9740 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
9741 let event = capture
9742 .events()
9743 .into_iter()
9744 .find(|event| {
9745 event.target == "control"
9746 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
9747 })
9748 .expect("route.open unknown-module refusal event");
9749 let logged = event.fields.get("module_id").expect("module_id field");
9750 assert!(!logged.bytes().any(|byte| byte < 0x20));
9751 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
9752 }
9753
9754 #[tokio::test(flavor = "current_thread")]
9755 async fn route_open_module_rejection_uses_daemon_counter_key() {
9756 let registry = Arc::new(Registry::default());
9757 let forwarding = Arc::new(ForwardingTable::default());
9758 let handler =
9759 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9760 let module_connection = ConnectionId::new(95);
9761 let (module_ctx, mut module_rx) = route_ctx(module_connection);
9762 hello_via_sink(
9763 &handler,
9764 &module_ctx,
9765 &mut module_rx,
9766 hello_frame("aft", PROTOCOL_VERSION, 395),
9767 )
9768 .await;
9769
9770 let client_connection = ConnectionId::new(96);
9771 let (client_ctx, _client_rx) = route_ctx(client_connection);
9772 let capture = EventCapture::default();
9773 let _subscriber =
9774 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9775 let (route_task, bind) = relay_route_open(
9776 &handler,
9777 client_connection,
9778 &client_ctx.egress,
9779 &mut module_rx,
9780 396,
9781 "aft",
9782 "hostile-module-code",
9783 )
9784 .await;
9785 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
9786 let rejection = Frame::build(
9787 FrameType::Error,
9788 control_flags(),
9789 0,
9790 0,
9791 bind.header.corr,
9792 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
9793 )
9794 .unwrap();
9795 handler
9796 .handle_control_frame(&module_ctx, rejection)
9797 .await
9798 .unwrap();
9799
9800 let response = route_task.await.unwrap();
9801 assert_eq!(parse_error(&response[0])["code"], hostile_code);
9802 let counters = handler.counters().snapshot();
9803 assert_eq!(
9804 counters["route_open_refused_by_code"],
9805 json!({ "module_rejected": 1 })
9806 );
9807 assert!(counters["route_open_refused_by_code"]
9808 .get(hostile_code)
9809 .is_none());
9810
9811 let event = capture
9812 .events()
9813 .into_iter()
9814 .find(|event| {
9815 event.target == "control"
9816 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
9817 })
9818 .expect("route.open module-rejection refusal event");
9819 let logged = event.fields.get("module_code").expect("module_code field");
9820 assert!(!logged.bytes().any(|byte| byte < 0x20));
9821 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
9822 }
9823
9824 #[tokio::test]
9825 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
9826 let registry = Arc::new(Registry::default());
9827 let supervisor_handle = SupervisorHandle::new();
9828 let missing_program = std::env::temp_dir().join(format!(
9829 "subc-route-open-missing-program-{}",
9830 std::process::id()
9831 ));
9832 let supervisor =
9833 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
9834 .with_handle(supervisor_handle.clone());
9835 let module = supervisor
9836 .supervise_configured(
9837 ModuleSpec {
9838 module_id: "failed".to_string(),
9839 program: missing_program,
9840 args: Vec::new(),
9841 env: Vec::new(),
9842 reserved: false,
9843 reserved_prefixes: Vec::new(),
9844 protocol: ModuleProtocol::Subc,
9845 overlap: Default::default(),
9846 },
9847 true,
9848 )
9849 .unwrap();
9850 assert_eq!(module.state().unwrap(), ModuleState::Failed);
9851
9852 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9853 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
9854 let response = handler
9855 .handle_control_frame(
9856 &ctx,
9857 route_open_frame(305, "failed", unique_project_root("failed")),
9858 )
9859 .await
9860 .unwrap();
9861
9862 assert_eq!(response[0].header.ty, FrameType::Error);
9863 let error = parse_error(&response[0]);
9864 assert_eq!(error["code"], "target_unavailable");
9865 assert!(error["message"]
9866 .as_str()
9867 .unwrap()
9868 .contains("state=failed, enabled=true, live=false"));
9869 }
9870
9871 #[tokio::test]
9872 async fn route_open_role_mismatch_remains_target_unavailable() {
9873 let registry = Arc::new(Registry::default());
9874 let handler = ControlHandler::new(Arc::clone(®istry));
9875 handler
9876 .handle_control(
9877 ConnectionId::new(41),
9878 non_routable_hello_frame_with_control_ops("health-only", 306, None),
9879 )
9880 .unwrap();
9881
9882 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
9883 let response = handler
9884 .handle_control_frame(
9885 &ctx,
9886 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
9887 )
9888 .await
9889 .unwrap();
9890
9891 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
9892 assert!(parse_error(&response[0])["message"]
9893 .as_str()
9894 .unwrap()
9895 .contains("does not provide the requested target"));
9896 }
9897
9898 #[tokio::test]
9899 async fn route_open_inactive_registration_remains_target_unavailable() {
9900 let registry = Arc::new(Registry::default());
9901 let handler = ControlHandler::new(Arc::clone(®istry));
9902 handler
9903 .handle_control(
9904 ConnectionId::new(43),
9905 hello_frame("inactive", PROTOCOL_VERSION, 308),
9906 )
9907 .unwrap();
9908 assert!(registry
9909 .set_module_state_for_test("inactive", ChannelState::Closed)
9910 .unwrap());
9911
9912 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
9913 let response = handler
9914 .handle_control_frame(
9915 &ctx,
9916 route_open_frame(309, "inactive", unique_project_root("inactive")),
9917 )
9918 .await
9919 .unwrap();
9920
9921 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
9922 assert!(parse_error(&response[0])["message"]
9923 .as_str()
9924 .unwrap()
9925 .contains("is not active"));
9926 }
9927
9928 #[tokio::test]
9929 async fn late_health_reply_is_recorded_through_the_module_response_path() {
9930 let registry = Arc::new(Registry::default());
9931 let forwarding = Arc::new(ForwardingTable::default());
9932 let supervisor_handle = SupervisorHandle::new();
9933 let supervisor = Supervisor::new(Arc::clone(®istry), crate::RestartPolicy::default())
9934 .with_forwarding(Arc::clone(&forwarding))
9935 .with_handle(supervisor_handle.clone());
9936 let module = supervisor
9937 .supervise_configured(
9938 crate::ModuleSpec {
9939 module_id: "late-health-response".to_string(),
9940 program: PathBuf::from("disabled-module"),
9941 args: Vec::new(),
9942 env: Vec::new(),
9943 reserved: false,
9944 reserved_prefixes: Vec::new(),
9945 protocol: ModuleProtocol::Subc,
9946 overlap: Default::default(),
9947 },
9948 false,
9949 )
9950 .unwrap();
9951 let handler =
9952 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9953 .with_supervisor(supervisor_handle);
9954 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
9955 handler
9956 .handle_control_frame(
9957 &module_ctx,
9958 hello_frame_with_control_ops(
9959 "late-health-response",
9960 PROTOCOL_VERSION,
9961 7,
9962 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9963 ),
9964 )
9965 .await
9966 .unwrap();
9967 let probe_started_at = Instant::now() - Duration::from_millis(80);
9968 let pending = forwarding
9969 .begin_health_probe_rpc_for(
9970 "late-health-response",
9971 MODULE_CONTROL_OP_HEALTH_CHECK,
9972 probe_started_at,
9973 Instant::now() - Duration::from_millis(1),
9974 )
9975 .unwrap();
9976 assert!(forwarding
9977 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
9978 .unwrap());
9979
9980 let responses = handler
9981 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
9982 .await
9983 .unwrap();
9984
9985 assert!(responses.is_empty());
9986 let health = module.status().unwrap().health;
9987 assert_eq!(health.late_answer_count, 1);
9988 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
9989 }
9990
9991 #[tokio::test]
9992 async fn health_probe_timeout_and_module_death_are_typed() {
9993 let registry = Arc::new(Registry::default());
9994 let forwarding = Arc::new(ForwardingTable::default());
9995 let handler =
9996 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9997 .with_health_probe_timeout(Duration::from_millis(50));
9998 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
9999 hello_via_sink(
10000 &handler,
10001 &module_ctx,
10002 &mut module_rx,
10003 hello_frame_with_control_ops(
10004 "aft",
10005 PROTOCOL_VERSION,
10006 7,
10007 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10008 ),
10009 )
10010 .await;
10011
10012 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10013 let responses = handler
10014 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
10015 .await
10016 .unwrap();
10017 assert_eq!(responses[0].header.ty, FrameType::Error);
10018 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
10019 let _ = module_rx.try_recv();
10020
10021 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
10022 let health_handler = handler.clone();
10023 let death_task = tokio::spawn(async move {
10024 health_handler
10025 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
10026 .await
10027 .unwrap()
10028 });
10029 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10030 .await
10031 .unwrap()
10032 .unwrap();
10033 handler
10034 .cleanup_connection(module_ctx.connection_id)
10035 .unwrap();
10036 let responses = death_task.await.unwrap();
10037 assert_eq!(responses[0].header.ty, FrameType::Error);
10038 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
10039 }
10040
10041 #[test]
10042 fn hello_requires_exact_protocol_version() {
10043 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
10044 let registry = Arc::new(Registry::default());
10045 let handler = ControlHandler::new(Arc::clone(®istry));
10046 let responses = handler
10047 .handle_control(
10048 ConnectionId::new(connection),
10049 hello_frame("aft", offered, 9),
10050 )
10051 .unwrap();
10052
10053 assert_eq!(responses.len(), 1);
10054 assert_eq!(responses[0].header.ty, FrameType::Error);
10055 let error = parse_error(&responses[0]);
10056 assert_eq!(error["code"], "version_unsupported");
10057 assert!(registry.get_module("aft").unwrap().is_none());
10058 assert_eq!(registry.active_registration_count().unwrap(), 0);
10059 }
10060 }
10061
10062 #[test]
10063 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
10064 let registry = Arc::new(Registry::default());
10065 let forwarding = Arc::new(ForwardingTable::default());
10066 let handler =
10067 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10068 let module_connection = ConnectionId::new(301);
10069 let registration = registry
10070 .register_with_control_ops(
10071 manifest("aft-push", PROTOCOL_VERSION),
10072 PROTOCOL_VERSION,
10073 module_connection,
10074 module_baseline_control_ops(),
10075 )
10076 .unwrap();
10077 let (module_tx, _module_rx) = mpsc::channel(8);
10078 let endpoint = forwarding
10079 .register_module_connection(
10080 module_connection,
10081 "aft-push".to_string(),
10082 PROTOCOL_VERSION,
10083 manifest_concurrency(®istration.manifest),
10084 FrameSink::new(module_tx),
10085 )
10086 .unwrap();
10087
10088 let unknown = Frame::build(
10090 FrameType::Push,
10091 control_flags(),
10092 0,
10093 0,
10094 5,
10095 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
10096 )
10097 .unwrap();
10098 let out = handler.handle_status_update(endpoint, unknown).unwrap();
10099 assert!(
10100 out.is_empty(),
10101 "unknown push op must be ignored, got {out:?}"
10102 );
10103
10104 let malformed = Frame::build(
10106 FrameType::Push,
10107 control_flags(),
10108 0,
10109 0,
10110 6,
10111 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
10112 )
10113 .unwrap();
10114 let out = handler.handle_status_update(endpoint, malformed).unwrap();
10115 assert_eq!(out.len(), 1);
10116 assert_eq!(out[0].header.ty, FrameType::Error);
10117 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
10118 }
10119
10120 #[test]
10121 fn hello_rejected_when_connection_already_owns_client_routes() {
10122 let registry = Arc::new(Registry::default());
10123 let forwarding = Arc::new(ForwardingTable::default());
10124 let handler =
10125 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10126 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
10128 let client_connection = ConnectionId::new(202);
10129
10130 let responses = handler
10133 .handle_control(
10134 client_connection,
10135 hello_frame("aft-second", PROTOCOL_VERSION, 9),
10136 )
10137 .unwrap();
10138 assert_eq!(responses[0].header.ty, FrameType::Error);
10139 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
10140 assert!(registry.get_module("aft-second").unwrap().is_none());
10141 }
10142
10143 #[test]
10144 fn reserved_module_hello_requires_matching_launch_nonce() {
10145 let registry = Arc::new(Registry::default());
10146 let supervisor = SupervisorHandle::new();
10147 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
10150 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
10151
10152 let no_nonce = handler
10154 .handle_control(
10155 ConnectionId::new(1),
10156 hello_frame("vault", PROTOCOL_VERSION, 1),
10157 )
10158 .unwrap();
10159 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
10160 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
10161 assert!(registry.get_module("vault").unwrap().is_none());
10162
10163 let wrong = handler
10165 .handle_control(
10166 ConnectionId::new(2),
10167 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
10168 )
10169 .unwrap();
10170 assert_eq!(wrong[0].header.ty, FrameType::Error);
10171 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
10172 assert!(registry.get_module("vault").unwrap().is_none());
10173
10174 let ok = handler
10176 .handle_control(
10177 ConnectionId::new(3),
10178 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
10179 )
10180 .unwrap();
10181 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
10182 assert!(registry.get_module("vault").unwrap().is_some());
10183 }
10184
10185 #[test]
10186 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
10187 let registry = Arc::new(Registry::default());
10188 let supervisor = SupervisorHandle::new();
10189 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
10190 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
10191 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
10192
10193 let squat = handler
10194 .handle_control(
10195 ConnectionId::new(1),
10196 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
10197 )
10198 .unwrap();
10199 assert_eq!(squat[0].header.ty, FrameType::Error);
10200 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
10201 assert!(parse_error(&squat[0])["message"]
10202 .as_str()
10203 .unwrap()
10204 .contains("fed:"));
10205
10206 let accepted_peer = handler
10207 .handle_control(
10208 ConnectionId::new(2),
10209 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
10210 )
10211 .unwrap();
10212 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
10213
10214 let accepted_short = handler
10215 .handle_control(
10216 ConnectionId::new(3),
10217 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
10218 )
10219 .unwrap();
10220 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
10221
10222 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
10223 let response = handler
10224 .handle_control(
10225 ConnectionId::new(conn),
10226 hello_frame(module_id, PROTOCOL_VERSION, conn),
10227 )
10228 .unwrap();
10229 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
10230 }
10231 }
10232
10233 #[test]
10234 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
10235 let registry = Arc::new(Registry::default());
10236 let supervisor = SupervisorHandle::new();
10237 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
10238 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
10239 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
10240 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
10241
10242 let owner_nonce = handler
10243 .handle_control(
10244 ConnectionId::new(1),
10245 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
10246 )
10247 .unwrap();
10248 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
10249 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
10250 assert!(registry.get_module("fed:special").unwrap().is_none());
10251
10252 let exact_nonce = handler
10253 .handle_control(
10254 ConnectionId::new(2),
10255 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
10256 )
10257 .unwrap();
10258 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
10259 assert!(registry.get_module("fed:special").unwrap().is_some());
10260 }
10261
10262 #[test]
10263 fn non_reserved_module_ignores_launch_nonce() {
10264 let registry = Arc::new(Registry::default());
10265 let handler = ControlHandler::new(Arc::clone(®istry));
10268 let no_nonce = handler
10269 .handle_control(
10270 ConnectionId::new(1),
10271 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
10272 )
10273 .unwrap();
10274 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
10275 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
10276
10277 let echoed_nonce = handler
10278 .handle_control(
10279 ConnectionId::new(2),
10280 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
10281 )
10282 .unwrap();
10283 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
10284 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
10285 }
10286
10287 #[test]
10288 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
10289 let handler = ControlHandler::default();
10290 let conn = ConnectionId::new(1);
10291 let malformed = Frame::build(
10292 FrameType::Hello,
10293 control_flags(),
10294 0,
10295 0,
10296 3,
10297 b"{not json".to_vec(),
10298 )
10299 .unwrap();
10300
10301 let error = handler.handle_control(conn, malformed).unwrap();
10302 assert_eq!(error[0].header.ty, FrameType::Error);
10303 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
10304
10305 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
10306 let pong = handler.handle_control(conn, ping).unwrap();
10307 assert_eq!(pong[0].header.ty, FrameType::Pong);
10308 assert_eq!(pong[0].header.corr, 4);
10309 }
10310
10311 #[test]
10312 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
10313 let registry = Arc::new(Registry::default());
10314 let handler = ControlHandler::new(Arc::clone(®istry));
10315
10316 handler
10317 .handle_control(
10318 ConnectionId::new(1),
10319 hello_frame("aft", PROTOCOL_VERSION, 1),
10320 )
10321 .unwrap();
10322 let duplicate = handler
10323 .handle_control(
10324 ConnectionId::new(2),
10325 hello_frame("aft", PROTOCOL_VERSION, 2),
10326 )
10327 .unwrap();
10328
10329 assert_eq!(duplicate[0].header.ty, FrameType::Error);
10330 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
10331 let registration = registry.get_module("aft").unwrap().unwrap();
10332 assert_eq!(registration.connection_id, ConnectionId::new(1));
10333 }
10334
10335 #[test]
10336 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
10337 let registry = Arc::new(Registry::default());
10338 let forwarding = Arc::new(ForwardingTable::default());
10339 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
10340 let handler =
10341 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10342 .with_process_liveness(process_liveness);
10343 let (ctx, route_channel, route_epoch) =
10344 bind_liveness_route(®istry, &forwarding, "aft-dead");
10345 let responses = handler
10346 .handle_route_poll(
10347 &ctx,
10348 route_poll_frame(41, PollKind::Liveness, route_channel),
10349 route_channel,
10350 route_epoch,
10351 PollKind::Liveness,
10352 )
10353 .unwrap();
10354
10355 assert_eq!(responses.len(), 1);
10356 assert_eq!(responses[0].header.ty, FrameType::Response);
10357 assert_route_poll_liveness(&responses[0], false);
10358 }
10359
10360 #[test]
10361 fn liveness_poll_without_process_source_uses_bound_route() {
10362 let registry = Arc::new(Registry::default());
10363 let forwarding = Arc::new(ForwardingTable::default());
10364 let handler =
10365 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10366 let (ctx, route_channel, route_epoch) =
10367 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
10368 let responses = handler
10369 .handle_route_poll(
10370 &ctx,
10371 route_poll_frame(42, PollKind::Liveness, route_channel),
10372 route_channel,
10373 route_epoch,
10374 PollKind::Liveness,
10375 )
10376 .unwrap();
10377
10378 assert_route_poll_liveness(&responses[0], true);
10379 }
10380
10381 #[test]
10382 fn liveness_poll_untracked_process_source_uses_bound_route() {
10383 let registry = Arc::new(Registry::default());
10384 let forwarding = Arc::new(ForwardingTable::default());
10385 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
10386 let handler =
10387 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10388 .with_process_liveness(process_liveness);
10389 let (ctx, route_channel, route_epoch) =
10390 bind_liveness_route(®istry, &forwarding, "aft-untracked");
10391 let responses = handler
10392 .handle_route_poll(
10393 &ctx,
10394 route_poll_frame(43, PollKind::Liveness, route_channel),
10395 route_channel,
10396 route_epoch,
10397 PollKind::Liveness,
10398 )
10399 .unwrap();
10400
10401 assert_route_poll_liveness(&responses[0], true);
10402 }
10403
10404 #[tokio::test]
10405 async fn unknown_op_returns_unknown_control_op() {
10406 let handler = ControlHandler::default();
10407 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
10408 let request = Frame::build(
10409 FrameType::Request,
10410 control_flags(),
10411 0,
10412 0,
10413 55,
10414 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
10415 )
10416 .unwrap();
10417
10418 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
10419
10420 assert_eq!(response.len(), 1);
10421 assert_eq!(response[0].header.ty, FrameType::Error);
10422 assert_eq!(response[0].header.corr, 55);
10423 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
10424 }
10425
10426 #[tokio::test]
10427 async fn supervisor_provenance_rejects_unknown_exact_module() {
10428 let handler = ControlHandler::default();
10429 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
10430 let request = Frame::build(
10431 FrameType::Request,
10432 control_flags(),
10433 0,
10434 0,
10435 57,
10436 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
10437 )
10438 .unwrap();
10439
10440 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
10441
10442 assert_eq!(response.len(), 1);
10443 assert_eq!(response[0].header.ty, FrameType::Error);
10444 assert_eq!(response[0].header.corr, 57);
10445 let error = parse_error(&response[0]);
10446 assert_eq!(error["code"], "unknown_module");
10447 assert_eq!(error["message"], "module_id 'missing' is not supervised");
10448 }
10449
10450 #[test]
10451 fn provenance_probe_override_keeps_handler_tests_deterministic() {
10452 let expected = subc_control::RunningImageAgreement::Unavailable {
10453 reason: subc_control::RunningImageUnavailableReason::HashFailed,
10454 };
10455 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
10456 assert_eq!(handler.provenance_probe_override, Some(expected));
10457 }
10458
10459 #[test]
10460 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
10461 let verdict = reload_verdict(
10462 std::path::Path::new("/bin/new"),
10463 Some(std::path::Path::new("/bin/old")),
10464 subc_control::RunningImageAgreement::Unavailable {
10465 reason: subc_control::RunningImageUnavailableReason::HashFailed,
10466 },
10467 );
10468 assert!(matches!(
10469 verdict.path,
10470 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
10471 if configured == std::path::Path::new("/bin/new")
10472 && spawned_from == std::path::Path::new("/bin/old")
10473 ));
10474 }
10475
10476 #[test]
10477 fn reload_verdict_detects_replaced_image_at_same_path() {
10478 let image = subc_control::RunningImageAgreement::Mismatch {
10479 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
10480 digest: "old".into(),
10481 },
10482 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
10483 digest: "new".into(),
10484 },
10485 };
10486 let verdict = reload_verdict(
10487 std::path::Path::new("/bin/same"),
10488 Some(std::path::Path::new("/bin/same")),
10489 image.clone(),
10490 );
10491 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
10492 assert_eq!(verdict.image, image);
10493 }
10494
10495 #[test]
10496 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
10497 let image = subc_control::RunningImageAgreement::Unavailable {
10498 reason: subc_control::RunningImageUnavailableReason::NotRunning,
10499 };
10500 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
10501 assert_eq!(
10502 verdict.path,
10503 subc_control::ReloadPathAgreement::Unavailable {
10504 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
10505 }
10506 );
10507 assert_eq!(verdict.image, image);
10508
10509 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
10510 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
10511 };
10512 let verdict = reload_verdict(
10513 std::path::Path::new("/bin/same"),
10514 Some(std::path::Path::new("/bin/same")),
10515 unconfirmed.clone(),
10516 );
10517 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
10518 assert_eq!(verdict.image, unconfirmed);
10519 }
10520
10521 #[test]
10522 fn reload_verdict_preserves_each_image_unavailability_reason() {
10523 use subc_control::RunningImageUnavailableReason as Reason;
10524
10525 for reason in [
10526 Reason::NotRunning,
10527 Reason::UnsupportedPlatform,
10528 Reason::RunningExecutableUnreadable,
10529 Reason::SpawnedPathUnreadable,
10530 Reason::HashFailed,
10531 Reason::ProcessIdentityUnconfirmed,
10532 Reason::Unknown("future_probe_reason".to_string()),
10533 ] {
10534 let image = subc_control::RunningImageAgreement::Unavailable {
10535 reason: reason.clone(),
10536 };
10537 let verdict = reload_verdict(
10538 std::path::Path::new("/bin/same"),
10539 Some(std::path::Path::new("/bin/same")),
10540 image.clone(),
10541 );
10542 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
10543 assert_eq!(verdict.image, image, "{reason:?}");
10544 }
10545 }
10546
10547 #[tokio::test]
10548 async fn malformed_control_bodies_return_invalid_control_body() {
10549 let handler = ControlHandler::default();
10550 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
10551
10552 for (corr, body) in [
10553 (56, br#"{"route_channel":1}"#.as_slice()),
10554 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
10555 (
10556 58,
10557 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
10558 ),
10559 ] {
10560 let request = Frame::build(
10561 FrameType::Request,
10562 control_flags(),
10563 0,
10564 0,
10565 corr,
10566 body.to_vec(),
10567 )
10568 .unwrap();
10569 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
10570
10571 assert_eq!(response.len(), 1);
10572 assert_eq!(response[0].header.ty, FrameType::Error);
10573 assert_eq!(response[0].header.corr, corr);
10574 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
10575 }
10576 }
10577
10578 #[tokio::test]
10579 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
10580 let registry = Arc::new(Registry::default());
10581 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
10582 let router = Router::with_control_handler(Arc::clone(&control));
10583 let connection = router.begin_connection();
10584 let (ctx, mut rx) = route_ctx(connection.id());
10585
10586 router
10587 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
10588 .await
10589 .unwrap();
10590 let response = rx.recv().await.unwrap();
10591 let ack = parse_ack(&response);
10592 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
10593 let channel = 1;
10594
10595 let goodbye =
10596 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
10597 router.route_for_connection(&ctx, goodbye).await.unwrap();
10598 assert!(rx.try_recv().is_err());
10599 assert!(registry.get_module("aft").unwrap().is_none());
10600
10601 router
10602 .route_for_connection(&ctx, channel_request(channel, 13))
10603 .await
10604 .unwrap();
10605 let error_frame = rx.recv().await.unwrap();
10606 assert_eq!(error_frame.header.ty, FrameType::Error);
10607 assert_eq!(error_frame.header.channel, channel);
10608 }
10609
10610 #[tokio::test]
10611 async fn dropping_router_connection_releases_registration() {
10612 let registry = Arc::new(Registry::default());
10613 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
10614 let router = Router::with_control_handler(control);
10615 let connection = router.begin_connection();
10616 let (ctx, mut rx) = route_ctx(connection.id());
10617
10618 router
10619 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
10620 .await
10621 .unwrap();
10622 let response = rx.recv().await.unwrap();
10623 let ack = parse_ack(&response);
10624 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
10625 assert!(registry.get_module("aft").unwrap().is_some());
10626
10627 drop(connection);
10628
10629 assert!(registry.get_module("aft").unwrap().is_none());
10630 assert_eq!(registry.active_registration_count().unwrap(), 0);
10631 }
10632
10633 fn capability_manifest(
10634 module_id: &str,
10635 provides: &[&str],
10636 must_never_reach: &[&str],
10637 ) -> ModuleManifest {
10638 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
10639 manifest.capabilities = Some(CapabilityDeclarations {
10640 provides: provides
10641 .iter()
10642 .map(|capability| (*capability).to_string())
10643 .collect(),
10644 requires: Vec::new(),
10645 must_never_reach: must_never_reach
10646 .iter()
10647 .map(|capability| (*capability).to_string())
10648 .collect(),
10649 });
10650 manifest
10651 }
10652
10653 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
10654 Frame::build(
10655 FrameType::Hello,
10656 control_flags(),
10657 0,
10658 0,
10659 corr,
10660 serde_json::to_vec(&ModuleHelloBody {
10661 protocol_ver: manifest.protocol_ver,
10662 manifest,
10663 control_ops: None,
10664 launch_nonce: None,
10665 })
10666 .expect("capability test HELLO serializes"),
10667 )
10668 .expect("capability test HELLO frame builds")
10669 }
10670
10671 fn catalog_update_with_capabilities_frame(
10672 corr: u64,
10673 capabilities: CapabilityDeclarations,
10674 ) -> Frame {
10675 Frame::build(
10676 FrameType::Request,
10677 control_flags(),
10678 0,
10679 0,
10680 corr,
10681 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
10682 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
10683 capabilities: Some(capabilities),
10684 ready: None,
10685 })
10686 .expect("capability catalog.update serializes"),
10687 )
10688 .expect("capability catalog.update frame builds")
10689 }
10690
10691 async fn register_capability_manifest(
10692 handler: &ControlHandler,
10693 ctx: &RouteCtx,
10694 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
10695 manifest: ModuleManifest,
10696 corr: u64,
10697 ) {
10698 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
10699 }
10700
10701 async fn open_route_for_capability_test(
10702 handler: &ControlHandler,
10703 target_ctx: &RouteCtx,
10704 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
10705 client_connection_id: u64,
10706 corr: u64,
10707 target_module_id: &str,
10708 consumer_identity: Option<ConsumerIdentity>,
10709 ) -> (
10710 mpsc::Receiver<crate::router::OutboundFrame>,
10711 ModuleControlRequest,
10712 ) {
10713 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
10714 let route_handler = handler.clone();
10715 let target_module_id = target_module_id.to_string();
10716 let route_task = tokio::spawn(async move {
10717 route_handler
10718 .handle_control_frame(
10719 &client_ctx,
10720 route_open_frame_with_admission_facts(
10721 corr,
10722 &target_module_id,
10723 unique_project_root("admission-facts"),
10724 consumer_identity,
10725 None,
10726 ),
10727 )
10728 .await
10729 .expect("capability test route.open succeeds")
10730 });
10731 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
10732 .await
10733 .expect("capability test route.open must reach route.bind")
10734 .expect("target control receiver stays open");
10735 let bind_request: ModuleControlRequest =
10736 serde_json::from_slice(&bind.body).expect("route.bind decodes");
10737 handler
10738 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
10739 .await
10740 .expect("capability test route.bind ACK succeeds");
10741 assert!(route_task.await.expect("route.open task joins").is_empty());
10742 let opened = client_rx
10743 .recv()
10744 .await
10745 .expect("successful route.open publishes a response");
10746 assert!(matches!(
10747 serde_json::from_slice::<ClientControlResponse>(&opened.body),
10748 Ok(ClientControlResponse::RouteOpen { .. })
10749 ));
10750 (client_rx, bind_request)
10751 }
10752
10753 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
10754 assert_eq!(frame.header.ty, FrameType::Push);
10755 assert_eq!(frame.header.channel, 0);
10756 assert_eq!(
10757 serde_json::from_slice::<ClientControlPush>(&frame.body)
10758 .expect("route.closed control push decodes"),
10759 ClientControlPush::RouteClosed {
10760 module_id: target_module_id.to_string(),
10761 reason: RouteCloseReason::CapabilityDenied,
10762 drained: false,
10763 abandoned: 0,
10764 excluded_subscriptions: 0,
10765 terminal: Some(false),
10766 }
10767 );
10768 }
10769
10770 #[tokio::test]
10771 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
10772 let registry = Arc::new(Registry::default());
10773 let forwarding = Arc::new(ForwardingTable::default());
10774 let supervisor = SupervisorHandle::new();
10775 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
10776 let handler =
10777 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10778 .with_supervisor(supervisor);
10779 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
10780 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
10781 register_capability_manifest(
10782 &handler,
10783 &target_ctx,
10784 &mut target_rx,
10785 capability_manifest("target", &["credentials-provider/v1"], &[]),
10786 1,
10787 )
10788 .await;
10789 register_capability_manifest(
10790 &handler,
10791 &opener_ctx,
10792 &mut opener_rx,
10793 capability_manifest("opener", &[], &["credentials-provider/v1"]),
10794 2,
10795 )
10796 .await;
10797
10798 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
10799 let replies = handler
10800 .handle_control_frame(
10801 &client_ctx,
10802 route_open_frame_with_admission_facts(
10803 3,
10804 "target",
10805 unique_project_root("admission-facts"),
10806 Some(ConsumerIdentity {
10807 module_id: "opener".to_string(),
10808 launch_nonce: "opener-nonce".to_string(),
10809 }),
10810 None,
10811 ),
10812 )
10813 .await
10814 .expect("denied route.open returns a typed frame");
10815 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
10816 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
10817 assert!(
10818 target_rx.try_recv().is_err(),
10819 "forbidden route.open must not relay route.bind"
10820 );
10821 }
10822
10823 #[tokio::test]
10824 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
10825 let registry = Arc::new(Registry::default());
10826 let forwarding = Arc::new(ForwardingTable::default());
10827 let supervisor = SupervisorHandle::new();
10828 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
10829 let handler =
10830 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10831 .with_supervisor(supervisor);
10832 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
10833 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
10834 register_capability_manifest(
10835 &handler,
10836 &target_ctx,
10837 &mut target_rx,
10838 capability_manifest("target", &["credentials-provider/v1"], &[]),
10839 1,
10840 )
10841 .await;
10842 register_capability_manifest(
10843 &handler,
10844 &old_opener_ctx,
10845 &mut old_opener_rx,
10846 capability_manifest("opener", &[], &[]),
10847 2,
10848 )
10849 .await;
10850 let (mut client_rx, _) = open_route_for_capability_test(
10851 &handler,
10852 &target_ctx,
10853 &mut target_rx,
10854 712,
10855 3,
10856 "target",
10857 Some(ConsumerIdentity {
10858 module_id: "opener".to_string(),
10859 launch_nonce: "opener-nonce".to_string(),
10860 }),
10861 )
10862 .await;
10863 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
10864
10865 handler
10866 .cleanup_connection(old_opener_ctx.connection_id)
10867 .expect("old opener registration cleans up");
10868 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
10869 register_capability_manifest(
10870 &handler,
10871 &new_opener_ctx,
10872 &mut new_opener_rx,
10873 capability_manifest("opener", &[], &["credentials-provider/v1"]),
10874 4,
10875 )
10876 .await;
10877
10878 assert_capability_denied_push(
10879 client_rx
10880 .try_recv()
10881 .expect("HELLO deny addition must emit route.closed")
10882 .frame,
10883 "target",
10884 );
10885 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
10886 assert!(matches!(
10887 target_rx.try_recv(),
10888 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
10889 ));
10890 }
10891
10892 #[tokio::test]
10893 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
10894 let registry = Arc::new(Registry::default());
10895 let forwarding = Arc::new(ForwardingTable::default());
10896 let supervisor = SupervisorHandle::new();
10897 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
10898 let handler =
10899 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10900 .with_supervisor(supervisor);
10901 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
10902 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
10903 register_capability_manifest(
10904 &handler,
10905 &target_ctx,
10906 &mut target_rx,
10907 capability_manifest("target", &[], &[]),
10908 1,
10909 )
10910 .await;
10911 register_capability_manifest(
10912 &handler,
10913 &opener_ctx,
10914 &mut opener_rx,
10915 capability_manifest("opener", &[], &["credentials-provider/v1"]),
10916 2,
10917 )
10918 .await;
10919 let (mut client_rx, _) = open_route_for_capability_test(
10920 &handler,
10921 &target_ctx,
10922 &mut target_rx,
10923 722,
10924 3,
10925 "target",
10926 Some(ConsumerIdentity {
10927 module_id: "opener".to_string(),
10928 launch_nonce: "opener-nonce".to_string(),
10929 }),
10930 )
10931 .await;
10932 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
10933
10934 let replies = handler
10935 .handle_control_frame(
10936 &target_ctx,
10937 catalog_update_with_capabilities_frame(
10938 4,
10939 CapabilityDeclarations {
10940 provides: vec!["credentials-provider/v1".to_string()],
10941 requires: Vec::new(),
10942 must_never_reach: Vec::new(),
10943 },
10944 ),
10945 )
10946 .await
10947 .expect("claim catalog.update succeeds");
10948 assert!(matches!(
10949 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
10950 Ok(ModuleControlResponseToModule::CatalogUpdate {})
10951 ));
10952 assert_capability_denied_push(
10953 client_rx
10954 .try_recv()
10955 .expect("claim addition must emit route.closed")
10956 .frame,
10957 "target",
10958 );
10959 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
10960 assert!(matches!(
10961 target_rx.try_recv(),
10962 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
10963 ));
10964 }
10965
10966 #[tokio::test]
10967 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
10968 let registry = Arc::new(Registry::default());
10969 let forwarding = Arc::new(ForwardingTable::default());
10970 let supervisor = SupervisorHandle::new();
10971 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
10972 let handler =
10973 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10974 .with_supervisor(supervisor);
10975 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
10976 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
10977 register_capability_manifest(
10978 &handler,
10979 &target_ctx,
10980 &mut target_rx,
10981 capability_manifest("target", &["credentials-provider/v1"], &[]),
10982 1,
10983 )
10984 .await;
10985 register_capability_manifest(
10986 &handler,
10987 &opener_ctx,
10988 &mut opener_rx,
10989 capability_manifest("opener", &[], &[]),
10990 2,
10991 )
10992 .await;
10993 let (mut client_rx, _) = open_route_for_capability_test(
10994 &handler,
10995 &target_ctx,
10996 &mut target_rx,
10997 732,
10998 3,
10999 "target",
11000 Some(ConsumerIdentity {
11001 module_id: "opener".to_string(),
11002 launch_nonce: "opener-nonce".to_string(),
11003 }),
11004 )
11005 .await;
11006 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
11007
11008 handler
11009 .handle_control_frame(
11010 &target_ctx,
11011 catalog_update_with_capabilities_frame(
11012 4,
11013 CapabilityDeclarations {
11014 provides: Vec::new(),
11015 requires: Vec::new(),
11016 must_never_reach: Vec::new(),
11017 },
11018 ),
11019 )
11020 .await
11021 .expect("claim removal catalog.update succeeds");
11022 assert_eq!(
11023 forwarding.active_binding_count().unwrap(),
11024 1,
11025 "removing an attested target claim must leave the route census unchanged"
11026 );
11027 assert!(
11028 client_rx.try_recv().is_err(),
11029 "claim removal must not emit route.closed capability_denied"
11030 );
11031 assert!(
11032 target_rx.try_recv().is_err(),
11033 "claim removal must not send the target a route GOODBYE"
11034 );
11035 }
11036
11037 #[tokio::test]
11040 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
11041 let registry = Arc::new(Registry::default());
11042 let forwarding = Arc::new(ForwardingTable::default());
11043 let supervisor = SupervisorHandle::new();
11044 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11045 let handler =
11046 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11047 .with_supervisor(supervisor);
11048 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
11049 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
11050 register_capability_manifest(
11051 &handler,
11052 &target_ctx,
11053 &mut target_rx,
11054 capability_manifest("target", &["credentials-provider/v1"], &[]),
11055 1,
11056 )
11057 .await;
11058 register_capability_manifest(
11059 &handler,
11060 &opener_ctx,
11061 &mut opener_rx,
11062 capability_manifest("opener", &[], &["credentials-provider/v1"]),
11063 2,
11064 )
11065 .await;
11066
11067 let (_client_rx, bind) = open_route_for_capability_test(
11068 &handler,
11069 &target_ctx,
11070 &mut target_rx,
11071 742,
11072 3,
11073 "target",
11074 None,
11075 )
11076 .await;
11077 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
11078 panic!("direct scope-honesty route must bind");
11079 };
11080 assert_eq!(principal, Some(Principal::Direct));
11081 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
11082 }
11083
11084 #[tokio::test]
11087 async fn must_never_reach_self_route_is_capability_forbidden() {
11088 let registry = Arc::new(Registry::default());
11089 let forwarding = Arc::new(ForwardingTable::default());
11090 let supervisor = SupervisorHandle::new();
11091 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
11092 let handler =
11093 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11094 .with_supervisor(supervisor);
11095 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
11096 register_capability_manifest(
11097 &handler,
11098 &self_ctx,
11099 &mut self_rx,
11100 capability_manifest(
11101 "self-provider",
11102 &["credentials-provider/v1"],
11103 &["credentials-provider/v1"],
11104 ),
11105 1,
11106 )
11107 .await;
11108
11109 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
11110 let replies = handler
11111 .handle_control_frame(
11112 &client_ctx,
11113 route_open_frame_with_admission_facts(
11114 2,
11115 "self-provider",
11116 unique_project_root("admission-facts"),
11117 Some(ConsumerIdentity {
11118 module_id: "self-provider".to_string(),
11119 launch_nonce: "self-nonce".to_string(),
11120 }),
11121 None,
11122 ),
11123 )
11124 .await
11125 .expect("self-route refusal returns a typed frame");
11126 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
11127 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
11128 assert!(
11129 self_rx.try_recv().is_err(),
11130 "self denial must not relay route.bind"
11131 );
11132 }
11133
11134 #[test]
11135 fn unsupported_channel_zero_frame_returns_error() {
11136 let handler = ControlHandler::default();
11137 let request = Frame::build(
11138 FrameType::Request,
11139 control_flags(),
11140 0,
11141 0,
11142 21,
11143 b"opaque".to_vec(),
11144 )
11145 .unwrap();
11146
11147 let response = handler
11148 .handle_control(ConnectionId::new(1), request)
11149 .unwrap();
11150
11151 assert_eq!(response[0].header.ty, FrameType::Error);
11152 assert_eq!(
11153 parse_error(&response[0])["code"],
11154 "unsupported_control_frame"
11155 );
11156 }
11157
11158 mod swap {
11163 use super::*;
11164
11165 const INCUMBENT: ConnectionId = ConnectionId::new(30);
11166 const CANDIDATE: ConnectionId = ConnectionId::new(40);
11167
11168 struct Swap {
11169 registry: Arc<Registry>,
11170 forwarding: Arc<ForwardingTable>,
11171 handler: ControlHandler,
11172 incumbent_ctx: RouteCtx,
11173 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
11174 candidate_ctx: RouteCtx,
11175 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
11176 }
11177
11178 async fn swap_with_incumbent() -> Swap {
11179 let registry = Arc::new(Registry::default());
11180 let forwarding = Arc::new(ForwardingTable::default());
11181 let handler =
11182 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11183 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
11184 hello_via_sink(
11185 &handler,
11186 &incumbent_ctx,
11187 &mut incumbent_rx,
11188 hello_frame("aft", PROTOCOL_VERSION, 7),
11189 )
11190 .await;
11191 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
11192 Swap {
11193 registry,
11194 forwarding,
11195 handler,
11196 incumbent_ctx,
11197 incumbent_rx,
11198 candidate_ctx,
11199 candidate_rx,
11200 }
11201 }
11202
11203 fn register_candidate(swap: &Swap, ready: Option<bool>) {
11204 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
11205 candidate_manifest.ready = ready;
11206 let registration = swap
11207 .registry
11208 .register_candidate_with_control_ops(
11209 candidate_manifest,
11210 PROTOCOL_VERSION,
11211 CANDIDATE,
11212 module_baseline_control_ops(),
11213 )
11214 .unwrap();
11215 swap.forwarding
11216 .register_candidate_module_connection(
11217 CANDIDATE,
11218 "aft".to_string(),
11219 PROTOCOL_VERSION,
11220 manifest_concurrency(®istration.manifest),
11221 swap.candidate_ctx.egress.clone(),
11222 )
11223 .unwrap();
11224 }
11225
11226 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
11227 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
11228 swap.registry.promote_candidate("aft").unwrap().unwrap();
11229 cutover.incumbent.unwrap()
11230 }
11231
11232 fn keyed_total(counters: &Value, key: &str) -> u64 {
11233 counters[key]
11234 .as_object()
11235 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
11236 .unwrap_or(0)
11237 }
11238
11239 #[tokio::test]
11245 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
11246 let mut swap = swap_with_incumbent().await;
11247 let handler = swap.handler.clone();
11248
11249 let cotenant = ConnectionId::new(31);
11251 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
11252 let (cotenant_task, cotenant_bind) = relay_route_open(
11253 &handler,
11254 cotenant,
11255 &cotenant_ctx.egress,
11256 &mut swap.incumbent_rx,
11257 100,
11258 "aft",
11259 "swap-cotenant",
11260 )
11261 .await;
11262 handler
11263 .handle_control_frame(
11264 &swap.incumbent_ctx,
11265 route_bind_ack(cotenant_bind.header.corr),
11266 )
11267 .await
11268 .unwrap();
11269 assert!(cotenant_task.await.unwrap().is_empty());
11270 let (cotenant_channel, cotenant_epoch) =
11271 published_route(&cotenant_rx.recv().await.unwrap());
11272
11273 let caller = ConnectionId::new(32);
11275 let (caller_ctx, mut caller_rx) = route_ctx(caller);
11276 let (caller_task, caller_bind) = relay_route_open(
11277 &handler,
11278 caller,
11279 &caller_ctx.egress,
11280 &mut swap.incumbent_rx,
11281 101,
11282 "aft",
11283 "swap-caller",
11284 )
11285 .await;
11286 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
11287
11288 register_candidate(&swap, None);
11289 cutover(&swap);
11290
11291 let ack = handler
11293 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
11294 .await;
11295 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
11296 if module_loop_error.is_some() {
11297 handler.cleanup_connection(INCUMBENT).unwrap();
11300 }
11301
11302 assert!(
11304 cotenant_rx.try_recv().is_err(),
11305 "the co-tenant route on the incumbent was torn down by one late ack: \
11306 {module_loop_error:?}"
11307 );
11308 assert!(matches!(
11309 swap.forwarding
11310 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
11311 .unwrap(),
11312 DataRoute::Client(DataRouteState::Bound(_))
11313 ));
11314 assert_eq!(module_loop_error, None);
11315 assert!(swap
11316 .registry
11317 .get_module_by_connection(INCUMBENT)
11318 .unwrap()
11319 .is_some());
11320
11321 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
11323 .await
11324 .expect("the incumbent is told to drop the abandoned binding")
11325 .unwrap()
11326 .frame;
11327 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11328 assert_eq!(goodbye.header.channel, abandoned_channel);
11329 assert_eq!(goodbye.header.epoch, abandoned_epoch);
11330 assert!(swap.incumbent_rx.try_recv().is_err());
11331
11332 let response = caller_task.await.unwrap();
11334 assert_eq!(response.len(), 1);
11335 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
11336 assert!(caller_rx.try_recv().is_err());
11337
11338 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
11342 let counters = handler.counters().snapshot();
11343 assert_eq!(
11344 keyed_total(&counters, "route_open_accepted_by_principal"),
11345 1
11346 );
11347 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
11348 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
11349 }
11350
11351 #[tokio::test]
11355 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
11356 let mut swap = swap_with_incumbent().await;
11357 register_candidate(&swap, None);
11358 let incumbent = cutover(&swap);
11359 swap.forwarding
11360 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
11361 .unwrap()
11362 .expect("the incumbent is still registered");
11363
11364 let client = ConnectionId::new(33);
11365 let (client_ctx, mut client_rx) = route_ctx(client);
11366 let route_handler = swap.handler.clone();
11367 let open_ctx = RouteCtx {
11368 connection_id: client,
11369 egress: client_ctx.egress.clone(),
11370 };
11371 let mut route_task = tokio::spawn(async move {
11372 route_handler
11373 .handle_control_frame(
11374 &open_ctx,
11375 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
11376 )
11377 .await
11378 .unwrap()
11379 });
11380 let bind = tokio::select! {
11381 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
11382 response = &mut route_task => {
11383 let response = response.unwrap();
11384 panic!(
11385 "post-cutover route.open was refused instead of relayed to the candidate: {}",
11386 parse_error(&response[0])["code"]
11387 );
11388 }
11389 };
11390 swap.handler
11391 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
11392 .await
11393 .unwrap();
11394 assert!(route_task.await.unwrap().is_empty());
11395 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
11396 match swap
11397 .forwarding
11398 .lookup_data_route(client, channel, epoch)
11399 .unwrap()
11400 {
11401 DataRoute::Client(DataRouteState::Bound(route)) => {
11402 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
11403 }
11404 other => panic!("expected a bound route on the candidate, got {other:?}"),
11405 }
11406 assert!(swap.incumbent_rx.try_recv().is_err());
11407 }
11408
11409 #[tokio::test]
11414 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
11415 let swap = swap_with_incumbent().await;
11416 register_candidate(&swap, Some(false));
11417 let update = Frame::build(
11418 FrameType::Request,
11419 control_flags(),
11420 0,
11421 0,
11422 55,
11423 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
11424 provides: manifest("aft", PROTOCOL_VERSION).provides,
11425 capabilities: None,
11426 ready: Some(true),
11427 })
11428 .unwrap(),
11429 )
11430 .unwrap();
11431
11432 let replies = swap
11433 .handler
11434 .handle_control_frame(&swap.candidate_ctx, update)
11435 .await
11436 .unwrap();
11437
11438 assert_eq!(replies.len(), 1);
11439 assert_eq!(
11440 replies[0].header.ty,
11441 FrameType::Response,
11442 "candidate catalog.update was refused: {:?}",
11443 serde_json::from_slice::<Value>(&replies[0].body).ok()
11444 );
11445 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
11446 assert_eq!(
11447 swap.registry
11448 .get_module("aft")
11449 .unwrap()
11450 .unwrap()
11451 .connection_id,
11452 INCUMBENT
11453 );
11454 }
11455 }
11456
11457 mod swap_admission {
11461 use super::*;
11462
11463 const INCUMBENT_NONCE: &str = "incumbent-nonce";
11464 const CANDIDATE_NONCE: &str = "candidate-nonce";
11465
11466 fn handler_with_incumbent(
11467 module_id: &str,
11468 reserved: bool,
11469 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
11470 let registry = Arc::new(Registry::default());
11471 let supervisor = SupervisorHandle::new();
11472 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
11473 if reserved {
11474 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
11475 }
11476 let handler =
11477 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
11478 let incumbent = handler
11479 .handle_control(
11480 ConnectionId::new(1),
11481 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
11482 )
11483 .unwrap();
11484 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
11485 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
11486 (registry, supervisor, handler)
11487 }
11488
11489 #[test]
11495 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
11496 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
11497
11498 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
11499 let replies = handler
11500 .handle_control(
11501 ConnectionId::new(connection),
11502 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
11503 )
11504 .unwrap();
11505 assert_eq!(replies[0].header.ty, FrameType::Error);
11506 assert_eq!(
11507 parse_error(&replies[0])["code"],
11508 "swap_token_invalid",
11509 "nonce {nonce:?}"
11510 );
11511 }
11512 assert!(registry.get_candidate("aft").unwrap().is_none());
11513 assert_eq!(
11514 registry.get_module("aft").unwrap().unwrap().connection_id,
11515 ConnectionId::new(1)
11516 );
11517
11518 let admitted = handler
11521 .handle_control(
11522 ConnectionId::new(4),
11523 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
11524 )
11525 .unwrap();
11526 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
11527 assert_eq!(
11528 registry
11529 .get_candidate("aft")
11530 .unwrap()
11531 .unwrap()
11532 .connection_id,
11533 ConnectionId::new(4)
11534 );
11535 assert_eq!(
11536 registry.get_module("aft").unwrap().unwrap().connection_id,
11537 ConnectionId::new(1),
11538 "the candidate must not take the active slot"
11539 );
11540 let replayed = handler
11541 .handle_control(
11542 ConnectionId::new(5),
11543 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
11544 )
11545 .unwrap();
11546 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
11547
11548 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
11552 let squatter = handler
11553 .handle_control(
11554 ConnectionId::new(6),
11555 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
11556 )
11557 .unwrap();
11558 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
11559 assert!(
11560 registry.get_module("aft").unwrap().is_none(),
11561 "a squatter took the active slot of an id being swapped"
11562 );
11563 }
11564
11565 #[test]
11570 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
11571 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
11572
11573 let replies = handler
11574 .handle_control(
11575 ConnectionId::new(2),
11576 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
11577 )
11578 .unwrap();
11579
11580 assert_eq!(
11581 replies[0].header.ty,
11582 FrameType::HelloAck,
11583 "reserved candidate refused: {:?}",
11584 serde_json::from_slice::<Value>(&replies[0].body).ok()
11585 );
11586 assert_eq!(
11587 registry
11588 .get_candidate("vault")
11589 .unwrap()
11590 .unwrap()
11591 .connection_id,
11592 ConnectionId::new(2)
11593 );
11594 }
11595
11596 #[test]
11599 fn without_an_open_swap_the_ordinary_gates_decide() {
11600 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
11601 supervisor.close_swap("vault");
11602
11603 let candidate = handler
11604 .handle_control(
11605 ConnectionId::new(2),
11606 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
11607 )
11608 .unwrap();
11609 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
11610 let duplicate = handler
11611 .handle_control(
11612 ConnectionId::new(3),
11613 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
11614 )
11615 .unwrap();
11616 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
11617 assert!(registry.get_candidate("vault").unwrap().is_none());
11618 }
11619 }
11620
11621 mod scopes {
11625 use subc_protocol::scope::{
11626 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
11627 ScopeStatus,
11628 };
11629
11630 use super::*;
11631
11632 const OWNER: &str = "prefrontal-core";
11633
11634 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
11635 ScopeRecord {
11636 scope_ref: scope_ref.to_string(),
11637 scope_epoch,
11638 kind: ScopeKind::Head,
11639 parent: None,
11640 child_owners: Vec::new(),
11641 carriers: Vec::new(),
11642 attributes: Default::default(),
11643 }
11644 }
11645
11646 async fn call(
11647 handler: &ControlHandler,
11648 ctx: &RouteCtx,
11649 request: &ModuleControlRequestFromModule,
11650 ) -> Frame {
11651 let body = serde_json::to_vec(request).unwrap();
11652 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
11653 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
11654 assert_eq!(replies.len(), 1, "{replies:?}");
11655 replies.pop().unwrap()
11656 }
11657
11658 async fn sync(
11659 handler: &ControlHandler,
11660 ctx: &RouteCtx,
11661 generation: u64,
11662 scopes: Vec<ScopeRecord>,
11663 ) -> Result<ModuleControlResponseToModule, String> {
11664 let reply = call(
11665 handler,
11666 ctx,
11667 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
11668 )
11669 .await;
11670 match reply.header.ty {
11671 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
11672 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
11673 }
11674 }
11675
11676 async fn describe(
11677 handler: &ControlHandler,
11678 ctx: &RouteCtx,
11679 owner: &str,
11680 scope_ref: &str,
11681 ) -> ModuleControlResponseToModule {
11682 let reply = call(
11683 handler,
11684 ctx,
11685 &ModuleControlRequestFromModule::ScopeDescribe {
11686 owner: Principal::Reserved {
11687 module_id: owner.to_string(),
11688 },
11689 scope_ref: scope_ref.to_string(),
11690 },
11691 )
11692 .await;
11693 assert_eq!(
11694 reply.header.ty,
11695 FrameType::Response,
11696 "{:?}",
11697 parse_error(&reply)
11698 );
11699 serde_json::from_slice(&reply.body).unwrap()
11700 }
11701
11702 async fn module(
11704 handler: &ControlHandler,
11705 connection: u64,
11706 module_id: &str,
11707 nonce: Option<&str>,
11708 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
11709 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
11710 hello_via_sink(
11711 handler,
11712 &ctx,
11713 &mut rx,
11714 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
11715 )
11716 .await;
11717 (ctx, rx)
11718 }
11719
11720 #[tokio::test]
11723 async fn a_client_connection_cannot_sync_or_describe() {
11724 let handler = ControlHandler::new(Arc::new(Registry::default()));
11725 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
11726 for request in [
11727 ModuleControlRequestFromModule::ScopeSync {
11728 generation: 1,
11729 scopes: vec![head("s", 1)],
11730 },
11731 ModuleControlRequestFromModule::ScopeDescribe {
11732 owner: Principal::Direct,
11733 scope_ref: "s".to_string(),
11734 },
11735 ] {
11736 let reply = call(&handler, &ctx, &request).await;
11737 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
11738 }
11739 assert!(
11740 !handler
11741 .scopes
11742 .read()
11743 .unwrap()
11744 .describe(
11745 &Principal::Reserved {
11746 module_id: OWNER.to_string()
11747 },
11748 "s"
11749 )
11750 .owner_synced
11751 );
11752 }
11753
11754 #[tokio::test]
11757 async fn a_module_without_a_supervised_launch_cannot_sync() {
11758 let handler = ControlHandler::new(Arc::new(Registry::default()));
11759 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
11760 assert_eq!(
11761 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
11762 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
11763 );
11764 }
11765
11766 #[tokio::test]
11767 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
11768 let supervisor = SupervisorHandle::new();
11769 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
11770 let handler = ControlHandler::new(Arc::new(Registry::default()))
11771 .with_supervisor(supervisor.clone());
11772 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
11773 sync(&handler, &incumbent, 1, vec![head("s", 1)])
11774 .await
11775 .expect("the current launch syncs");
11776
11777 supervisor.open_swap(OWNER, "n2".to_string());
11780 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
11781 assert_eq!(
11782 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
11783 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
11784 );
11785 sync(&handler, &incumbent, 2, vec![head("s", 1)])
11786 .await
11787 .expect("the serving owner syncs during the swap");
11788
11789 supervisor.close_swap(OWNER);
11792 assert_eq!(
11793 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
11794 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
11795 );
11796 sync(&handler, &incumbent, 3, vec![head("s", 1)])
11797 .await
11798 .expect("the serving owner syncs after the rollback");
11799 handler.cleanup_connection(candidate.connection_id).unwrap();
11800
11801 supervisor.open_swap(OWNER, "n3".to_string());
11806 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
11807 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
11808 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
11809 .await
11810 .expect("the promoted launch takes authority");
11811 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
11812 panic!("unexpected reply {reply:?}");
11813 };
11814 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
11815 assert_eq!(
11816 sync(&handler, &incumbent, 4, Vec::new()).await,
11817 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
11818 );
11819 }
11820
11821 #[tokio::test]
11824 async fn closing_the_authority_connection_frees_sync_authority() {
11825 let supervisor = SupervisorHandle::new();
11826 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
11827 let handler = ControlHandler::new(Arc::new(Registry::default()))
11828 .with_supervisor(supervisor.clone());
11829 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
11830 sync(&handler, &first, 10, vec![head("s", 1)])
11831 .await
11832 .unwrap();
11833 handler.cleanup_connection(first.connection_id).unwrap();
11834
11835 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
11836 sync(&handler, &second, 1, vec![head("s", 1)])
11837 .await
11838 .expect("the next connection takes the released authority");
11839 }
11840
11841 #[tokio::test]
11842 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
11843 let registry = Arc::new(Registry::default());
11844 let supervisor_handle = SupervisorHandle::new();
11845 let supervisor = Supervisor::new(Arc::clone(®istry), RestartPolicy::default())
11846 .with_handle(supervisor_handle.clone())
11847 .with_daemon_incarnation("incarnation-7".to_string());
11848 supervisor
11851 .supervise_configured(
11852 ModuleSpec {
11853 module_id: OWNER.to_string(),
11854 program: PathBuf::from("/nonexistent/prefrontal-core"),
11855 args: Vec::new(),
11856 env: Vec::new(),
11857 reserved: false,
11858 reserved_prefixes: Vec::new(),
11859 protocol: ModuleProtocol::Subc,
11860 overlap: Default::default(),
11861 },
11862 false,
11863 )
11864 .unwrap();
11865 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
11866 let handler =
11867 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11868 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
11869
11870 let ModuleControlResponseToModule::ScopeDescribe {
11872 status,
11873 daemon_incarnation,
11874 owner_synced,
11875 owner_configured,
11876 scope,
11877 ..
11878 } = describe(&handler, &reader, OWNER, "s").await
11879 else {
11880 panic!("not a describe reply");
11881 };
11882 assert_eq!(status, ScopeStatus::NotLive);
11883 assert_eq!(daemon_incarnation, "incarnation-7");
11884 assert!(!owner_synced);
11885 assert!(owner_configured);
11886 assert!(scope.is_none());
11887
11888 let ModuleControlResponseToModule::ScopeDescribe {
11891 status,
11892 owner_configured,
11893 ..
11894 } = describe(&handler, &reader, "ghost", "s").await
11895 else {
11896 panic!("not a describe reply");
11897 };
11898 assert_eq!(status, ScopeStatus::NotLive);
11899 assert!(!owner_configured);
11900
11901 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
11903 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
11904 let ModuleControlResponseToModule::ScopeDescribe {
11905 status,
11906 scope_epoch,
11907 owner_synced,
11908 scope,
11909 ..
11910 } = describe(&handler, &reader, OWNER, "s").await
11911 else {
11912 panic!("not a describe reply");
11913 };
11914 assert_eq!(status, ScopeStatus::Live);
11915 assert_eq!(scope_epoch, Some(4));
11916 assert!(owner_synced);
11917 let stamp = scope.expect("a live scope carries its stamp");
11918 assert!(
11919 stamp.owner_authorized,
11920 "prefrontal-core is the default authority"
11921 );
11922 assert_eq!(stamp.kind, ScopeKind::Head);
11923 }
11924
11925 #[tokio::test]
11926 async fn scope_authority_owners_decides_owner_authorized() {
11927 let supervisor = SupervisorHandle::new();
11928 supervisor.set_spawn_nonce("broca", "b1".to_string());
11929 let handler = ControlHandler::new(Arc::new(Registry::default()))
11930 .with_supervisor(supervisor)
11931 .with_scope_authority_owners(vec!["broca".to_string()]);
11932 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
11933 let mut gated = head("s", 1);
11934 gated.attributes.agent_id = Some("agent".to_string());
11935 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
11936 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
11937 describe(&handler, &broca, "broca", "s").await
11938 else {
11939 panic!("not a describe reply");
11940 };
11941 assert!(scope.unwrap().owner_authorized);
11942 }
11943
11944 #[tokio::test]
11948 async fn scope_ops_and_the_scopes_capability_are_advertised() {
11949 let handler = ControlHandler::new(Arc::new(Registry::default()));
11950 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
11951 let ack = hello_via_sink(
11952 &handler,
11953 &ctx,
11954 &mut rx,
11955 hello_frame("m", PROTOCOL_VERSION, 1),
11956 )
11957 .await;
11958 let ack = parse_ack(&ack);
11959 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
11960 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
11961 }
11962 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
11963
11964 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
11965 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11966 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11967 let reply = handler
11968 .handle_control_frame(&client, frame)
11969 .await
11970 .unwrap()
11971 .pop()
11972 .unwrap();
11973 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11974 serde_json::from_slice(&reply.body).unwrap()
11975 else {
11976 panic!("not a server.describe reply");
11977 };
11978 assert!(
11979 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
11980 "{capabilities:?}"
11981 );
11982 }
11983
11984 const PLEXUS: &str = "plexus";
11987 const OTHER: &str = "other";
11988 const AFT: &str = "aft";
11989 const BROCA: &str = "broca";
11990 const MAGIC: &str = "magic-context";
11991
11992 fn nonce(module_id: &str) -> String {
11993 format!("nonce-{module_id}")
11994 }
11995
11996 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
11997 let (tx, rx) = mpsc::channel(64);
11998 (
11999 RouteCtx {
12000 connection_id: ConnectionId::new(connection),
12001 egress: FrameSink::new(tx),
12002 },
12003 rx,
12004 )
12005 }
12006
12007 struct Rig {
12011 handler: ControlHandler,
12012 forwarding: Arc<ForwardingTable>,
12013 owner: RouteCtx,
12014 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12015 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
12016 generation: u64,
12017 next_connection: u64,
12018 _supervisor: Supervisor,
12019 }
12020
12021 async fn rig() -> Rig {
12022 let registry = Arc::new(Registry::default());
12023 let forwarding = Arc::new(ForwardingTable::default());
12024 let supervisor_handle = SupervisorHandle::new();
12025 let supervisor = Supervisor::new(Arc::clone(®istry), RestartPolicy::default())
12026 .with_handle(supervisor_handle.clone());
12027 supervisor
12028 .supervise_configured(
12029 ModuleSpec {
12030 module_id: OWNER.to_string(),
12031 program: PathBuf::from("/nonexistent/prefrontal-core"),
12032 args: Vec::new(),
12033 env: Vec::new(),
12034 reserved: false,
12035 reserved_prefixes: Vec::new(),
12036 protocol: ModuleProtocol::Subc,
12037 overlap: Default::default(),
12038 },
12039 false,
12040 )
12041 .unwrap();
12042 for module_id in [OWNER, AFT, BROCA, MAGIC] {
12043 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
12044 }
12045 let handler =
12046 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12047 .with_supervisor(supervisor_handle);
12048 let (owner, mut owner_rx) = wide_ctx(1);
12049 hello_via_sink(
12050 &handler,
12051 &owner,
12052 &mut owner_rx,
12053 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
12054 )
12055 .await;
12056 let mut modules = BTreeMap::new();
12057 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
12058 let (ctx, mut rx) = wide_ctx(connection);
12059 hello_via_sink(
12060 &handler,
12061 &ctx,
12062 &mut rx,
12063 hello_frame(module_id, PROTOCOL_VERSION, connection),
12064 )
12065 .await;
12066 modules.insert(module_id.to_string(), (ctx, rx));
12067 }
12068 Rig {
12069 handler,
12070 forwarding,
12071 owner,
12072 _owner_rx: owner_rx,
12073 modules,
12074 generation: 0,
12075 next_connection: 100,
12076 _supervisor: supervisor,
12077 }
12078 }
12079
12080 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
12081 ScopeCarrier {
12082 principal: Principal::Reserved {
12083 module_id: module_id.to_string(),
12084 },
12085 targets: targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()),
12086 }
12087 }
12088
12089 fn session(scope_epoch: u64) -> ScopeRecord {
12092 let mut record = head("s", scope_epoch);
12093 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
12094 record.attributes.agent_id = Some("agent-1".to_string());
12095 record.attributes.delegates = true;
12096 record
12097 }
12098
12099 impl Rig {
12100 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
12101 self.generation += 1;
12102 sync(&self.handler, &self.owner, self.generation, scopes)
12103 .await
12104 .expect("the owner's sync is accepted");
12105 }
12106
12107 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
12108 ScopeSelector {
12109 owner: Principal::Reserved {
12110 module_id: OWNER.to_string(),
12111 },
12112 scope_ref: scope_ref.to_string(),
12113 scope_epoch,
12114 }
12115 }
12116
12117 fn open_frame(
12118 &mut self,
12119 opener: Option<&str>,
12120 target: &str,
12121 scope: Option<ScopeSelector>,
12122 ) -> (
12123 RouteCtx,
12124 mpsc::Receiver<crate::router::OutboundFrame>,
12125 Frame,
12126 ) {
12127 self.next_connection += 1;
12128 let (ctx, rx) = wide_ctx(self.next_connection);
12129 let root = unique_project_root("scoped-open");
12130 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
12131 target: RouteTarget::ToolProvider {
12132 module_id: target.to_string(),
12133 },
12134 identity: BindIdentity::new(
12135 root.path().to_path_buf(),
12136 "unit".to_string(),
12137 "session".to_string(),
12138 ),
12139 consumer_identity: opener.map(|module_id| ConsumerIdentity {
12140 module_id: module_id.to_string(),
12141 launch_nonce: nonce(module_id),
12142 }),
12143 consumer_capabilities: None,
12144 role_versions: None,
12145 admission_facts: None,
12146 scope,
12147 })
12148 .unwrap();
12149 let frame = Frame::build(
12150 FrameType::Request,
12151 control_flags(),
12152 0,
12153 0,
12154 self.next_connection,
12155 body,
12156 )
12157 .unwrap();
12158 (ctx, rx, frame)
12159 }
12160
12161 async fn refused(
12163 &mut self,
12164 opener: Option<&str>,
12165 target: &str,
12166 scope: Option<ScopeSelector>,
12167 ) -> String {
12168 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
12169 let replies = self
12170 .handler
12171 .handle_control_frame(&ctx, frame)
12172 .await
12173 .unwrap();
12174 assert_eq!(replies.len(), 1, "{replies:?}");
12175 assert_eq!(replies[0].header.ty, FrameType::Error);
12176 let (_, module_rx) = self.modules.get_mut(target).unwrap();
12177 assert!(
12178 module_rx.try_recv().is_err(),
12179 "a refused open relays nothing"
12180 );
12181 parse_error(&replies[0])["code"]
12182 .as_str()
12183 .unwrap()
12184 .to_string()
12185 }
12186
12187 async fn relayed(
12189 &mut self,
12190 opener: Option<&str>,
12191 target: &str,
12192 scope: Option<ScopeSelector>,
12193 ) -> Relayed {
12194 let (ctx, rx, frame) = self.open_frame(opener, target, scope);
12195 let handler = self.handler.clone();
12196 let task_ctx = ctx.clone();
12197 let task = tokio::spawn(async move {
12198 handler
12199 .handle_control_frame(&task_ctx, frame)
12200 .await
12201 .unwrap()
12202 });
12203 let (_, module_rx) = self.modules.get_mut(target).unwrap();
12204 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
12205 .await
12206 .expect("the target receives the relayed route.bind")
12207 .unwrap()
12208 .frame;
12209 Relayed {
12210 target: target.to_string(),
12211 client: ctx,
12212 client_rx: rx,
12213 task,
12214 bind,
12215 }
12216 }
12217
12218 async fn ack(&self, relayed: &Relayed) {
12219 let (module, _) = &self.modules[&relayed.target];
12220 self.handler
12221 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
12222 .await
12223 .unwrap();
12224 }
12225
12226 async fn bound(
12228 &mut self,
12229 opener: Option<&str>,
12230 target: &str,
12231 scope: Option<ScopeSelector>,
12232 ) -> Bound {
12233 let relayed = self.relayed(opener, target, scope).await;
12234 self.ack(&relayed).await;
12235 let Relayed {
12236 target,
12237 client,
12238 mut client_rx,
12239 task,
12240 bind,
12241 } = relayed;
12242 assert!(
12243 task.await.unwrap().is_empty(),
12244 "the open is answered by commit"
12245 );
12246 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
12247 Bound {
12248 target,
12249 client,
12250 client_rx,
12251 channel,
12252 epoch,
12253 bind,
12254 }
12255 }
12256
12257 fn live(&self, route: &Bound) -> bool {
12258 matches!(
12259 self.forwarding
12260 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
12261 .unwrap(),
12262 DataRoute::Client(DataRouteState::Bound(_))
12263 )
12264 }
12265 }
12266
12267 struct Relayed {
12268 target: String,
12269 client: RouteCtx,
12270 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12271 task: tokio::task::JoinHandle<Vec<Frame>>,
12272 bind: Frame,
12273 }
12274
12275 struct Bound {
12276 target: String,
12277 client: RouteCtx,
12278 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12279 channel: u16,
12280 epoch: u32,
12281 bind: Frame,
12282 }
12283
12284 impl Bound {
12285 fn closed_reason(&mut self) -> RouteCloseReason {
12288 let mut reason = None;
12289 let mut goodbye = false;
12290 while let Ok(outbound) = self.client_rx.try_recv() {
12291 let frame = outbound.frame;
12292 match frame.header.ty {
12293 FrameType::Goodbye => {
12294 assert_eq!(
12295 (frame.header.channel, frame.header.epoch),
12296 (self.channel, self.epoch)
12297 );
12298 goodbye = true;
12299 }
12300 FrameType::Push => {
12301 let ClientControlPush::RouteClosed {
12302 reason: r,
12303 module_id,
12304 ..
12305 } = serde_json::from_slice(&frame.body).unwrap()
12306 else {
12307 panic!("unexpected push");
12308 };
12309 assert_eq!(module_id, self.target);
12310 reason = Some(r);
12311 }
12312 other => panic!("unexpected frame {other:?}"),
12313 }
12314 }
12315 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
12316 reason.expect("the client is told why the route closed")
12317 }
12318
12319 fn untouched(&mut self) -> bool {
12320 self.client_rx.try_recv().is_err()
12321 }
12322
12323 fn stamp(&self) -> Option<ScopeStamp> {
12324 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
12325 ModuleControlRequest::RouteBind { scope, .. } => scope,
12326 other => panic!("expected a route.bind, got {other:?}"),
12327 }
12328 }
12329 }
12330
12331 #[tokio::test]
12332 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
12333 ) {
12334 let mut rig = rig().await;
12335 let mut record = session(1);
12336 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
12337 record.child_owners = vec![Principal::Reserved {
12338 module_id: MAGIC.to_string(),
12339 }];
12340 rig.sync(vec![record]).await;
12341 let scope = || Some(rig_selector("s", Some(1)));
12342
12343 rig.bound(Some(OWNER), PLEXUS, scope()).await;
12346 rig.bound(Some(AFT), OTHER, scope()).await;
12347 rig.bound(Some(BROCA), PLEXUS, scope()).await;
12348
12349 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
12353 assert_eq!(
12354 rig.refused(opener, target, scope()).await,
12355 error_codes::SCOPE_NOT_CARRIER,
12356 "{opener:?} -> {target}"
12357 );
12358 }
12359 }
12360
12361 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
12362 ScopeSelector {
12363 owner: Principal::Reserved {
12364 module_id: OWNER.to_string(),
12365 },
12366 scope_ref: scope_ref.to_string(),
12367 scope_epoch,
12368 }
12369 }
12370
12371 #[tokio::test]
12372 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
12373 let mut rig = rig().await;
12374 rig.sync(vec![session(1)]).await;
12375 for opener in [OWNER, AFT] {
12376 assert_eq!(
12377 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
12378 .await,
12379 error_codes::SCOPE_EPOCH_REQUIRED,
12380 "{opener}"
12381 );
12382 }
12383 }
12384
12385 #[tokio::test]
12386 async fn admission_separates_not_synced_not_live_and_ended() {
12387 let mut rig = rig().await;
12388 let code = rig
12390 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12391 .await;
12392 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
12393 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
12394
12395 let ghost = ScopeSelector {
12397 owner: Principal::Reserved {
12398 module_id: "ghost".to_string(),
12399 },
12400 scope_ref: "s".to_string(),
12401 scope_epoch: Some(1),
12402 };
12403 assert_eq!(
12404 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
12405 error_codes::SCOPE_NOT_LIVE
12406 );
12407
12408 rig.sync(vec![session(2)]).await;
12409 assert_eq!(
12410 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
12411 .await,
12412 error_codes::SCOPE_NOT_LIVE
12413 );
12414 for epoch in [1, 3] {
12415 assert_eq!(
12416 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
12417 .await,
12418 error_codes::SCOPE_ENDED,
12419 "epoch {epoch}"
12420 );
12421 }
12422 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
12424 .await;
12425 }
12426
12427 #[tokio::test]
12428 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
12429 let mut rig = rig().await;
12430 rig.sync(vec![session(1)]).await;
12431 let route = rig
12432 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12433 .await;
12434 let stamp = route.stamp().expect("a scoped bind carries the stamp");
12435 assert_eq!(stamp.scope_ref, "s");
12436 assert_eq!(stamp.scope_epoch, 1);
12437 assert_eq!(stamp.kind, ScopeKind::Head);
12438 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
12439 assert!(stamp.attributes.delegates);
12440 assert!(stamp.owner_authorized);
12441 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
12442 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
12443
12444 let (broca, mut broca_rx) = wide_ctx(50);
12447 hello_via_sink(
12448 &rig.handler,
12449 &broca,
12450 &mut broca_rx,
12451 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
12452 )
12453 .await;
12454 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
12455 .await
12456 .unwrap();
12457 let own = ScopeSelector {
12458 owner: Principal::Reserved {
12459 module_id: BROCA.to_string(),
12460 },
12461 scope_ref: "b".to_string(),
12462 scope_epoch: Some(1),
12463 };
12464 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
12465 assert!(!route.stamp().unwrap().owner_authorized);
12466 }
12467
12468 #[tokio::test]
12473 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
12474 let mut rig = rig().await;
12475 rig.sync(vec![session(1)]).await;
12476 let mut cotenant = rig
12477 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
12478 .await;
12479
12480 let mut changed = session(1);
12481 changed.child_owners.push(Principal::Reserved {
12482 module_id: MAGIC.to_string(),
12483 });
12484 let mut ended = None;
12485 for (code, next) in [
12486 (error_codes::SCOPE_CHANGED, vec![changed]),
12487 (error_codes::SCOPE_ENDED, Vec::new()),
12488 ] {
12489 let relayed = rig
12490 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12491 .await;
12492 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
12493 ended = Some(next.is_empty());
12494 rig.sync(next).await;
12495 rig.ack(&relayed).await;
12496 let replies = relayed.task.await.unwrap();
12497 assert_eq!(replies.len(), 1, "{replies:?}");
12498 assert_eq!(parse_error(&replies[0])["code"], code);
12499 assert_eq!(
12500 subc_protocol::error_codes::is_retryable_route_open(code),
12501 code == error_codes::SCOPE_CHANGED
12502 );
12503 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
12504 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
12506 let mut goodbyes = Vec::new();
12509 while let Ok(outbound) = plexus_rx.try_recv() {
12510 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
12511 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
12512 }
12513 assert!(
12514 goodbyes.contains(&(bind_channel, bind_epoch)),
12515 "{goodbyes:?}"
12516 );
12517 assert!(rig
12518 .handler
12519 .registry
12520 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
12521 .unwrap()
12522 .is_some());
12523 }
12524 assert_eq!(ended, Some(true));
12525 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
12528 }
12529
12530 #[tokio::test]
12533 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
12534 struct Case {
12535 name: &'static str,
12536 change: fn(&mut ScopeRecord),
12537 closed: [Option<RouteCloseReason>; 4],
12540 }
12541 use RouteCloseReason::*;
12542 let cases = [
12543 Case {
12544 name: "a carrier entry removed",
12545 change: |r| {
12546 r.carriers.retain(|c| {
12547 c.principal
12548 != Principal::Reserved {
12549 module_id: AFT.to_string(),
12550 }
12551 })
12552 },
12553 closed: [None, Some(ScopeCarrierRemoved), None, None],
12554 },
12555 Case {
12556 name: "a target removed from a carrier",
12557 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
12558 closed: [None, None, None, Some(ScopeCarrierRemoved)],
12559 },
12560 Case {
12561 name: "a bare carrier narrowed to targets",
12562 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
12563 closed: [None, Some(ScopeCarrierRemoved), None, None],
12564 },
12565 Case {
12566 name: "delegates turned off",
12567 change: |r| r.attributes.delegates = false,
12568 closed: [Some(ScopeDelegationChanged); 4],
12569 },
12570 Case {
12571 name: "agent_id changed",
12572 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
12573 closed: [Some(ScopeDelegationChanged); 4],
12574 },
12575 Case {
12576 name: "a carrier added, child owners changed, the record re-sent",
12577 change: |r| {
12578 r.carriers.push(carrier(MAGIC, None));
12579 r.child_owners.push(Principal::Reserved {
12580 module_id: MAGIC.to_string(),
12581 });
12582 },
12583 closed: [None; 4],
12584 },
12585 Case {
12586 name: "a target added",
12587 change: |r| {
12588 r.carriers[1]
12589 .targets
12590 .as_mut()
12591 .unwrap()
12592 .push("third".to_string())
12593 },
12594 closed: [None; 4],
12595 },
12596 Case {
12597 name: "delegates turned on",
12598 change: |r| r.attributes.delegates = true,
12599 closed: [None; 4],
12600 },
12601 ];
12602 for case in cases {
12603 let mut rig = rig().await;
12604 rig.sync(vec![session(1)]).await;
12605 let scope = || Some(rig_selector("s", Some(1)));
12606 let mut routes = [
12607 rig.bound(Some(OWNER), PLEXUS, scope()).await,
12608 rig.bound(Some(AFT), PLEXUS, scope()).await,
12609 rig.bound(Some(BROCA), PLEXUS, scope()).await,
12610 rig.bound(Some(BROCA), OTHER, scope()).await,
12611 ];
12612 let mut record = session(1);
12613 (case.change)(&mut record);
12614 rig.sync(vec![record]).await;
12615 for (index, expected) in case.closed.iter().enumerate() {
12616 let route = &mut routes[index];
12617 match expected {
12618 Some(reason) => {
12619 assert!(!rig.live(route), "{}: route {index} still live", case.name);
12620 assert_eq!(
12621 route.closed_reason(),
12622 *reason,
12623 "{}: route {index}",
12624 case.name
12625 );
12626 }
12627 None => {
12628 assert!(rig.live(route), "{}: route {index} closed", case.name);
12629 assert!(
12630 route.untouched(),
12631 "{}: route {index} was told something",
12632 case.name
12633 );
12634 }
12635 }
12636 }
12637 }
12638 }
12639
12640 #[tokio::test]
12641 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
12642 for next in [Vec::new(), vec![session(2)]] {
12644 let mut rig = rig().await;
12645 rig.sync(vec![session(1)]).await;
12646 let mut route = rig
12647 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12648 .await;
12649 rig.sync(next).await;
12650 assert!(!rig.live(&route));
12651 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
12652 }
12653
12654 let mut rig = rig().await;
12657 let mut child = session(1);
12658 child.scope_ref = "child".to_string();
12659 child.kind = ScopeKind::Worker;
12660 child.parent = Some(ScopeParent {
12661 owner: Principal::Reserved {
12662 module_id: OWNER.to_string(),
12663 },
12664 scope_ref: "s".to_string(),
12665 scope_epoch: 1,
12666 });
12667 rig.sync(vec![session(1), child.clone()]).await;
12668 let mut child_route = rig
12669 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
12670 .await;
12671 assert_eq!(
12672 child_route.stamp().unwrap().parent_state,
12673 Some(ParentState::Linked)
12674 );
12675 rig.sync(vec![child]).await;
12676 assert!(!rig.live(&child_route));
12677 assert_eq!(
12678 child_route.closed_reason(),
12679 RouteCloseReason::ScopeParentEnded
12680 );
12681 }
12682
12683 #[tokio::test]
12684 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
12685 ) {
12686 let mut rig = rig().await;
12687 rig.sync(vec![session(1)]).await;
12688 let mut route = rig
12689 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12690 .await;
12691 let before = rig.forwarding.published_scope_tag(OWNER, "s");
12692 rig.sync(vec![session(1)]).await;
12693 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
12694 assert!(rig.live(&route) && route.untouched());
12695
12696 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
12700 .forwarding
12701 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
12702 .unwrap()
12703 else {
12704 panic!("the route is bound");
12705 };
12706 binding.flow.acquire_tagged(9, false).await.unwrap();
12707 let mut widened = session(1);
12708 widened.carriers.push(carrier(MAGIC, None));
12709 rig.sync(vec![widened]).await;
12710 assert!(rig.live(&route) && route.untouched());
12711 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
12712 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
12713 assert_eq!(binding.flow.in_flight(), 1);
12716 binding
12717 .flow
12718 .acquire_tagged(10, false)
12719 .await
12720 .expect("the flow is still open");
12721 }
12722
12723 #[tokio::test]
12726 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
12727 let mut rig = rig().await;
12728 rig.sync(vec![session(1)]).await;
12729 let mut on_incumbent = rig
12730 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12731 .await;
12732
12733 let (candidate, _candidate_rx) = wide_ctx(9);
12736 let registration = rig
12737 .handler
12738 .registry
12739 .register_candidate_with_control_ops(
12740 manifest(PLEXUS, PROTOCOL_VERSION),
12741 PROTOCOL_VERSION,
12742 candidate.connection_id,
12743 module_baseline_control_ops(),
12744 )
12745 .unwrap();
12746 rig.forwarding
12747 .register_candidate_module_connection(
12748 candidate.connection_id,
12749 PLEXUS.to_string(),
12750 PROTOCOL_VERSION,
12751 manifest_concurrency(®istration.manifest),
12752 candidate.egress.clone(),
12753 )
12754 .unwrap();
12755 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
12756 rig.handler
12757 .registry
12758 .promote_candidate(PLEXUS)
12759 .unwrap()
12760 .unwrap();
12761 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
12762
12763 rig.sync(Vec::new()).await;
12764 assert!(!rig.live(&on_incumbent));
12765 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
12766 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
12767 let goodbye = incumbent_rx
12768 .try_recv()
12769 .expect("the superseded endpoint is told")
12770 .frame;
12771 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
12772 }
12773 }
12774}
12775
12776#[cfg(test)]
12777mod concurrency_default_exposure_tests {
12778 use super::*;
12779
12780 fn hello_body(role_json: &str) -> Vec<u8> {
12781 format!(
12782 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
12783 )
12784 .into_bytes()
12785 }
12786
12787 fn manifest_from(body: &[u8]) -> ModuleManifest {
12788 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
12789 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
12790 .expect("manifest parses")
12791 }
12792
12793 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
12794
12795 #[test]
12796 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
12797 let body = hello_body(&format!(
12798 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
12799 ));
12800 let manifest = manifest_from(&body);
12801 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
12804 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
12805 }
12806
12807 #[test]
12808 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
12809 let body = hello_body(&format!(
12810 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
12811 ));
12812 let manifest = manifest_from(&body);
12813 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
12814 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
12815 }
12816
12817 #[test]
12818 fn non_management_roles_are_never_reported() {
12819 let body = hello_body(
12820 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
12821 );
12822 let manifest = manifest_from(&body);
12823 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
12824 }
12825}