1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 path::{Path, PathBuf},
5 sync::{Arc, Mutex, RwLock},
6 time::{Duration, Instant as StdInstant},
7};
8
9use serde::{Deserialize, Serialize};
10use subc_control::{
11 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
12 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
13 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
14 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
15 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
16 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
17 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
18};
19use subc_protocol::{
20 error_codes,
21 manifest::{
22 validate_hello_capability_grammar, validate_hello_self_signal_declarations,
23 CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
24 ProviderRole,
25 },
26 scope::{
27 ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
28 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_DESCRIBE_OP, SCOPE_SYNC_OP,
29 },
30 session::{
31 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
32 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
33 MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE, ROLE_VERSIONS_FIELD,
34 },
35 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
36 Priority, RouteTarget, PROTOCOL_VERSION,
37};
38use tokio::time::{timeout_at, Instant};
39use tracing::{debug, info, warn};
40
41use crate::{
42 capability_requirements::{
43 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
44 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
45 RuntimeModule,
46 },
47 daemon_config::RestartRequiredSection,
48 forwarding::{
49 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
50 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
51 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
52 },
53 observability::{
54 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
55 },
56 provenance::{
57 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
58 },
59 registry::{ChannelState, ConnectionId, Registry, RegistryError},
60 router::{RouteCtx, RouterError},
61 scopes::{BoundScope, HelloLaunchNonces, ScopeTable},
62 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
63 stderr_tail::{CaptureState, TailEntry},
64 supervise::{
65 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
66 SupervisorHandle, SwapHelloAdmission,
67 },
68 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
69};
70
71pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
77
78const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
79const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
80const CAP_PING_PONG: &str = "ping_pong_v1";
81const CAP_SESSION_ATTACH: &str = "session_attach_v1";
82const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
83
84const SUBC_CONTROL_OPS: &[&str] = &[
85 ops::SERVER_DESCRIBE,
86 ops::CATALOG_LIST,
87 ops::ROUTE_OPEN,
88 ops::ROUTE_POLL,
89 ops::ROUTE_CLOSING,
90 ops::ROUTE_CLOSED,
91 ops::SUPERVISOR_LIST,
92 ops::SUPERVISOR_RESTART,
93 ops::SUPERVISOR_SWAP,
94 ops::SUPERVISOR_RELOAD,
95 ops::SUPERVISOR_RESCAN,
96 ops::SUPERVISOR_RELEASE_RESERVED,
97 ops::SUPERVISOR_SET_ENABLED,
98 ops::SUPERVISOR_HEALTH_PROBE,
99 ops::SUPERVISOR_HEALTH,
100 ops::SUPERVISOR_STDERR_TAIL,
101 ops::SUPERVISOR_TERMINALS,
102 ops::SUPERVISOR_ROUTES,
103 ops::SUPERVISOR_PROVENANCE,
104 ops::SUPERVISOR_SPAWN_SNAPSHOT,
105 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
106];
107
108const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
109 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
110 "supervisor.live_roots",
111 SCOPE_SYNC_OP,
112 SCOPE_DESCRIBE_OP,
113];
114
115const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
119
120const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
121
122pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
130
131pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
146
147pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
165
166const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
167const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
168
169fn reload_verdict(
170 configured: &Path,
171 spawned_from: Option<&Path>,
172 image: subc_control::RunningImageAgreement,
173) -> PendingReloadVerdict {
174 let path = match spawned_from {
175 Some(spawned_from) if configured == spawned_from => ReloadPathAgreement::Match,
176 Some(spawned_from) => ReloadPathAgreement::Mismatch {
177 configured: configured.to_path_buf(),
178 spawned_from: spawned_from.to_path_buf(),
179 },
180 None => ReloadPathAgreement::Unavailable {
181 reason: if matches!(
182 image,
183 subc_control::RunningImageAgreement::Unavailable {
184 reason: subc_control::RunningImageUnavailableReason::NotRunning
185 }
186 ) {
187 ReloadPathUnavailableReason::NotRunning
188 } else {
189 ReloadPathUnavailableReason::SpawnedPathUnavailable
190 },
191 },
192 };
193 PendingReloadVerdict { path, image }
194}
195
196#[derive(Clone)]
197struct DaemonProvenanceFacts {
198 build: DaemonBuildProvenance,
199 pid: Option<u32>,
200 started_at_ms: Option<u64>,
201 start_clock: Option<crate::clock::StartClock>,
202 executable_path: Option<PathBuf>,
203 executable_identity: Option<SpawnedFileIdentity>,
204 process_start_time: Option<u64>,
205 probe: ExecutableIdentityProbe,
206}
207
208impl Default for DaemonProvenanceFacts {
209 fn default() -> Self {
210 Self {
211 build: DaemonBuildProvenance {
212 build_git_sha: None,
213 build_lock_digest: None,
214 },
215 pid: None,
216 started_at_ms: None,
217 start_clock: None,
218 executable_path: None,
219 executable_identity: None,
220 process_start_time: None,
221 probe: ExecutableIdentityProbe::default(),
222 }
223 }
224}
225
226#[derive(Debug, Clone)]
227struct SupervisorRescanContext {
228 supervisor: Supervisor,
229 config_path: PathBuf,
230 configured_port: Option<u16>,
231 storage_config: Option<crate::daemon_config::StorageConfig>,
232 admission_facts_carrier_module_id: Option<String>,
233 admission_facts_targets: Option<Vec<String>>,
234 scope_authority_owners: Vec<String>,
235}
236
237const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
258 "reloading",
259 "supervisor_not_live",
260 "registration_not_active",
261 "no_forwarding_connection",
262 "relay_send_failed",
263];
264
265#[derive(Clone)]
267pub struct ControlHandler {
268 registry: Arc<Registry>,
269 forwarding: Arc<ForwardingTable>,
270 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
271 supervisor: SupervisorHandle,
272 subc_capabilities: Arc<[String]>,
273 route_bind_relay_timeout: Duration,
277 route_bind_relay_timeouts: BTreeMap<String, Duration>,
281 route_bind_breakers: RouteBindBreakers,
284 route_bind_concurrency: RouteBindConcurrency,
287 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
292 route_bind_breaker_threshold: u32,
294 route_bind_breaker_cooldown: Duration,
296 health_probe_timeout: Duration,
297 storage_config: Option<crate::daemon_config::StorageConfig>,
300 machine_id: Option<crate::machine_id::MachineId>,
304 admission_facts_carrier_module_id: Option<String>,
305 admission_facts_targets: Option<Vec<String>>,
306 scopes: Arc<RwLock<ScopeTable>>,
310 scope_authority_owners: Vec<String>,
313 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
316 rescan: Option<SupervisorRescanContext>,
317 connected_clients: ConnectedClients,
318 counters: DaemonCounters,
319 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
320 daemon_provenance: DaemonProvenanceFacts,
321 #[cfg(test)]
322 control_dispatch_delay: Option<Duration>,
323 #[cfg(test)]
324 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
325}
326
327impl fmt::Debug for ControlHandler {
328 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
329 f.debug_struct("ControlHandler")
330 .field("registry", &self.registry)
331 .field("forwarding", &self.forwarding)
332 .field("process_liveness", &self.process_liveness.is_some())
333 .field("supervisor", &self.supervisor)
334 .field("subc_capabilities", &self.subc_capabilities)
335 .finish()
336 }
337}
338
339struct RouteOpenRequest {
340 target: RouteTarget,
341 identity: BindIdentity,
342 consumer_identity: Option<ConsumerIdentity>,
343 consumer_capabilities: Option<Vec<String>>,
344 role_versions: Option<BTreeMap<String, String>>,
345 admission_facts: Option<serde_json::Value>,
346 scope: Option<ScopeSelector>,
347}
348
349struct RouteBindReservationGuard {
350 forwarding: Arc<ForwardingTable>,
351 endpoint: ModuleEndpointId,
352 relay_corr: u64,
353 armed: bool,
354}
355
356struct ModuleControlRpcGuard {
357 forwarding: Arc<ForwardingTable>,
358 endpoint: ModuleEndpointId,
359 corr: u64,
360 armed: bool,
361}
362
363impl ModuleControlRpcGuard {
364 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
365 Self {
366 forwarding,
367 endpoint,
368 corr,
369 armed: true,
370 }
371 }
372
373 fn disarm(&mut self) {
374 self.armed = false;
375 }
376}
377
378impl Drop for ModuleControlRpcGuard {
379 fn drop(&mut self) {
380 if self.armed {
381 let _ = self
382 .forwarding
383 .cancel_module_control_rpc(self.endpoint, self.corr);
384 }
385 }
386}
387
388impl RouteBindReservationGuard {
389 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
390 Self {
391 forwarding,
392 endpoint,
393 relay_corr,
394 armed: true,
395 }
396 }
397
398 fn release_and_disarm(&mut self) {
399 if !self.armed {
400 return;
401 }
402 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
403 self.endpoint,
404 self.relay_corr,
405 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
406 ) {
407 send_goodbye_target_best_effort(
408 &self.forwarding.counters(),
409 &target,
410 "canceled route.bind",
411 );
412 }
413 self.armed = false;
414 }
415
416 fn disarm(&mut self) {
417 self.armed = false;
418 }
419}
420
421impl Drop for RouteBindReservationGuard {
422 fn drop(&mut self) {
423 self.release_and_disarm();
424 }
425}
426
427#[derive(Debug, Clone, Default)]
457pub(crate) struct RouteBindBreakers {
458 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
459}
460
461#[derive(Debug, Clone, Default)]
462pub(crate) struct RouteBindConcurrency {
463 modules: Arc<Mutex<HashMap<String, usize>>>,
464}
465
466struct RouteBindConcurrencyGuard {
467 concurrency: RouteBindConcurrency,
468 module_id: String,
469}
470
471impl RouteBindConcurrency {
472 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
475 let mut modules = self
476 .modules
477 .lock()
478 .expect("route.bind concurrency mutex poisoned");
479 let in_flight = modules.entry(module_id.to_string()).or_default();
480 if *in_flight >= limit {
481 return Err(*in_flight);
482 }
483 *in_flight += 1;
484 Ok(RouteBindConcurrencyGuard {
485 concurrency: self.clone(),
486 module_id: module_id.to_string(),
487 })
488 }
489}
490
491impl Drop for RouteBindConcurrencyGuard {
492 fn drop(&mut self) {
493 let mut modules = self
494 .concurrency
495 .modules
496 .lock()
497 .expect("route.bind concurrency mutex poisoned");
498 let remove = {
499 let in_flight = modules
500 .get_mut(&self.module_id)
501 .expect("admitted route.bind has a concurrency entry");
502 *in_flight -= 1;
503 *in_flight == 0
504 };
505 if remove {
506 modules.remove(&self.module_id);
507 }
508 }
509}
510
511#[derive(Debug, Default)]
512struct ModuleBreakerState {
513 consecutive_timeouts: u32,
515 cooldown_until: Option<Instant>,
518 probe_in_flight: Option<Arc<()>>,
523}
524
525enum RouteBindAdmission<'a> {
527 Admitted {
528 guard: RouteBindBreakerGuard<'a>,
529 probe: bool,
532 },
533 Refused {
534 consecutive_timeouts: u32,
535 retry_in: Duration,
539 probe_in_flight: bool,
540 },
541}
542
543struct RouteBindBreakerGuard<'a> {
550 breakers: RouteBindBreakers,
551 module_id: &'a str,
552 probe_token: Option<Arc<()>>,
553 settled: bool,
554}
555
556impl RouteBindBreakerGuard<'_> {
557 fn record_accepted(&mut self) -> bool {
561 self.settled = true;
562 self.breakers.record_accepted(self.module_id)
563 }
564
565 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
568 self.settled = true;
569 self.breakers.record_timeout(
570 self.module_id,
571 self.probe_token.as_ref(),
572 threshold,
573 cooldown,
574 )
575 }
576
577 fn record_inconclusive(&mut self) {
586 self.settled = true;
587 self.breakers
588 .record_inconclusive(self.module_id, self.probe_token.as_ref());
589 }
590}
591
592impl Drop for RouteBindBreakerGuard<'_> {
593 fn drop(&mut self) {
594 if !self.settled {
595 self.breakers
596 .record_inconclusive(self.module_id, self.probe_token.as_ref());
597 }
598 }
599}
600
601struct BreakerOpened {
605 consecutive_timeouts: u32,
606 reopened_after_probe: bool,
609}
610
611impl RouteBindBreakers {
612 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
613 self.modules
614 .lock()
615 .expect("route.bind breaker mutex poisoned")
616 }
617
618 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
621 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
622 probe: probe_token.is_some(),
623 guard: RouteBindBreakerGuard {
624 breakers: self.clone(),
625 module_id,
626 probe_token,
627 settled: false,
628 },
629 };
630
631 let mut modules = self.lock();
632 let Some(state) = modules.get_mut(module_id) else {
633 return admitted(None);
634 };
635 let Some(cooldown_until) = state.cooldown_until else {
636 return admitted(None);
637 };
638 if state.probe_in_flight.is_some() {
639 return RouteBindAdmission::Refused {
640 consecutive_timeouts: state.consecutive_timeouts,
641 retry_in: Duration::ZERO,
642 probe_in_flight: true,
643 };
644 }
645 let now = Instant::now();
646 if now < cooldown_until {
647 return RouteBindAdmission::Refused {
648 consecutive_timeouts: state.consecutive_timeouts,
649 retry_in: cooldown_until - now,
650 probe_in_flight: false,
651 };
652 }
653 let token = Arc::new(());
654 state.probe_in_flight = Some(Arc::clone(&token));
655 admitted(Some(token))
656 }
657
658 fn record_accepted(&self, module_id: &str) -> bool {
659 self.lock()
660 .remove(module_id)
661 .is_some_and(|state| state.cooldown_until.is_some())
662 }
663
664 fn record_timeout(
665 &self,
666 module_id: &str,
667 probe_token: Option<&Arc<()>>,
668 threshold: u32,
669 cooldown: Duration,
670 ) -> Option<BreakerOpened> {
671 let mut modules = self.lock();
672 let state = modules.entry(module_id.to_string()).or_default();
673 let was_open = state.cooldown_until.is_some();
674 let was_probe = Self::owns_probe(state, probe_token);
675 if was_probe {
676 state.probe_in_flight = None;
677 }
678 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
679 if state.consecutive_timeouts < threshold {
680 return None;
681 }
682 state.cooldown_until = Some(Instant::now() + cooldown);
683 Some(BreakerOpened {
684 consecutive_timeouts: state.consecutive_timeouts,
685 reopened_after_probe: was_open && was_probe,
686 })
687 }
688
689 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
690 state
694 .probe_in_flight
695 .as_ref()
696 .zip(token)
697 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
698 }
699
700 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
701 if let Some(state) = self.lock().get_mut(module_id) {
702 if Self::owns_probe(state, probe_token) {
703 state.probe_in_flight = None;
704 }
705 }
706 }
707
708 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
723 self.lock()
724 .remove(module_id)
725 .map(|state| state.consecutive_timeouts)
726 .filter(|discarded| *discarded > 0)
727 }
728
729 fn open_snapshot(&self) -> Option<serde_json::Value> {
736 let now = Instant::now();
737 let modules = self.lock();
738 let open = modules
739 .iter()
740 .filter_map(|(module_id, state)| {
741 let cooldown_until = state.cooldown_until?;
742 Some((
743 module_id.clone(),
744 serde_json::json!({
745 "consecutive_timeouts": state.consecutive_timeouts,
746 "cooldown_remaining_ms":
747 cooldown_until.saturating_duration_since(now).as_millis() as u64,
748 "probe_in_flight": state.probe_in_flight.is_some(),
749 }),
750 ))
751 })
752 .collect::<serde_json::Map<String, serde_json::Value>>();
753 (!open.is_empty()).then_some(serde_json::Value::Object(open))
754 }
755}
756
757impl ControlHandler {
758 pub fn new(registry: Arc<Registry>) -> Self {
759 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
760 }
761
762 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
763 let counters = forwarding.counters();
764 let route_bind_breakers = forwarding.route_bind_breakers();
768 let route_bind_concurrency = forwarding.route_bind_concurrency();
769 let route_outages = forwarding.route_outages();
770 Self {
771 registry,
772 forwarding,
773 process_liveness: None,
774 supervisor: SupervisorHandle::new(),
775 subc_capabilities: Arc::from([
776 CAP_MANIFEST_REGISTRATION.to_string(),
777 CAP_CHANNEL_LIFECYCLE.to_string(),
778 CAP_PING_PONG.to_string(),
779 CAP_SESSION_ATTACH.to_string(),
780 CAP_ADMISSION_FACTS_RELAY.to_string(),
781 CAP_SCOPES_V1.to_string(),
782 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
783 ]),
784 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
785 route_bind_relay_timeouts: BTreeMap::new(),
786 route_bind_breakers,
787 route_bind_concurrency,
788 route_outages,
789 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
790 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
791 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
792 storage_config: None,
793 machine_id: None,
794 admission_facts_carrier_module_id: None,
795 admission_facts_targets: None,
796 scopes: Arc::new(RwLock::new(ScopeTable::new(
797 crate::daemon_config::default_scope_authority_owners(),
798 ))),
799 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
800 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
801 rescan: None,
802 connected_clients: ConnectedClients::new(),
803 counters,
804 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
805 daemon_provenance: DaemonProvenanceFacts::default(),
806 #[cfg(test)]
807 control_dispatch_delay: None,
808 #[cfg(test)]
809 provenance_probe_override: None,
810 }
811 }
812
813 pub fn with_storage_config(
816 mut self,
817 storage_config: Option<crate::daemon_config::StorageConfig>,
818 ) -> Self {
819 self.storage_config = storage_config;
820 self
821 }
822
823 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
826 self.machine_id = machine_id;
827 self
828 }
829
830 pub fn with_admission_facts_config(
834 mut self,
835 carrier_module_id: Option<String>,
836 targets: Option<Vec<String>>,
837 ) -> Self {
838 self.admission_facts_carrier_module_id = carrier_module_id;
839 self.admission_facts_targets = targets;
840 self
841 }
842
843 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
847 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
848 self.scope_authority_owners = owners;
849 self
850 }
851
852 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
855 self.route_bind_relay_timeout = timeout;
856 self
857 }
858
859 pub fn with_route_bind_relay_timeouts(
865 mut self,
866 timeouts: impl IntoIterator<Item = (String, Duration)>,
867 ) -> Self {
868 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
869 self
870 }
871
872 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
878 self.route_bind_relay_timeouts
879 .get(module_id)
880 .copied()
881 .unwrap_or(self.route_bind_relay_timeout)
882 }
883
884 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
892 self.route_bind_breaker_threshold = threshold.max(1);
893 self.route_bind_breaker_cooldown = cooldown;
894 self
895 }
896
897 #[cfg(test)]
898 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
899 self.health_probe_timeout = timeout;
900 self
901 }
902
903 #[cfg(test)]
904 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
905 self.control_dispatch_delay = Some(delay);
906 self
907 }
908
909 pub fn with_process_liveness(
910 mut self,
911 process_liveness: Arc<dyn ModuleProcessLiveness>,
912 ) -> Self {
913 self.process_liveness = Some(process_liveness);
914 self
915 }
916
917 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
918 self.supervisor = supervisor;
919 self
920 }
921
922 pub fn with_daemon_provenance(
923 mut self,
924 pid: u32,
925 started_at_ms: u64,
926 executable_path: Option<PathBuf>,
927 build_git_sha: Option<String>,
928 build_lock_digest: Option<String>,
929 ) -> Self {
930 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
931 let process_start_time = process_start_time(pid);
932 self.daemon_provenance = DaemonProvenanceFacts {
933 build: DaemonBuildProvenance {
934 build_git_sha,
935 build_lock_digest,
936 },
937 pid: Some(pid),
938 started_at_ms: Some(started_at_ms),
939 start_clock: None,
940 executable_path,
941 executable_identity,
942 process_start_time,
943 probe: ExecutableIdentityProbe::default(),
944 };
945 self
946 }
947
948 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
949 self.daemon_provenance.start_clock = Some(clock);
950 self
951 }
952
953 #[cfg(test)]
954 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
955 self.provenance_probe_override = Some(result);
956 self
957 }
958
959 pub fn with_capability_config(
963 self,
964 modules: impl IntoIterator<Item = (String, bool)>,
965 reserved_capabilities: BTreeMap<String, String>,
966 ) -> Self {
967 self.capability_evaluator
968 .configure(modules, reserved_capabilities);
969 self
970 }
971
972 pub fn with_supervisor_rescan(
973 mut self,
974 supervisor: Supervisor,
975 config_path: impl Into<PathBuf>,
976 configured_port: Option<u16>,
977 ) -> Self {
978 self.rescan = Some(SupervisorRescanContext {
979 supervisor,
980 config_path: config_path.into(),
981 configured_port,
982 storage_config: self.storage_config.clone(),
983 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
984 admission_facts_targets: self.admission_facts_targets.clone(),
985 scope_authority_owners: self.scope_authority_owners.clone(),
986 });
987 self
988 }
989
990 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
991 self.connected_clients = connected_clients;
992 self
993 }
994
995 pub fn forwarding(&self) -> Arc<ForwardingTable> {
996 Arc::clone(&self.forwarding)
997 }
998
999 pub(crate) fn counters(&self) -> DaemonCounters {
1000 self.counters.clone()
1001 }
1002
1003 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1006 tokio::spawn(async move {
1007 loop {
1008 self.capability_evaluator
1009 .wait_for_change_or_deadline()
1010 .await;
1011 self.refresh_capability_requirements();
1012 }
1013 });
1014 }
1015
1016 fn runtime_capability_snapshot(
1017 &self,
1018 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1019 let runtime = self
1020 .supervisor
1021 .list()
1022 .into_iter()
1023 .map(|module| {
1024 let status = module.status().map_err(|err| {
1025 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1026 })?;
1027 Ok(RuntimeModule {
1028 module_id: status.module_id,
1029 state: status.state,
1030 enabled: status.enabled,
1031 })
1032 })
1033 .collect::<Result<Vec<_>, RouterError>>()?;
1034 let (_, registrations) = self.registry.list_modules().map_err(|err| {
1035 RouterError::backend(
1036 0,
1037 0,
1038 format!("failed to list capability registrations: {err}"),
1039 )
1040 })?;
1041 let registrations = registrations
1042 .into_iter()
1043 .map(|registration| RegisteredModule {
1044 module_id: registration.manifest.module_id,
1045 module_version: registration.manifest.module_version,
1046 capabilities: registration.manifest.capabilities,
1047 })
1048 .collect();
1049 Ok((runtime, registrations))
1050 }
1051
1052 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1059 let cached_registration = RegisteredModule {
1060 module_id: registration.manifest.module_id.clone(),
1061 module_version: registration.manifest.module_version.clone(),
1062 capabilities: registration.manifest.capabilities.clone(),
1063 };
1064 if self.capability_evaluator.record_hello(&cached_registration) {
1065 warn!(
1066 module_id = %cached_registration.module_id,
1067 "capability claims drifted from the cached manifest"
1068 );
1069 }
1070 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1071 self.enforce_capability_denies();
1072 }
1073 self.refresh_capability_requirements();
1074 }
1075
1076 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1080 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1081 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1082 self.supervisor.set_swap_promotion_observer(observer);
1083 }
1084
1085 pub fn refresh_capability_requirements(&self) {
1086 match self.runtime_capability_snapshot() {
1087 Ok((runtime, registrations)) => {
1088 log_requirement_events(
1089 self.capability_evaluator
1090 .evaluate_now(&runtime, ®istrations),
1091 );
1092 }
1093 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1094 }
1095 }
1096
1097 fn enforce_capability_denies(&self) {
1101 let (_, registrations) = match self.registry.list_modules() {
1102 Ok(snapshot) => snapshot,
1103 Err(err) => {
1104 warn!(error = %err, "failed to read registrations for capability deny census");
1105 return;
1106 }
1107 };
1108 let manifests = registrations
1109 .into_iter()
1110 .map(|registration| {
1111 (
1112 registration.manifest.module_id.clone(),
1113 registration.manifest,
1114 )
1115 })
1116 .collect::<BTreeMap<_, _>>();
1117 let census = match self.forwarding.route_census(None) {
1118 Ok(census) => census,
1119 Err(err) => {
1120 warn!(error = %err, "failed to read route census for capability deny enforcement");
1121 return;
1122 }
1123 };
1124
1125 for (target_module_id, routes) in census {
1126 let Some(target_manifest) = manifests.get(&target_module_id) else {
1127 continue;
1128 };
1129 let mut closed_routes = Vec::new();
1130 let mut module_goodbyes = Vec::new();
1131 for route in routes {
1132 let Principal::Reserved {
1133 module_id: opening_module_id,
1134 } = &route.principal
1135 else {
1136 continue;
1137 };
1138 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1139 continue;
1140 };
1141 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1142 continue;
1143 };
1144
1145 match self.forwarding.release_client_route(
1146 route.goodbye_target.connection_id,
1147 route.goodbye_target.channel,
1148 route.goodbye_target.epoch,
1149 ) {
1150 Ok(RouteRelease::Removed(module_goodbye)) => {
1151 warn!(
1152 opening_module_id,
1153 target_module_id,
1154 capability,
1155 "force-closing route because an attested capability deny edge now matches"
1156 );
1157 closed_routes.push(route);
1158 module_goodbyes.push(module_goodbye);
1159 }
1160 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1161 Err(err) => warn!(
1162 opening_module_id,
1163 target_module_id,
1164 capability,
1165 error = %err,
1166 "failed to force-close capability-denied route"
1167 ),
1168 }
1169 }
1170
1171 if closed_routes.is_empty() {
1172 continue;
1173 }
1174 send_route_control_pushes(
1175 &self.forwarding,
1176 closed_routes,
1177 ClientControlPush::RouteClosed {
1178 module_id: target_module_id,
1179 channels: Vec::new(),
1180 reason: RouteCloseReason::CapabilityDenied,
1181 drained: false,
1182 abandoned: 0,
1183 excluded_subscriptions: 0,
1184 terminal: Some(false),
1185 },
1186 );
1187 self.emit_route_goodbyes(module_goodbyes);
1188 }
1189 }
1190
1191 fn not_ready_reason(
1197 &self,
1198 registration: &crate::registry::ModuleRegistration,
1199 ) -> Option<NotReadyReason> {
1200 if !registration.ready {
1201 return Some(NotReadyReason {
1202 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1203 capability: None,
1204 });
1205 }
1206 self.first_unprovided_required_capability(registration)
1207 .map(|capability| NotReadyReason {
1208 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1209 capability: Some(capability),
1210 })
1211 }
1212
1213 fn first_unprovided_required_capability(
1231 &self,
1232 registration: &crate::registry::ModuleRegistration,
1233 ) -> Option<String> {
1234 let required = registration
1235 .manifest
1236 .capabilities
1237 .iter()
1238 .flat_map(|declarations| declarations.requires.iter())
1239 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1240 .map(|requirement| requirement.capability.as_str())
1241 .collect::<BTreeSet<_>>();
1242 if required.is_empty() {
1243 return None;
1244 }
1245 let module_id = registration.manifest.module_id.as_str();
1246 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1247 if required
1248 .iter()
1249 .any(|capability| verdict(capability).is_none())
1250 {
1251 self.refresh_capability_requirements();
1252 }
1253 required
1254 .into_iter()
1255 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1256 .map(str::to_string)
1257 }
1258
1259 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1260 self.capability_evaluator
1261 .statuses()
1262 .into_iter()
1263 .map(capability_requirement_status)
1264 .collect()
1265 }
1266
1267 fn deregister_connection(
1274 &self,
1275 connection_id: ConnectionId,
1276 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1277 self.registry.deregister_connection(connection_id)
1278 }
1279
1280 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1281 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1282 return None;
1283 }
1284 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1285 parse_client_control_request(&frame.body)
1286 else {
1287 return None;
1288 };
1289 Some(target_module_id(&target).to_string())
1290 }
1291
1292 pub(crate) fn route_open_capacity_refusal(
1293 &self,
1294 ctx: &RouteCtx,
1295 frame: &Frame,
1296 target_module_id: &str,
1297 in_flight: usize,
1298 limit: usize,
1299 ) -> Result<Frame, RouterError> {
1300 self.route_open_admission_refusal_frame(
1301 ctx,
1302 frame,
1303 target_module_id,
1304 "open_admission_full",
1305 (in_flight, limit),
1306 format!(
1307 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1308 ),
1309 )
1310 }
1311
1312 fn route_open_target_capacity_refusal(
1313 &self,
1314 ctx: &RouteCtx,
1315 frame: &Frame,
1316 target_module_id: &str,
1317 in_flight: usize,
1318 ) -> Result<Frame, RouterError> {
1319 self.route_open_admission_refusal_frame(
1320 ctx,
1321 frame,
1322 target_module_id,
1323 "target_binds_full",
1324 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1325 format!(
1326 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1327 ),
1328 )
1329 }
1330
1331 fn route_open_admission_refusal_frame(
1339 &self,
1340 ctx: &RouteCtx,
1341 frame: &Frame,
1342 target_module_id: &str,
1343 reason: &'static str,
1344 (in_flight, limit): (usize, usize),
1345 message: impl Into<String>,
1346 ) -> Result<Frame, RouterError> {
1347 let code = error_codes::TARGET_UNAVAILABLE;
1348 self.counters.increment_route_open_refused(code);
1349 info!(
1350 target: "control",
1351 code,
1352 reason,
1353 module_id = ?target_module_id,
1354 connection_id = ctx.connection_id.get(),
1355 in_flight,
1356 limit,
1357 "route.open refused"
1358 );
1359 control_error_frame(frame, code, message.into())
1360 }
1361
1362 #[cfg(test)]
1368 pub fn handle_control(
1369 &self,
1370 connection_id: ConnectionId,
1371 frame: Frame,
1372 ) -> Result<Vec<Frame>, RouterError> {
1373 match frame.header.ty {
1374 FrameType::Ping => Ok(vec![pong(&frame)?]),
1375 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1376 FrameType::Goodbye => self.handle_goodbye(connection_id),
1377 ty => Ok(vec![control_error_frame(
1378 &frame,
1379 "unsupported_control_frame",
1380 format!("unsupported channel-0 frame {ty:?}"),
1381 )?]),
1382 }
1383 }
1384
1385 pub async fn handle_control_frame(
1386 &self,
1387 ctx: &RouteCtx,
1388 frame: Frame,
1389 ) -> Result<Vec<Frame>, RouterError> {
1390 self.handle_control_frame_timed(ctx, frame, None).await
1391 }
1392
1393 pub(crate) async fn handle_control_frame_timed(
1394 &self,
1395 ctx: &RouteCtx,
1396 frame: Frame,
1397 dispatch_started_at: Option<StdInstant>,
1398 ) -> Result<Vec<Frame>, RouterError> {
1399 match frame.header.ty {
1400 FrameType::Ping => Ok(vec![pong(&frame)?]),
1401 FrameType::Hello => {
1402 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1403 }
1404 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1405 FrameType::Cancel => {
1406 if self
1407 .supervisor
1408 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1409 {
1410 Ok(Vec::new())
1411 } else {
1412 Ok(vec![control_error_frame(
1413 &frame,
1414 "unknown_subscription",
1415 "no supervisor spawn subscription has this correlation id",
1416 )?])
1417 }
1418 }
1419 FrameType::Request => {
1420 if self
1421 .forwarding
1422 .module_endpoint_for_connection(ctx.connection_id)
1423 .map_err(RouterError::Forwarding)?
1424 .is_some()
1425 {
1426 if !is_known_module_request_op(&frame.body) {
1427 return Ok(vec![control_error_frame(
1428 &frame,
1429 "unsupported_control_frame",
1430 "module-originated channel-0 REQUEST is not supported",
1431 )?]);
1432 }
1433 let request = match parse_module_control_request_from_module(&frame.body) {
1434 Ok(request) => request,
1435 Err((err, ControlRequestBodyError::UnknownOp)) => {
1436 return Ok(vec![control_error_frame(
1437 &frame,
1438 "unsupported_control_frame",
1439 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1440 )?])
1441 }
1442 Err((err, ControlRequestBodyError::InvalidBody)) => {
1443 return Ok(vec![control_error_frame(
1444 &frame,
1445 "invalid_control_body",
1446 format!("malformed module control body: {err}"),
1447 )?])
1448 }
1449 };
1450 let op = module_control_request_op(&request);
1451 let corr = frame.header.corr;
1452 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1453 let result =
1454 self.handle_module_control_request(ctx.connection_id, frame, request);
1455 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1456 return result;
1457 }
1458
1459 if is_known_module_request_op(&frame.body) {
1460 return Ok(vec![control_error_frame(
1461 &frame,
1462 "not_registered",
1463 "catalog.update requires an active module registration owned by this connection",
1464 )?]);
1465 }
1466
1467 let request = match parse_client_control_request(&frame.body) {
1468 Ok(request) => request,
1469 Err((err, ControlRequestBodyError::UnknownOp)) => {
1470 return Ok(vec![control_error_frame(
1471 &frame,
1472 "unknown_control_op",
1473 format!("unknown client control op: {err}"),
1474 )?])
1475 }
1476 Err((err, ControlRequestBodyError::InvalidBody)) => {
1477 return Ok(vec![control_error_frame(
1478 &frame,
1479 "invalid_control_body",
1480 format!("malformed client control body: {err}"),
1481 )?])
1482 }
1483 };
1484 let op = client_control_request_op(&request);
1485 let corr = frame.header.corr;
1486 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1487 #[cfg(test)]
1488 if let Some(delay) = self.control_dispatch_delay {
1489 tokio::time::sleep(delay).await;
1490 }
1491 let result = self
1492 .handle_client_control_request(ctx, frame, request)
1493 .await;
1494 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1495 result
1496 }
1497 FrameType::Push => {
1498 let Some(endpoint) = self
1499 .forwarding
1500 .module_endpoint_for_connection(ctx.connection_id)
1501 .map_err(RouterError::Forwarding)?
1502 else {
1503 return Ok(vec![control_error_frame(
1504 &frame,
1505 "unsupported_control_frame",
1506 "client-originated channel-0 PUSH is not supported",
1507 )?]);
1508 };
1509 self.handle_status_update(endpoint, frame)
1510 }
1511 FrameType::Response | FrameType::Error
1512 if self
1513 .forwarding
1514 .module_endpoint_for_connection(ctx.connection_id)
1515 .map_err(RouterError::Forwarding)?
1516 .is_some() =>
1517 {
1518 self.handle_module_relay_response(ctx.connection_id, frame)
1519 }
1520 ty => Ok(vec![control_error_frame(
1521 &frame,
1522 "unsupported_control_frame",
1523 format!("unsupported channel-0 frame {ty:?}"),
1524 )?]),
1525 }
1526 }
1527
1528 pub fn cleanup_connection(
1529 &self,
1530 connection_id: ConnectionId,
1531 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1532 let crash_closed = self
1533 .registry
1534 .get_module_by_connection(connection_id)?
1535 .and_then(|registration| {
1536 self.forwarding
1537 .module_endpoint_for_connection(connection_id)
1538 .ok()
1539 .flatten()
1540 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1541 .map(|routes| (registration.manifest.module_id, routes))
1542 });
1543 let crash_closed = crash_closed.map(|(module_id, routes)| {
1544 let terminal = match self.supervisor.get(&module_id) {
1545 None => false,
1546 Some(module) => match module.will_recover_after_connection_loss() {
1547 Ok(will_recover) => !will_recover,
1548 Err(err) => {
1549 warn!(
1550 %module_id,
1551 error = %err,
1552 "failed to read crash recovery verdict; reporting non-terminal conservatively"
1553 );
1554 false
1555 }
1556 },
1557 };
1558 let reason = match self.forwarding.is_daemon_draining() {
1562 Ok(true) => RouteCloseReason::Restart,
1563 Ok(false) => RouteCloseReason::Crash,
1564 Err(err) => {
1565 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
1566 RouteCloseReason::Crash
1567 }
1568 };
1569 (module_id, routes, reason, terminal)
1570 });
1571 let registrations = self.deregister_connection(connection_id);
1572 let cleanup = if crash_closed.is_some() {
1573 self.forwarding.cleanup_connection_counted(connection_id)
1574 } else {
1575 self.forwarding
1581 .cleanup_connection(connection_id)
1582 .map(|released| crate::forwarding::ConnectionCleanup {
1583 released,
1584 abandoned_relays: 0,
1585 })
1586 };
1587 if let Some((module_id, routes, reason, terminal)) = crash_closed {
1592 let abandoned = cleanup
1593 .as_ref()
1594 .map_or(0, |cleanup| cleanup.abandoned_relays);
1595 send_route_control_pushes(
1596 &self.forwarding,
1597 routes,
1598 ClientControlPush::RouteClosed {
1599 module_id,
1600 channels: Vec::new(),
1601 reason,
1602 drained: false,
1603 abandoned,
1604 excluded_subscriptions: 0,
1605 terminal: Some(terminal),
1606 },
1607 );
1608 }
1609 if let Ok(cleanup) = cleanup {
1610 self.emit_route_goodbyes(cleanup.released);
1611 }
1612 if matches!(®istrations, Ok(r) if !r.is_empty()) {
1616 crate::supervise::notify_registration_release();
1617 self.capability_evaluator.wake_deadline_loop();
1618 self.refresh_capability_requirements();
1619 }
1620 self.supervisor.remove_spawn_subscribers(connection_id);
1621 self.hello_launch_nonces
1624 .lock()
1625 .unwrap_or_else(|poisoned| poisoned.into_inner())
1626 .forget(connection_id);
1627 self.scopes
1628 .write()
1629 .unwrap_or_else(|poisoned| poisoned.into_inner())
1630 .release_connection(connection_id);
1631 registrations
1632 }
1633
1634 pub(crate) fn handle_route_goodbye(
1635 &self,
1636 connection_id: ConnectionId,
1637 route_channel: u16,
1638 route_epoch: u32,
1639 ) -> Result<bool, RouterError> {
1640 debug!(
1641 connection_id = connection_id.get(),
1642 route_channel, route_epoch, "handling route GOODBYE"
1643 );
1644 let RouteRelease::Removed(released_route) = self
1645 .forwarding
1646 .release_client_route(connection_id, route_channel, route_epoch)
1647 .map_err(RouterError::Forwarding)?
1648 else {
1649 return Ok(false);
1650 };
1651 self.emit_route_goodbyes(vec![released_route]);
1652 Ok(true)
1653 }
1654
1655 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
1656 for released in released_routes {
1657 let frame = match Frame::build_with_version(
1658 released.negotiated_ver,
1659 FrameType::Goodbye,
1660 control_flags(),
1661 released.channel,
1662 released.epoch,
1663 0,
1664 Vec::new(),
1665 ) {
1666 Ok(frame) => frame,
1667 Err(err) => {
1668 warn!(
1669 route_channel = released.channel,
1670 error = %err,
1671 "failed to build route GOODBYE frame"
1672 );
1673 continue;
1674 }
1675 };
1676 if !released.close_on_delivery_failure() {
1677 crate::forwarding::send_module_route_goodbye(
1678 &self.counters,
1679 &released.sink,
1680 frame,
1681 released.module_id.as_deref(),
1682 "client route released",
1683 );
1684 continue;
1685 }
1686 if let Err(err) = released.sink.try_send(frame) {
1687 warn!(
1688 target_connection_id = released.connection_id.get(),
1689 route_channel = released.channel,
1690 error = %err,
1691 "route GOODBYE was not delivered to client; closing target connection"
1692 );
1693 if self
1694 .forwarding
1695 .escalate_client_delivery_failure(
1696 released.connection_id,
1697 released.channel,
1698 released.epoch,
1699 CloseReason::new(
1700 "route_goodbye_delivery_failed",
1701 format!(
1702 "failed to enqueue route GOODBYE for channel {}: {err}",
1703 released.channel
1704 ),
1705 ),
1706 crate::forwarding::UndeliveredFrame {
1707 module_id: released.module_id.as_deref(),
1708 sink: &released.sink,
1709 },
1710 )
1711 .unwrap_or(false)
1712 {
1713 self.counters.increment_goodbye_relay_client_failed();
1714 }
1715 }
1716 }
1717 }
1718
1719 fn send_abandoned_route_bind_goodbye(
1731 &self,
1732 module_sink: &crate::FrameSink,
1733 negotiated_ver: u8,
1734 module_channel: u16,
1735 module_epoch: u32,
1736 ) {
1737 let frame = match Frame::build_with_version(
1738 negotiated_ver,
1739 FrameType::Goodbye,
1740 control_flags(),
1741 module_channel,
1742 module_epoch,
1743 0,
1744 Vec::new(),
1745 ) {
1746 Ok(frame) => frame,
1747 Err(err) => {
1748 warn!(
1749 route_channel = module_channel,
1750 error = %err,
1751 "failed to build GOODBYE for abandoned route.bind"
1752 );
1753 return;
1754 }
1755 };
1756 crate::forwarding::send_module_route_goodbye(
1757 &self.counters,
1758 module_sink,
1759 frame,
1760 None,
1761 "abandoned route.bind",
1762 );
1763 }
1764
1765 fn handle_hello(
1766 &self,
1767 connection_id: ConnectionId,
1768 sink: Option<crate::FrameSink>,
1769 frame: Frame,
1770 ) -> Result<Vec<Frame>, RouterError> {
1771 debug!(
1772 connection_id = connection_id.get(),
1773 corr = frame.header.corr,
1774 "handling HELLO"
1775 );
1776 if self
1780 .registry
1781 .get_module_by_connection(connection_id)
1782 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
1783 .is_some()
1784 {
1785 return Ok(vec![control_error_frame(
1786 &frame,
1787 "invalid_hello",
1788 "connection is already registered as a module",
1789 )?]);
1790 }
1791 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
1792 Ok(value) => value,
1793 Err(err) => {
1794 return Ok(vec![control_error_frame(
1795 &frame,
1796 "invalid_hello",
1797 format!("malformed HELLO body: {err}"),
1798 )?])
1799 }
1800 };
1801 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
1802 return Ok(vec![control_error_frame(
1803 &frame,
1804 "invalid_capability_grammar",
1805 err.to_string(),
1806 )?]);
1807 }
1808 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
1809 return Ok(vec![control_error_frame(
1810 &frame,
1811 "invalid_manifest",
1812 err.to_string(),
1813 )?]);
1814 }
1815 if let Some(provenance) = hello_value
1816 .get("manifest")
1817 .and_then(|manifest| manifest.get("provenance"))
1818 {
1819 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
1820 return Ok(vec![control_error_frame(
1821 &frame,
1822 "invalid_manifest",
1823 format!("malformed manifest provenance: {err}"),
1824 )?]);
1825 }
1826 }
1827 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
1828 Ok(hello) => hello,
1829 Err(err) => {
1830 return Ok(vec![control_error_frame(
1831 &frame,
1832 "invalid_hello",
1833 format!("malformed HELLO body: {err}"),
1834 )?])
1835 }
1836 };
1837
1838 if hello.protocol_ver != hello.manifest.protocol_ver {
1839 return Ok(vec![control_error_frame(
1840 &frame,
1841 "invalid_manifest",
1842 format!(
1843 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
1844 hello.protocol_ver, hello.manifest.protocol_ver
1845 ),
1846 )?]);
1847 }
1848
1849 if hello.manifest.module_id.trim().is_empty() {
1850 return Ok(vec![control_error_frame(
1851 &frame,
1852 "invalid_manifest",
1853 "manifest module_id must not be empty",
1854 )?]);
1855 }
1856
1857 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
1858 Ok(negotiated_ver) => negotiated_ver,
1859 Err(message) => {
1860 return Ok(vec![control_error_frame(
1861 &frame,
1862 "version_unsupported",
1863 message,
1864 )?])
1865 }
1866 };
1867
1868 let swap_admission = self
1876 .supervisor
1877 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
1878 if swap_admission == SwapHelloAdmission::Refused {
1879 warn!(
1880 module_id = %hello.manifest.module_id,
1881 connection_id = connection_id.get(),
1882 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
1883 );
1884 return Ok(vec![control_error_frame(
1885 &frame,
1886 "swap_token_invalid",
1887 format!(
1888 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
1889 hello.manifest.module_id
1890 ),
1891 )?]);
1892 }
1893 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
1894
1895 if let Some(rejection) = (!swap_candidate)
1903 .then(|| {
1904 self.supervisor.reserved_hello_rejection(
1905 &hello.manifest.module_id,
1906 hello.launch_nonce.as_deref(),
1907 )
1908 })
1909 .flatten()
1910 {
1911 let message = match rejection {
1912 ReservedHelloRejection::Exact { module_id } => format!(
1913 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
1914 ),
1915 ReservedHelloRejection::Prefix {
1916 prefix,
1917 owner_module_id,
1918 } => format!(
1919 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
1920 hello.manifest.module_id
1921 ),
1922 };
1923 return Ok(vec![control_error_frame(
1924 &frame,
1925 "reserved_module",
1926 message,
1927 )?]);
1928 }
1929
1930 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
1931 &hello.manifest.module_id,
1932 hello.manifest.capabilities.as_ref(),
1933 );
1934 if let Some(refusal) = reserved_capability_refusals.first() {
1935 let capability = refusal.capability.clone();
1936 let bound_module = refusal.claimants[0].clone();
1937 log_duplicate_claim_events(reserved_capability_refusals);
1938 return Ok(vec![control_error_frame(
1939 &frame,
1940 "reserved_capability",
1941 format!(
1942 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
1943 capability, bound_module, hello.manifest.module_id
1944 ),
1945 )?]);
1946 }
1947
1948 if self
1951 .forwarding
1952 .connection_has_client_routes(connection_id)
1953 .map_err(RouterError::Forwarding)?
1954 {
1955 return Ok(vec![control_error_frame(
1956 &frame,
1957 "invalid_hello",
1958 "connection has open client routes and cannot also register as a module",
1959 )?]);
1960 }
1961
1962 self.hello_launch_nonces
1967 .lock()
1968 .unwrap_or_else(|poisoned| poisoned.into_inner())
1969 .record(connection_id, hello.launch_nonce.as_deref());
1970 let control_ops = effective_module_control_ops(hello.control_ops);
1971 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
1974 if swap_candidate {
1975 return self.register_swap_candidate(
1976 connection_id,
1977 sink,
1978 &frame,
1979 hello.manifest,
1980 negotiated_ver,
1981 control_ops,
1982 hello_ack,
1983 );
1984 }
1985 let registration = match self.registry.register_with_control_ops(
1986 hello.manifest,
1987 negotiated_ver,
1988 connection_id,
1989 control_ops,
1990 ) {
1991 Ok(registration) => registration,
1992 Err(RegistryError::DuplicateModuleId { module_id }) => {
1993 return Ok(vec![control_error_frame(
1994 &frame,
1995 "duplicate_module_id",
1996 format!(
1997 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
1998 ),
1999 )?])
2000 }
2001 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2002 return Ok(vec![control_error_frame(
2003 &frame,
2004 "invalid_module_id",
2005 err.to_string(),
2006 )?])
2007 }
2008 Err(err) => {
2009 return Ok(vec![control_error_frame(
2010 &frame,
2011 "registry_error",
2012 err.to_string(),
2013 )?])
2014 }
2015 };
2016
2017 let reply = if let Some(sink) = sink {
2018 let concurrency = manifest_concurrency(®istration.manifest);
2034 if let Err(err) = self.forwarding.register_module_connection_acked(
2035 connection_id,
2036 registration.manifest.module_id.clone(),
2037 negotiated_ver,
2038 concurrency,
2039 sink,
2040 hello_ack,
2041 ) {
2042 if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
2046 crate::supervise::notify_registration_release();
2047 }
2048 return Ok(vec![control_error_frame(
2049 &frame,
2050 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2051 "invalid_hello"
2052 } else {
2053 forwarding_error_code(&err)
2054 },
2055 err.to_string(),
2056 )?]);
2057 }
2058 Vec::new()
2059 } else {
2060 vec![hello_ack]
2063 };
2064
2065 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2074 info!(
2075 module_id = %registration.manifest.module_id,
2076 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2077 );
2078 }
2079
2080 self.apply_registration_capabilities(®istration);
2081
2082 info!(
2083 module_id = %registration.manifest.module_id,
2084 module_version = %registration.manifest.module_version,
2085 negotiated_ver,
2086 routable_provider = manifest_provides_routable_role(®istration.manifest),
2087 connection_id = connection_id.get(),
2088 "module registered"
2089 );
2090
2091 Ok(reply)
2092 }
2093
2094 #[allow(clippy::too_many_arguments)]
2104 fn register_swap_candidate(
2105 &self,
2106 connection_id: ConnectionId,
2107 sink: Option<crate::FrameSink>,
2108 frame: &Frame,
2109 manifest: ModuleManifest,
2110 negotiated_ver: u8,
2111 control_ops: Vec<String>,
2112 hello_ack: Frame,
2113 ) -> Result<Vec<Frame>, RouterError> {
2114 let module_id = manifest.module_id.clone();
2115 let registration = match self.registry.register_candidate_with_control_ops(
2116 manifest,
2117 negotiated_ver,
2118 connection_id,
2119 control_ops,
2120 ) {
2121 Ok(registration) => registration,
2122 Err(RegistryError::DuplicateModuleId { module_id }) => {
2123 return Ok(vec![control_error_frame(
2124 frame,
2125 "duplicate_module_id",
2126 format!(
2127 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2128 ),
2129 )?])
2130 }
2131 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2132 return Ok(vec![control_error_frame(
2133 frame,
2134 "invalid_module_id",
2135 err.to_string(),
2136 )?])
2137 }
2138 Err(err) => {
2139 return Ok(vec![control_error_frame(
2140 frame,
2141 "registry_error",
2142 err.to_string(),
2143 )?])
2144 }
2145 };
2146 let reply = if let Some(sink) = sink {
2147 let concurrency = manifest_concurrency(®istration.manifest);
2151 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2152 connection_id,
2153 module_id.clone(),
2154 negotiated_ver,
2155 concurrency,
2156 sink,
2157 hello_ack,
2158 ) {
2159 if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
2160 crate::supervise::notify_registration_release();
2161 }
2162 return Ok(vec![control_error_frame(
2163 frame,
2164 forwarding_error_code(&err),
2165 err.to_string(),
2166 )?]);
2167 }
2168 Vec::new()
2169 } else {
2170 vec![hello_ack]
2171 };
2172 self.supervisor.mark_swap_candidate_admitted(&module_id);
2173 info!(
2174 module_id = %module_id,
2175 module_version = %registration.manifest.module_version,
2176 negotiated_ver,
2177 ready = registration.ready,
2178 connection_id = connection_id.get(),
2179 "swap candidate registered; not routable until cutover"
2180 );
2181 Ok(reply)
2182 }
2183
2184 fn build_hello_ack(
2185 &self,
2186 frame: &Frame,
2187 negotiated_ver: u8,
2188 module_id: &str,
2189 ) -> Result<Frame, RouterError> {
2190 let ack = ModuleHelloAckBody {
2191 negotiated_ver,
2192 subc_ops: module_subc_ops(),
2193 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2194 storage: self
2195 .storage_config
2196 .as_ref()
2197 .map(|cfg| cfg.descriptor_for(module_id)),
2198 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2199 };
2200 let body = serde_json::to_vec(&ack).map_err(|err| {
2201 RouterError::backend(
2202 0,
2203 frame.header.corr,
2204 format!("failed to encode HELLO_ACK: {err}"),
2205 )
2206 })?;
2207
2208 Frame::build_with_version(
2209 negotiated_ver,
2210 FrameType::HelloAck,
2211 control_flags(),
2212 0,
2213 0,
2214 frame.header.corr,
2215 body,
2216 )
2217 .map_err(RouterError::FrameBuild)
2218 }
2219
2220 async fn handle_client_control_request(
2221 &self,
2222 ctx: &RouteCtx,
2223 frame: Frame,
2224 request: ClientControlRequest,
2225 ) -> Result<Vec<Frame>, RouterError> {
2226 match request {
2227 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2228 ClientControlRequest::CatalogList { module_id } => {
2229 self.handle_catalog_list(frame, module_id)
2230 }
2231 ClientControlRequest::RouteOpen {
2232 target,
2233 identity,
2234 consumer_identity,
2235 consumer_capabilities,
2236 role_versions,
2237 admission_facts,
2238 scope,
2239 } => {
2240 self.handle_route_open(
2241 ctx,
2242 frame,
2243 RouteOpenRequest {
2244 target,
2245 identity,
2246 consumer_identity,
2247 consumer_capabilities,
2248 role_versions,
2249 admission_facts,
2250 scope,
2251 },
2252 )
2253 .await
2254 }
2255 ClientControlRequest::RoutePoll {
2256 route_channel,
2257 route_epoch,
2258 kind,
2259 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2260 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2261 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2262 self.handle_supervisor_spawn_snapshot(frame)
2263 }
2264 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2265 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2266 }
2267 ClientControlRequest::SupervisorRestart {
2268 module_id,
2269 drain_timeout_ms,
2270 } => {
2271 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms)
2272 .await
2273 }
2274 ClientControlRequest::SupervisorSwap {
2275 module_id,
2276 ready_timeout_ms,
2277 } => {
2278 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms)
2279 .await
2280 }
2281 ClientControlRequest::SupervisorReload { module_id } => {
2282 self.handle_supervisor_reload(frame, module_id).await
2283 }
2284 ClientControlRequest::SupervisorRescan { preview } => {
2285 self.handle_supervisor_rescan(frame, preview).await
2286 }
2287 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2288 self.handle_supervisor_release_reserved(frame, module_id)
2289 .await
2290 }
2291 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2292 self.handle_supervisor_set_enabled(frame, module_id, enabled)
2293 .await
2294 }
2295 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2296 self.handle_supervisor_health_probe(frame, module_id).await
2297 }
2298 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2299 ClientControlRequest::SupervisorRoutes { module_id } => {
2300 self.handle_supervisor_routes(frame, module_id)
2301 }
2302 ClientControlRequest::SupervisorProvenance { module_id } => {
2303 self.handle_supervisor_provenance(frame, module_id).await
2304 }
2305 ClientControlRequest::SupervisorStderrTail {
2306 module_id,
2307 max_lines,
2308 max_bytes,
2309 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2310 ClientControlRequest::SupervisorTerminals { module_id } => {
2311 self.handle_supervisor_terminals(frame, module_id).await
2312 }
2313 }
2314 }
2315
2316 fn handle_module_control_request(
2317 &self,
2318 connection_id: ConnectionId,
2319 frame: Frame,
2320 request: ModuleControlRequestFromModule,
2321 ) -> Result<Vec<Frame>, RouterError> {
2322 match request {
2323 ModuleControlRequestFromModule::CatalogUpdate {
2324 provides,
2325 capabilities,
2326 ready,
2327 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2328 ModuleControlRequestFromModule::LiveRoots {} => {
2329 let registered = self
2330 .registry
2331 .get_module_by_connection(connection_id)
2332 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2333 let Some(registration) = registered else {
2334 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2335 };
2336 let response = self
2337 .forwarding
2338 .live_roots(®istration.manifest.module_id)
2339 .map_err(RouterError::Forwarding)?;
2340 Ok(vec![control_response_body_frame(
2341 &frame,
2342 &response,
2343 "ModuleControlResponseToModule::LiveRoots",
2344 )?])
2345 }
2346 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
2347 self.handle_scope_sync(connection_id, frame, generation, scopes)
2348 }
2349 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
2350 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
2351 }
2352 }
2353 }
2354
2355 fn handle_scope_sync(
2359 &self,
2360 connection_id: ConnectionId,
2361 frame: Frame,
2362 generation: u64,
2363 scopes: Vec<ScopeRecord>,
2364 ) -> Result<Vec<Frame>, RouterError> {
2365 let Some(registration) = self
2366 .registry
2367 .get_module_by_connection(connection_id)
2368 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2369 else {
2370 return Ok(vec![control_error_frame(
2371 &frame,
2372 "not_registered",
2373 "scope.sync requires an active module registration owned by this connection",
2374 )?]);
2375 };
2376 let owner = registration.manifest.module_id;
2377 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
2378 let is_current_launch = |connection: ConnectionId| {
2379 self.hello_launch_nonces
2380 .lock()
2381 .unwrap_or_else(|poisoned| poisoned.into_inner())
2382 .presented(connection, current_nonce.as_deref())
2383 };
2384 let mut table = self
2389 .scopes
2390 .write()
2391 .unwrap_or_else(|poisoned| poisoned.into_inner());
2392 let outcome = table.sync(&owner, connection_id, is_current_launch, generation, scopes);
2393 let drained = match &outcome {
2394 Ok(applied) => self
2395 .forwarding
2396 .publish_scope_changes(&applied.tag_changes)
2397 .map_err(RouterError::Forwarding)?,
2398 Err(_) => Vec::new(),
2399 };
2400 drop(table);
2401 match outcome {
2402 Ok(applied) => {
2403 let counts = ScopeOutcomeCounts::of(&applied.results);
2404 info!(
2405 owner = %owner,
2406 generation,
2407 records = applied.results.len(),
2408 created = counts.created,
2409 replaced = counts.replaced,
2410 updated = counts.updated,
2411 unchanged = counts.unchanged,
2412 refused = counts.refused,
2413 ended = applied.ended.len(),
2414 tag_changes = applied.tag_changes.len(),
2415 routes_closed = drained.len(),
2416 "scope sync accepted"
2417 );
2418 for refused in applied
2424 .results
2425 .iter()
2426 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
2427 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
2428 {
2429 warn!(
2430 owner = %owner,
2431 generation,
2432 scope_ref = %refused.scope_ref,
2433 scope_epoch = refused.scope_epoch,
2434 code = refused.code.as_deref().unwrap_or(""),
2435 "scope record refused"
2436 );
2437 }
2438 self.close_scope_drained_routes(drained);
2439 let response = ModuleControlResponseToModule::ScopeSync {
2440 generation,
2441 results: applied.results,
2442 ended: applied.ended,
2443 };
2444 Ok(vec![control_response_body_frame(
2445 &frame,
2446 &response,
2447 "ModuleControlResponseToModule::ScopeSync",
2448 )?])
2449 }
2450 Err(refusal) => {
2451 info!(
2452 owner = %owner,
2453 generation,
2454 code = refusal.code,
2455 "scope sync refused"
2456 );
2457 Ok(vec![control_error_frame(
2458 &frame,
2459 refusal.code,
2460 refusal.message,
2461 )?])
2462 }
2463 }
2464 }
2465
2466 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
2472 if drained.is_empty() {
2473 return;
2474 }
2475 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
2476 BTreeMap::new();
2477 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
2478 for route in drained {
2479 warn!(
2480 module_id = %route.module_id,
2481 reason = ?route.reason,
2482 client_connection_id = route.client.connection_id.get(),
2483 route_channel = route.client.channel,
2484 "closing route because its scope changed"
2485 );
2486 pushes
2487 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
2488 .or_insert_with(|| (route.reason, Vec::new()))
2489 .1
2490 .push(EndpointRoute {
2491 goodbye_target: route.client.clone(),
2492 principal: Principal::Unverified,
2493 bound_at: Instant::now(),
2494 draining: false,
2495 drain_reason: None,
2496 });
2497 goodbyes.push(route.module);
2498 goodbyes.push(route.client);
2499 }
2500 for ((module_id, _), (reason, routes)) in pushes {
2501 send_route_control_pushes(
2502 &self.forwarding,
2503 routes,
2504 ClientControlPush::RouteClosed {
2505 module_id,
2506 channels: Vec::new(),
2507 reason,
2508 drained: false,
2509 abandoned: 0,
2510 excluded_subscriptions: 0,
2511 terminal: Some(false),
2512 },
2513 );
2514 }
2515 self.emit_route_goodbyes(goodbyes);
2516 }
2517
2518 fn handle_scope_describe(
2521 &self,
2522 connection_id: ConnectionId,
2523 frame: Frame,
2524 owner: Principal,
2525 scope_ref: String,
2526 ) -> Result<Vec<Frame>, RouterError> {
2527 let registered = self
2528 .registry
2529 .get_module_by_connection(connection_id)
2530 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2531 if registered.is_none() {
2532 return Ok(vec![control_error_frame(
2533 &frame,
2534 "not_registered",
2535 "scope.describe requires an active module registration owned by this connection",
2536 )?]);
2537 }
2538 let description = self
2539 .scopes
2540 .read()
2541 .unwrap_or_else(|poisoned| poisoned.into_inner())
2542 .describe(&owner, &scope_ref);
2543 let owner_configured = match &owner {
2544 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
2549 _ => false,
2550 };
2551 let response = ModuleControlResponseToModule::ScopeDescribe {
2552 status: description.status,
2553 scope_epoch: description.scope_epoch,
2554 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
2555 owner_synced: description.owner_synced,
2556 owner_configured,
2557 scope: description.stamp,
2558 };
2559 Ok(vec![control_response_body_frame(
2560 &frame,
2561 &response,
2562 "ModuleControlResponseToModule::ScopeDescribe",
2563 )?])
2564 }
2565
2566 fn handle_catalog_update(
2567 &self,
2568 connection_id: ConnectionId,
2569 frame: Frame,
2570 provides: Vec<ProviderRole>,
2571 capabilities: Option<CapabilityDeclarations>,
2572 ready: Option<bool>,
2573 ) -> Result<Vec<Frame>, RouterError> {
2574 self.refresh_capability_requirements();
2575 let Some(registration) = self
2576 .registry
2577 .get_module_by_connection(connection_id)
2578 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2579 else {
2580 return Ok(vec![control_error_frame(
2581 &frame,
2582 "not_registered",
2583 "catalog.update requires an active module registration owned by this connection",
2584 )?]);
2585 };
2586
2587 if let Some(message) =
2588 catalog_update_frozen_field_message(®istration.manifest, &provides)
2589 {
2590 return Ok(vec![control_error_frame(
2591 &frame,
2592 "catalog_update_frozen_field",
2593 message,
2594 )?]);
2595 }
2596
2597 let mut candidate = registration.manifest.clone();
2598 candidate.provides = provides.clone();
2599 candidate.capabilities = capabilities
2600 .clone()
2601 .or_else(|| registration.manifest.capabilities.clone());
2602 if let Err(err) = candidate.validate_capability_grammar() {
2603 return Ok(vec![control_error_frame(
2604 &frame,
2605 "invalid_capability_grammar",
2606 err.to_string(),
2607 )?]);
2608 }
2609
2610 let mut conflicts = self
2613 .capability_evaluator
2614 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
2615 if let Some(conflict) = conflicts.first() {
2616 let message = format!(
2617 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2618 conflict.capability, conflict.claimants[0], candidate.module_id
2619 );
2620 for conflict in &mut conflicts {
2621 conflict.source = DuplicateClaimSource::CatalogUpdate;
2622 }
2623 log_duplicate_claim_events(conflicts);
2624 return Ok(vec![control_error_frame(
2625 &frame,
2626 "reserved_capability",
2627 message,
2628 )?]);
2629 }
2630
2631 let updated = self
2632 .registry
2633 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
2634 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2635 if updated.is_none() {
2636 return Ok(vec![control_error_frame(
2637 &frame,
2638 "not_registered",
2639 "catalog.update requires an active module registration owned by this connection",
2640 )?]);
2641 }
2642 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
2643 log_duplicate_claim_events(
2644 self.capability_evaluator
2645 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
2646 );
2647 }
2648 if capability_census_trigger(
2649 registration.manifest.capabilities.as_ref(),
2650 updated
2651 .as_ref()
2652 .and_then(|entry| entry.manifest.capabilities.as_ref()),
2653 ) {
2654 self.enforce_capability_denies();
2655 }
2656 self.refresh_capability_requirements();
2657
2658 let response = ModuleControlResponseToModule::CatalogUpdate {};
2659 control_response_body_frame(
2660 &frame,
2661 &response,
2662 "ModuleControlResponseToModule::CatalogUpdate",
2663 )
2664 .map(|frame| vec![frame])
2665 }
2666
2667 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
2668 self.refresh_capability_requirements();
2669 let mut counters = self.counters.snapshot();
2675 if let (Ok((connections_with_routes, max)), Some(obj)) = (
2676 self.forwarding.client_route_concentration(),
2677 counters.as_object_mut(),
2678 ) {
2679 obj.insert(
2680 "client_connections_with_routes".into(),
2681 connections_with_routes.into(),
2682 );
2683 obj.insert("max_routes_on_one_connection".into(), max.into());
2684 }
2685 if let (Some(open_breakers), Some(obj)) = (
2691 self.route_bind_breakers.open_snapshot(),
2692 counters.as_object_mut(),
2693 ) {
2694 obj.insert("route_bind_breakers_open".into(), open_breakers);
2695 }
2696 let response = ClientControlResponse::ServerDescribe {
2697 protocol_ver: PROTOCOL_VERSION,
2698 subc_ops: subc_ops(),
2699 capabilities: self.subc_capabilities.as_ref().to_vec(),
2700 connected_clients: self.connected_clients.count(),
2701 counters: Some(counters),
2702 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
2703 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
2704 capability_requirements: self.capability_requirement_statuses(),
2705 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2706 };
2707 Ok(vec![control_response_body_frame(
2708 &frame,
2709 &response,
2710 "ClientControlResponse::ServerDescribe",
2711 )?])
2712 }
2713
2714 fn handle_catalog_list(
2715 &self,
2716 frame: Frame,
2717 module_id: Option<String>,
2718 ) -> Result<Vec<Frame>, RouterError> {
2719 let (generation, modules) = self.registry.list_modules().map_err(|err| {
2720 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
2721 })?;
2722 let entries = modules
2723 .into_iter()
2724 .filter(|registration| {
2725 module_id
2726 .as_deref()
2727 .map(|wanted| registration.manifest.module_id == wanted)
2728 .unwrap_or(true)
2729 })
2730 .map(|registration| {
2731 let not_ready = self.not_ready_reason(®istration);
2732 let roles = registration.manifest.provides;
2733 CatalogEntry {
2734 module_id: registration.manifest.module_id,
2735 ready: not_ready.is_none(),
2736 not_ready,
2737 module_version: Some(registration.manifest.module_version),
2738 roles,
2739 control_ops: registration.control_ops,
2740 capabilities: registration.manifest.capabilities,
2741 self_signals: registration.manifest.self_signals,
2742 }
2743 })
2744 .collect();
2745 let response = ClientControlResponse::CatalogList {
2746 generation,
2747 modules: entries,
2748 subc_ops: subc_ops(),
2749 };
2750 Ok(vec![control_response_body_frame(
2751 &frame,
2752 &response,
2753 "ClientControlResponse::CatalogList",
2754 )?])
2755 }
2756
2757 fn route_open_principal(
2758 &self,
2759 frame: &Frame,
2760 consumer_identity: Option<ConsumerIdentity>,
2761 ) -> Result<Result<Principal, Frame>, RouterError> {
2762 let Some(consumer_identity) = consumer_identity else {
2763 return Ok(Ok(Principal::Direct));
2764 };
2765
2766 if self.supervisor.spawned_consumer_authorized(
2767 &consumer_identity.module_id,
2768 &consumer_identity.launch_nonce,
2769 ) {
2770 return Ok(Ok(Principal::Reserved {
2771 module_id: consumer_identity.module_id,
2772 }));
2773 }
2774
2775 Ok(Err(control_error_frame(
2776 frame,
2777 "bad_consumer_identity",
2778 format!(
2779 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
2780 consumer_identity.module_id
2781 ),
2782 )?))
2783 }
2784
2785 fn route_open_refusal_frame(
2792 &self,
2793 ctx: &RouteCtx,
2794 frame: &Frame,
2795 module_id: &str,
2796 reason: &'static str,
2797 code: &'static str,
2798 message: impl Into<String>,
2799 ) -> Result<Frame, RouterError> {
2800 self.observe_route_open_refusal(ctx, module_id, reason, code);
2801 control_error_frame(frame, code, message.into())
2802 }
2803
2804 fn route_open_breaker_refusal_frame(
2822 &self,
2823 ctx: &RouteCtx,
2824 frame: &Frame,
2825 module_id: &str,
2826 consecutive_timeouts: u32,
2827 retry_in: Duration,
2828 probe_in_flight: bool,
2829 ) -> Result<Frame, RouterError> {
2830 self.counters
2831 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
2832 debug!(
2833 target: "control",
2834 code = "module_timeout",
2835 module_id = ?module_id,
2836 connection_id = ctx.connection_id.get(),
2837 consecutive_timeouts,
2838 retry_in_ms = retry_in.as_millis() as u64,
2839 probe_in_flight,
2840 "route.open refused by open bind-relay breaker"
2841 );
2842 let detail = if probe_in_flight {
2849 "one test route is already being tried; retry once it settles".to_string()
2850 } else {
2851 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
2852 };
2853 control_error_frame(
2854 frame,
2855 "module_timeout",
2856 format!(
2857 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
2858 timed out in a row); {detail}; its established routes are unaffected"
2859 ),
2860 )
2861 }
2862
2863 fn observe_route_open_refusal(
2876 &self,
2877 ctx: &RouteCtx,
2878 module_id: &str,
2879 reason: &'static str,
2880 code: &'static str,
2881 ) {
2882 self.counters.increment_route_open_refused(code);
2883 info!(
2884 target: "control",
2885 code,
2886 reason,
2887 module_id = ?module_id,
2888 connection_id = ctx.connection_id.get(),
2889 "route.open refused"
2890 );
2891 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
2892 self.route_outages.record_not_serving(module_id, reason);
2893 }
2894 }
2895
2896 fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
2941 self.route_outages.record_accepted(module_id);
2942 self.counters.increment_route_open_accepted(principal);
2943 info!(
2944 target: "control",
2945 principal,
2946 module_id,
2947 connection_id = ctx.connection_id.get(),
2948 "route.open accepted"
2949 );
2950 }
2951
2952 fn supervised_absent_route_open_refusal_frame(
2953 &self,
2954 ctx: &RouteCtx,
2955 frame: &Frame,
2956 module_id: &str,
2957 code: &'static str,
2958 status: &crate::supervise::ModuleStatus,
2959 ) -> Result<Frame, RouterError> {
2960 self.counters.increment_route_open_refused(code);
2961 info!(
2962 target: "control",
2963 code,
2964 reason = "supervised_not_registered",
2965 module_id = ?module_id,
2966 connection_id = ctx.connection_id.get(),
2967 state = %status.state,
2968 enabled = status.enabled,
2969 live = status.live,
2970 "route.open refused"
2971 );
2972 self.route_outages
2976 .record_not_serving(module_id, "supervised_not_registered");
2977 control_error_frame(
2978 frame,
2979 code,
2980 format!(
2981 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
2982 status.state, status.enabled, status.live
2983 ),
2984 )
2985 }
2986
2987 async fn handle_route_open(
2988 &self,
2989 ctx: &RouteCtx,
2990 frame: Frame,
2991 request: RouteOpenRequest,
2992 ) -> Result<Vec<Frame>, RouterError> {
2993 let RouteOpenRequest {
2994 target,
2995 mut identity,
2996 consumer_identity,
2997 consumer_capabilities,
2998 role_versions,
2999 admission_facts,
3000 scope,
3001 } = request;
3002 let target_module_id = target_module_id(&target).to_string();
3003 if self
3004 .registry
3005 .get_module_by_connection(ctx.connection_id)
3006 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3007 .is_some()
3008 {
3009 return Ok(vec![control_error_frame(
3010 &frame,
3011 "invalid_request",
3012 "module connections cannot open client routes",
3013 )?]);
3014 }
3015 debug!(
3016 connection_id = ctx.connection_id.get(),
3017 corr = frame.header.corr,
3018 module_id = %target_module_id,
3019 "handling route.open"
3020 );
3021
3022 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
3028 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
3029 self.observe_route_open_refusal(
3030 ctx,
3031 &target_module_id,
3032 "invalid_role_versions",
3033 error_codes::INVALID_REQUEST,
3034 );
3035 return Ok(vec![control_error_body_frame(
3036 &frame,
3037 ErrorBody {
3038 code: error_codes::INVALID_REQUEST.to_string(),
3039 message: error.to_string(),
3040 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
3041 },
3042 )?]);
3043 }
3044
3045 let Some(registration) = self
3064 .registry
3065 .get_module(&target_module_id)
3066 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3067 else {
3068 if let Some((status, warming)) =
3069 self.supervisor_status(&target_module_id, frame.header.corr)?
3070 {
3071 if status.protocol == ModuleProtocol::None {
3078 return Ok(vec![self.route_open_refusal_frame(
3079 ctx,
3080 &frame,
3081 &target_module_id,
3082 "protocol_none",
3083 error_codes::MODULE_NO_PROTOCOL,
3084 format!(
3085 "module_id '{target_module_id}' is declared protocol: none; \
3086 it speaks no subc wire and serves no routes"
3087 ),
3088 )?]);
3089 }
3090 let code = if warming {
3091 "module_warming"
3092 } else {
3093 "target_unavailable"
3094 };
3095 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
3096 ctx,
3097 &frame,
3098 &target_module_id,
3099 code,
3100 &status,
3101 )?]);
3102 }
3103 if let Some(removed_ago_ms) =
3104 self.supervisor.removal_tombstone_age_ms(&target_module_id)
3105 {
3106 return Ok(vec![self.route_open_refusal_frame(
3107 ctx,
3108 &frame,
3109 &target_module_id,
3110 "removed",
3111 error_codes::MODULE_REMOVED,
3112 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
3113 )?]);
3114 }
3115 return Ok(vec![self.route_open_refusal_frame(
3116 ctx,
3117 &frame,
3118 &target_module_id,
3119 "not_registered",
3120 error_codes::UNKNOWN_MODULE,
3121 format!("module_id '{target_module_id}' is not registered"),
3122 )?]);
3123 };
3124
3125 if !registration.ready {
3129 self.counters
3130 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
3131 info!(
3132 target: "control",
3133 code = error_codes::MODULE_WARMING,
3134 module_id = ?target_module_id,
3135 connection_id = ctx.connection_id.get(),
3136 reason = "declared_not_ready",
3137 "route.open refused"
3138 );
3139 self.route_outages
3142 .record_not_serving(&target_module_id, "declared_not_ready");
3143 return Ok(vec![control_error_body_frame(
3144 &frame,
3145 ErrorBody {
3146 code: error_codes::MODULE_WARMING.to_string(),
3147 message: format!(
3148 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
3149 ),
3150 detail: Some(serde_json::json!({
3151 "reason": "declared_not_ready"
3152 })),
3153 },
3154 )?]);
3155 }
3156
3157 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
3176 self.counters
3177 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
3178 info!(
3179 target: "control",
3180 code = error_codes::MODULE_WARMING,
3181 module_id = ?target_module_id,
3182 connection_id = ctx.connection_id.get(),
3183 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3184 capability = %capability,
3185 "route.open refused"
3186 );
3187 return Ok(vec![control_error_body_frame(
3188 &frame,
3189 ErrorBody {
3190 code: error_codes::MODULE_WARMING.to_string(),
3191 message: format!(
3192 "module_id '{target_module_id}' requires capability '{capability}', \
3193 which no registered module provides; retry"
3194 ),
3195 detail: Some(serde_json::json!({
3196 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3197 "capability": capability,
3198 })),
3199 },
3200 )?]);
3201 }
3202
3203 if !target_has_required_role(&target, ®istration.manifest.provides) {
3204 return Ok(vec![self.route_open_refusal_frame(
3205 ctx,
3206 &frame,
3207 &target_module_id,
3208 "role_not_provided",
3209 "target_unavailable",
3210 format!("module_id '{target_module_id}' does not provide the requested target"),
3211 )?]);
3212 }
3213
3214 if registration.state != ChannelState::Active {
3215 return Ok(vec![self.route_open_refusal_frame(
3216 ctx,
3217 &frame,
3218 &target_module_id,
3219 "registration_not_active",
3220 "target_unavailable",
3221 format!("module_id '{target_module_id}' is not active"),
3222 )?]);
3223 }
3224
3225 if self
3226 .forwarding
3227 .module_is_draining(&target_module_id)
3228 .map_err(RouterError::Forwarding)?
3229 {
3230 return Ok(vec![self.route_open_refusal_frame(
3231 ctx,
3232 &frame,
3233 &target_module_id,
3234 "reloading",
3235 "module_reloading",
3236 format!("module_id '{target_module_id}' is reloading"),
3237 )?]);
3238 }
3239
3240 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
3241 process_liveness.process_live(&target_module_id) == Some(false)
3242 }) {
3243 if process_liveness.process_replacing(&target_module_id) {
3250 return Ok(vec![self.route_open_refusal_frame(
3251 ctx,
3252 &frame,
3253 &target_module_id,
3254 "reloading",
3255 "module_reloading",
3256 format!("module_id '{target_module_id}' is reloading"),
3257 )?]);
3258 }
3259 return Ok(vec![self.route_open_refusal_frame(
3260 ctx,
3261 &frame,
3262 &target_module_id,
3263 "supervisor_not_live",
3264 "target_unavailable",
3265 format!("module_id '{target_module_id}' is not live"),
3266 )?]);
3267 }
3268
3269 if !self
3270 .forwarding
3271 .has_live_module_connection(&target_module_id)
3272 .map_err(RouterError::Forwarding)?
3273 {
3274 return Ok(vec![self.route_open_refusal_frame(
3275 ctx,
3276 &frame,
3277 &target_module_id,
3278 "no_forwarding_connection",
3279 "target_unavailable",
3280 format!("module_id '{target_module_id}' has no live forwarding connection"),
3281 )?]);
3282 }
3283
3284 if let Some(error) =
3285 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
3286 {
3287 self.observe_route_open_refusal(
3288 ctx,
3289 &target_module_id,
3290 "op_not_allowed",
3291 "op_not_allowed",
3292 );
3293 return Ok(vec![error]);
3294 }
3295
3296 let principal = match self.route_open_principal(&frame, consumer_identity)? {
3297 Ok(principal) => principal,
3298 Err(error) => {
3299 self.observe_route_open_refusal(
3300 ctx,
3301 &target_module_id,
3302 "bad_consumer_identity",
3303 "bad_consumer_identity",
3304 );
3305 return Ok(vec![error]);
3306 }
3307 };
3308
3309 if let Principal::Reserved {
3313 module_id: opening_module_id,
3314 } = &principal
3315 {
3316 if let Some(opening_registration) = self
3317 .registry
3318 .get_module(opening_module_id)
3319 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3320 {
3321 if let Some(capability) =
3322 denied_capability(&opening_registration.manifest, ®istration.manifest)
3323 {
3324 warn!(
3325 opening_module_id,
3326 target_module_id,
3327 capability,
3328 "refusing route.open because an attested capability deny edge matches"
3329 );
3330 return Ok(vec![self.route_open_refusal_frame(
3331 ctx,
3332 &frame,
3333 &target_module_id,
3334 "capability_deny_edge",
3335 "capability_forbidden",
3336 format!(
3337 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
3338 ),
3339 )?]);
3340 }
3341 }
3342 }
3343
3344 if admission_facts.is_some() {
3345 let carrier_matches = matches!(
3346 &principal,
3347 Principal::Reserved { module_id }
3348 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
3349 );
3350 if !carrier_matches {
3351 return Ok(vec![self.route_open_refusal_frame(
3352 ctx,
3353 &frame,
3354 &target_module_id,
3355 "admission_facts_carrier_not_permitted",
3356 "admission_facts_not_permitted",
3357 "admission facts may only be carried by the configured reserved module",
3358 )?]);
3359 }
3360
3361 let target_allowed = self
3362 .admission_facts_targets
3363 .as_ref()
3364 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
3365 if !target_allowed {
3366 return Ok(vec![self.route_open_refusal_frame(
3367 ctx,
3368 &frame,
3369 &target_module_id,
3370 "admission_facts_target_not_listed",
3371 "admission_facts_target_not_allowed",
3372 format!(
3373 "admission facts are not permitted for target module_id '{target_module_id}'"
3374 ),
3375 )?]);
3376 }
3377
3378 }
3382
3383 let (bound_scope, scope_stamp) = match scope {
3389 None => (None, None),
3390 Some(selector) => {
3391 let owner_configured = match &selector.owner {
3392 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3398 _ => false,
3399 };
3400 let admitted = self
3401 .scopes
3402 .read()
3403 .unwrap_or_else(|poisoned| poisoned.into_inner())
3404 .admit(&principal, &target_module_id, &selector, owner_configured)
3405 .and_then(|admission| {
3406 crate::scopes::check_target_flow_support(
3407 &admission.stamp,
3408 &target_module_id,
3409 registration.manifest.capabilities.as_ref(),
3410 )?;
3411 Ok(admission)
3412 });
3413 match admitted {
3414 Ok(admission) => (
3415 Some(BoundScope {
3416 owner: admission.owner,
3417 scope_ref: admission.stamp.scope_ref.clone(),
3418 tag: admission.tag,
3419 }),
3420 Some(admission.stamp),
3421 ),
3422 Err(refusal) => {
3423 return Ok(vec![self.route_open_refusal_frame(
3424 ctx,
3425 &frame,
3426 &target_module_id,
3427 refusal.code,
3428 refusal.code,
3429 refusal.message,
3430 )?]);
3431 }
3432 }
3433 }
3434 };
3435
3436 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
3458 Ok(project_root) => project_root,
3459 Err(err) => {
3460 return Ok(vec![control_error_frame(
3461 &frame,
3462 "invalid_project_root",
3463 err.to_string(),
3464 )?])
3465 }
3466 };
3467 identity.project_root = project_root.as_path().to_path_buf();
3468
3469 let _concurrency_guard = match self
3480 .route_bind_concurrency
3481 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
3482 {
3483 Ok(guard) => guard,
3484 Err(in_flight) => {
3485 return Ok(vec![self.route_open_target_capacity_refusal(
3486 ctx,
3487 &frame,
3488 &target_module_id,
3489 in_flight,
3490 )?]);
3491 }
3492 };
3493
3494 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
3497 RouteBindAdmission::Admitted { guard, probe } => {
3498 if probe {
3499 info!(
3500 module_id = %target_module_id,
3501 connection_id = ctx.connection_id.get(),
3502 "route.bind breaker half-open: admitting one probe"
3503 );
3504 }
3505 guard
3506 }
3507 RouteBindAdmission::Refused {
3508 consecutive_timeouts,
3509 retry_in,
3510 probe_in_flight,
3511 } => {
3512 return Ok(vec![self.route_open_breaker_refusal_frame(
3513 ctx,
3514 &frame,
3515 &target_module_id,
3516 consecutive_timeouts,
3517 retry_in,
3518 probe_in_flight,
3519 )?]);
3520 }
3521 };
3522
3523 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
3528 let relay_deadline = Instant::now() + route_bind_relay_timeout;
3529 let pending = match self
3530 .forwarding
3531 .begin_route_bind_relay_for(
3532 ctx.connection_id,
3533 ctx.egress.clone(),
3534 response_version(&frame),
3535 frame.header.corr,
3536 &target_module_id,
3537 principal.clone(),
3538 bound_scope,
3539 Some(project_root),
3540 relay_deadline,
3541 )
3542 .await
3543 {
3544 Ok(pending) => pending,
3545 Err(err) => {
3546 return Ok(vec![self.route_open_refusal_frame(
3547 ctx,
3548 &frame,
3549 &target_module_id,
3550 "relay_reservation_failed",
3551 forwarding_error_code(&err),
3552 err.to_string(),
3553 )?])
3554 }
3555 };
3556 let crate::forwarding::PendingRouteBindRelay {
3557 endpoint,
3558 module_sink,
3559 negotiated_ver,
3560 client_channel,
3561 client_epoch,
3562 module_channel,
3563 module_epoch,
3564 corr: relay_corr,
3565 receiver,
3566 } = pending;
3567 let mut reservation =
3568 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
3569
3570 if let Some(stamp) = scope_stamp
3575 .as_ref()
3576 .filter(|stamp| stamp.attributes.flow_id.is_some())
3577 {
3578 let relay_registration = self
3579 .registry
3580 .get_module_by_connection(endpoint.connection_id)
3581 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3582 if let Err(refusal) = crate::scopes::check_target_flow_support(
3583 stamp,
3584 &target_module_id,
3585 relay_registration
3586 .as_ref()
3587 .and_then(|registration| registration.manifest.capabilities.as_ref()),
3588 ) {
3589 reservation.release_and_disarm();
3590 return Ok(vec![self.route_open_refusal_frame(
3591 ctx,
3592 &frame,
3593 &target_module_id,
3594 refusal.code,
3595 refusal.code,
3596 refusal.message,
3597 )?]);
3598 }
3599 }
3600
3601 debug!(
3602 connection_id = ctx.connection_id.get(),
3603 client_channel,
3604 client_epoch,
3605 module_channel,
3606 module_epoch,
3607 "reserved route handle pair"
3608 );
3609 let principal_label = match &principal {
3612 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
3613 Principal::Direct => "direct".to_string(),
3614 other => format!("{other:?}"),
3615 };
3616 let relay = ModuleControlRequest::RouteBind {
3617 route_channel: module_channel,
3618 epoch: module_epoch,
3619 target,
3620 identity,
3621 principal: Some(principal),
3622 consumer_capabilities,
3623 role_versions,
3624 admission_facts,
3625 scope: scope_stamp,
3626 };
3627 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
3628 RouterError::backend(
3629 0,
3630 frame.header.corr,
3631 format!("failed to encode route.bind request: {err}"),
3632 )
3633 })?;
3634 let relay_frame = Frame::build_with_version(
3635 negotiated_ver,
3636 FrameType::Request,
3637 control_flags(),
3638 0,
3639 0,
3640 relay_corr,
3641 relay_body,
3642 )
3643 .map_err(RouterError::FrameBuild)?;
3644
3645 if let Err(err) = module_sink.send(relay_frame).await {
3646 reservation.release_and_disarm();
3647 return Ok(vec![self.route_open_refusal_frame(
3648 ctx,
3649 &frame,
3650 &target_module_id,
3651 "relay_send_failed",
3652 "target_unavailable",
3653 err.to_string(),
3654 )?]);
3655 }
3656
3657 if !self
3658 .forwarding
3659 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
3660 .map_err(RouterError::Forwarding)?
3661 {
3662 self.send_abandoned_route_bind_goodbye(
3663 &module_sink,
3664 negotiated_ver,
3665 module_channel,
3666 module_epoch,
3667 );
3668 }
3669
3670 match timeout_at(relay_deadline, receiver).await {
3671 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
3672 reservation.disarm();
3673 if breaker.record_accepted() {
3674 info!(
3675 module_id = %target_module_id,
3676 "route.bind breaker closed: the probe was accepted"
3677 );
3678 }
3679 self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
3680 Ok(Vec::new())
3681 }
3682 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
3683 reservation.release_and_disarm();
3684 breaker.record_inconclusive();
3688 let scope_code = match body.code.as_str() {
3692 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
3693 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
3694 _ => None,
3695 };
3696 if let Some(code) = scope_code {
3697 self.observe_route_open_refusal(
3698 ctx,
3699 &target_module_id,
3700 "scope_changed_before_commit",
3701 code,
3702 );
3703 return Ok(vec![control_error_body_frame(&frame, body)?]);
3704 }
3705 self.counters
3706 .increment_route_open_refused("module_rejected");
3707 info!(
3708 target: "control",
3709 code = "module_rejected",
3710 module_code = ?body.code,
3711 module_id = ?target_module_id,
3712 connection_id = ctx.connection_id.get(),
3713 "route.open refused"
3714 );
3715 Ok(vec![control_error_body_frame(&frame, body)?])
3716 }
3717 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
3718 reservation.release_and_disarm();
3719 breaker.record_inconclusive();
3720 tracing::warn!(
3727 module_id = %target_module_id,
3728 "route.bind relay abandoned: {message}"
3729 );
3730 Ok(vec![self.route_open_refusal_frame(
3731 ctx,
3732 &frame,
3733 &target_module_id,
3734 "relay_abandoned",
3735 "target_unavailable",
3736 message,
3737 )?])
3738 }
3739 Ok(Err(_)) => {
3740 reservation.release_and_disarm();
3741 breaker.record_inconclusive();
3742 Ok(vec![self.route_open_refusal_frame(
3743 ctx,
3744 &frame,
3745 &target_module_id,
3746 "relay_waiter_canceled",
3747 "target_unavailable",
3748 "route.bind relay waiter was canceled before the module responded",
3749 )?])
3750 }
3751 Err(_) => {
3752 reservation.release_and_disarm();
3753 if let Some(opened) = breaker.record_timeout(
3757 self.route_bind_breaker_threshold,
3758 self.route_bind_breaker_cooldown,
3759 ) {
3760 warn!(
3761 module_id = %target_module_id,
3762 consecutive_timeouts = opened.consecutive_timeouts,
3763 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
3764 reopened_after_probe = opened.reopened_after_probe,
3765 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
3766 );
3767 }
3768 tracing::warn!(
3776 module_id = %target_module_id,
3777 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
3778 "route.bind relay timed out: module did not ack within budget"
3779 );
3780 Ok(vec![self.route_open_refusal_frame(
3781 ctx,
3782 &frame,
3783 &target_module_id,
3784 "relay_timed_out",
3785 "module_timeout",
3786 format!(
3787 "module_id '{target_module_id}' did not answer route.bind within {:?}",
3788 route_bind_relay_timeout
3789 ),
3790 )?])
3791 }
3792 }
3793 }
3794
3795 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3796 let response = ClientControlResponse::SupervisorSpawnSnapshot {
3797 snapshot: self.supervisor.spawn_snapshot(),
3798 };
3799 Ok(vec![control_response_body_frame(
3800 &frame,
3801 &response,
3802 "ClientControlResponse::SupervisorSpawnSnapshot",
3803 )?])
3804 }
3805
3806 fn handle_supervisor_spawn_subscribe(
3807 &self,
3808 ctx: &RouteCtx,
3809 frame: Frame,
3810 since: Option<SpawnCursor>,
3811 ) -> Result<Vec<Frame>, RouterError> {
3812 match self.supervisor.subscribe_spawns(
3813 ctx.connection_id,
3814 frame.header.corr,
3815 response_version(&frame),
3816 since,
3817 ctx.egress.clone(),
3818 ) {
3819 Ok(()) => Ok(Vec::new()),
3820 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
3821 Ok(vec![control_error_body_frame(
3822 &frame,
3823 ErrorBody {
3824 code: "spawn_cursor_incarnation_mismatch".to_string(),
3825 message: "spawn cursor belongs to a different daemon incarnation"
3826 .to_string(),
3827 detail: Some(serde_json::json!({
3828 "current_daemon_incarnation": current
3829 })),
3830 },
3831 )?])
3832 }
3833 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
3834 &frame,
3835 ErrorBody {
3836 code: "spawn_cursor_too_old".to_string(),
3837 message: "spawn cursor predates the retained event ring".to_string(),
3838 detail: Some(serde_json::json!({
3839 "oldest_retained_cursor": oldest
3840 })),
3841 },
3842 )?]),
3843 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
3844 0,
3845 frame.header.corr,
3846 format!("failed to open supervisor spawn subscription: {error}"),
3847 )),
3848 }
3849 }
3850
3851 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3852 let generation = self
3853 .registry
3854 .generation()
3855 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3856 let mut modules = Vec::new();
3857 for module in self.supervisor.list() {
3858 let status = module.status_for_control("list").map_err(|err| {
3859 RouterError::backend(
3860 0,
3861 frame.header.corr,
3862 format!("failed to read supervisor status: {err}"),
3863 )
3864 })?;
3865 let (configured, _) = module.configuration().map_err(|err| {
3866 RouterError::backend(
3867 0,
3868 frame.header.corr,
3869 format!("failed to read module configuration: {err}"),
3870 )
3871 })?;
3872 let image = module.running_image_agreement().await;
3874 let resources = Some(module.child_resource_usage());
3877 let pending_reload = Some(reload_verdict(
3878 &configured.program,
3879 status.spawned_from.as_deref(),
3880 image,
3881 ));
3882 modules.push(SupervisorEntry {
3883 launch_nonce_env: Some(!cfg!(unix)),
3886 module_id: status.module_id,
3887 state: status.state.to_string(),
3888 enabled: status.enabled,
3889 live: status.live,
3890 protocol: status.protocol,
3891 health: status.health.status,
3892 pending_reload,
3893 last_probe_ms: status.health.last_probe_ms,
3894 last_exit_code: status.last_exit.as_ref().and_then(|e| e.code),
3895 last_exit_signal: status.last_exit.as_ref().and_then(|e| e.signal),
3896 last_exit_ms: status.last_exit.as_ref().map(|e| e.at_ms),
3897 last_exit_kind: status.last_exit.as_ref().map(|e| e.kind.into()),
3898 restart_count: Some(status.restart_count),
3899 max_restarts: Some(status.max_restarts),
3900 lifetime_restarts: Some(status.lifetime_restarts),
3901 spawn_generation: Some(status.spawn_generation),
3902 restart_window_secs: Some(status.restart_window.as_secs()),
3903 drain_timeout_ms: Some(status.drain_timeout.as_millis() as u64),
3904 restart_backoff_ms: Some(status.restart_backoff.as_millis() as u64),
3905 restart_max_backoff_ms: Some(status.restart_max_backoff.as_millis() as u64),
3906 resources,
3907 });
3908 }
3909 let response = ClientControlResponse::SupervisorList {
3910 generation,
3911 modules,
3912 };
3913 Ok(vec![control_response_body_frame(
3914 &frame,
3915 &response,
3916 "ClientControlResponse::SupervisorList",
3917 )?])
3918 }
3919
3920 fn handle_supervisor_stderr_tail(
3921 &self,
3922 frame: Frame,
3923 module_id: String,
3924 max_lines: Option<u32>,
3925 max_bytes: Option<u32>,
3926 ) -> Result<Vec<Frame>, RouterError> {
3927 let Some(module) = self.supervisor.get(&module_id) else {
3928 return Ok(vec![control_error_frame(
3929 &frame,
3930 "unknown_module",
3931 format!("module_id '{module_id}' is not supervised"),
3932 )?]);
3933 };
3934
3935 let snapshot = module.stderr_tail(
3936 max_lines.map(|value| value as usize),
3937 max_bytes.map(|value| value as usize),
3938 );
3939
3940 let response = ClientControlResponse::SupervisorStderrTail {
3941 module_id,
3942 tail: StderrTail {
3943 capture: match snapshot.capture {
3944 CaptureState::Captured => StderrCaptureState::Captured,
3945 CaptureState::Incomplete { reason } => {
3946 StderrCaptureState::Incomplete { reason }
3947 }
3948 CaptureState::NotCaptured { reason } => {
3949 StderrCaptureState::NotCaptured { reason }
3950 }
3951 },
3952 entries: snapshot
3953 .entries
3954 .into_iter()
3955 .map(|entry| match entry {
3956 TailEntry::Line {
3957 text,
3958 truncated,
3959 at_ms,
3960 } => StderrTailEntry::Line {
3961 text,
3962 truncated,
3963 at_ms,
3964 },
3965 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
3966 })
3967 .collect(),
3968 dropped_lines: snapshot.dropped_lines,
3969 },
3970 };
3971 Ok(vec![control_response_body_frame(
3972 &frame,
3973 &response,
3974 "ClientControlResponse::SupervisorStderrTail",
3975 )?])
3976 }
3977
3978 async fn handle_supervisor_terminals(
3979 &self,
3980 frame: Frame,
3981 module_id: String,
3982 ) -> Result<Vec<Frame>, RouterError> {
3983 let Some(module) = self.supervisor.get(&module_id) else {
3984 return Ok(vec![control_error_frame(
3985 &frame,
3986 "unknown_module",
3987 format!("module_id '{module_id}' is not supervised"),
3988 )?]);
3989 };
3990
3991 let terminals = module
3994 .read_durable_terminal_history()
3995 .await
3996 .map_err(|error| {
3997 RouterError::backend(
3998 0,
3999 frame.header.corr,
4000 format!("failed to read terminal history: {error}"),
4001 )
4002 })?;
4003 let response = ClientControlResponse::SupervisorTerminals {
4004 module_id,
4005 terminals,
4006 };
4007 Ok(vec![control_response_body_frame(
4008 &frame,
4009 &response,
4010 "ClientControlResponse::SupervisorTerminals",
4011 )?])
4012 }
4013
4014 fn handle_supervisor_routes(
4015 &self,
4016 frame: Frame,
4017 module_id: Option<String>,
4018 ) -> Result<Vec<Frame>, RouterError> {
4019 let modules = self
4020 .forwarding
4021 .route_census(module_id.as_deref())
4022 .map_err(RouterError::Forwarding)?
4023 .into_iter()
4024 .map(|(module_id, routes)| SupervisorRouteModule {
4025 module_id,
4026 routes: routes
4027 .into_iter()
4028 .map(|route| SupervisorRoute {
4029 consumer: match route.principal {
4030 Principal::Reserved { module_id } => {
4031 SupervisorRouteConsumer::Reserved { module_id }
4032 }
4033 Principal::Direct | Principal::Unverified => {
4034 SupervisorRouteConsumer::Direct {
4035 connection_id: route.goodbye_target.connection_id.get(),
4036 }
4037 }
4038 },
4039 age_ms: Instant::now()
4040 .saturating_duration_since(route.bound_at)
4041 .as_millis()
4042 .try_into()
4043 .unwrap_or(u64::MAX),
4044 draining: route.draining,
4045 drain_reason: route.drain_reason,
4046 })
4047 .collect(),
4048 })
4049 .collect();
4050 let response = ClientControlResponse::SupervisorRoutes { modules };
4051 Ok(vec![control_response_body_frame(
4052 &frame,
4053 &response,
4054 "ClientControlResponse::SupervisorRoutes",
4055 )?])
4056 }
4057
4058 async fn handle_supervisor_provenance(
4059 &self,
4060 frame: Frame,
4061 module_id: Option<String>,
4062 ) -> Result<Vec<Frame>, RouterError> {
4063 let mut selected = if let Some(module_id) = module_id {
4064 let Some(module) = self.supervisor.get(&module_id) else {
4065 return Ok(vec![control_error_frame(
4066 &frame,
4067 "unknown_module",
4068 format!("module_id '{module_id}' is not supervised"),
4069 )?]);
4070 };
4071 vec![module]
4072 } else {
4073 self.supervisor.list()
4074 };
4075
4076 let mut modules = Vec::with_capacity(selected.len());
4077 for module in selected.drain(..) {
4078 let status = module.status().map_err(|err| {
4079 RouterError::backend(
4080 0,
4081 frame.header.corr,
4082 format!("failed to read supervisor status: {err}"),
4083 )
4084 })?;
4085 let module_declared = self
4086 .registry
4087 .get_module(&status.module_id)
4088 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4089 .and_then(|registration| registration.manifest.provenance)
4090 .map(|build| ModuleDeclaredProvenance::Reported { build })
4091 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
4092 #[cfg(test)]
4093 let running_image = match &self.provenance_probe_override {
4094 Some(result) => result.clone(),
4095 None => module.running_image_agreement().await,
4096 };
4097 #[cfg(not(test))]
4098 let running_image = module.running_image_agreement().await;
4099 modules.push(SupervisorModuleProvenance {
4100 module_id: status.module_id,
4101 module_declared,
4102 daemon_observed: SupervisorObservedProcess {
4103 pid: status.pid,
4104 spawned_at_ms: status.spawned_at_ms,
4105 spawned_from: status.spawned_from,
4106 running_image,
4107 },
4108 });
4109 }
4110 let daemon = SupervisorDaemonProvenance {
4111 daemon_build: self.daemon_provenance.build.clone(),
4112 daemon_observed: DaemonObservedProcess {
4113 pid: self.daemon_provenance.pid,
4114 started_at_ms: self
4115 .daemon_provenance
4116 .start_clock
4117 .map(|clock| clock.started_at_ms())
4118 .or(self.daemon_provenance.started_at_ms),
4119 running_image: self
4120 .daemon_provenance
4121 .probe
4122 .observe(
4123 self.daemon_provenance.pid,
4124 self.daemon_provenance.executable_path.as_deref(),
4125 self.daemon_provenance.executable_identity,
4126 self.daemon_provenance.process_start_time,
4127 )
4128 .await,
4129 },
4130 };
4131 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
4132 Ok(vec![control_response_body_frame(
4133 &frame,
4134 &response,
4135 "ClientControlResponse::SupervisorProvenance",
4136 )?])
4137 }
4138
4139 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4140 self.refresh_capability_requirements();
4141 let generation = self
4142 .registry
4143 .generation()
4144 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4145 let modules = self
4146 .supervisor
4147 .list()
4148 .into_iter()
4149 .map(|module| {
4150 let status = module.status_for_control("health").map_err(|err| {
4151 RouterError::backend(
4152 0,
4153 frame.header.corr,
4154 format!("failed to read supervisor health: {err}"),
4155 )
4156 })?;
4157 let module_id = status.module_id;
4158 let capability_detail = self
4159 .capability_evaluator
4160 .required_problem_detail(&module_id);
4161 Ok(SupervisorHealthEntry {
4162 module_id,
4163 status: status.health.status,
4164 detail: append_capability_problem_detail(
4165 status.health.detail,
4166 capability_detail,
4167 ),
4168 metrics: status.health.metrics,
4169 consecutive_failures: status.health.consecutive_failures,
4170 late_answer_count: status.health.late_answer_count,
4171 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
4172 last_action: status.health.last_action,
4173 last_action_ms: status.health.last_action_ms,
4174 last_probe_ms: status.health.last_probe_ms,
4175 })
4176 })
4177 .collect::<Result<Vec<_>, RouterError>>()?;
4178 let response = ClientControlResponse::SupervisorHealth {
4179 generation,
4180 modules,
4181 };
4182 Ok(vec![control_response_body_frame(
4183 &frame,
4184 &response,
4185 "ClientControlResponse::SupervisorHealth",
4186 )?])
4187 }
4188
4189 async fn handle_supervisor_restart(
4190 &self,
4191 frame: Frame,
4192 module_id: String,
4193 drain_timeout_ms: Option<u64>,
4194 ) -> Result<Vec<Frame>, RouterError> {
4195 let operation_lock = self.supervisor.operation_lock();
4196 let _operation_guard = operation_lock.lock().await;
4197 let Some(module) = self.supervisor.get(&module_id) else {
4198 return Ok(vec![control_error_frame(
4199 &frame,
4200 "unknown_module",
4201 format!("module_id '{module_id}' is not supervised"),
4202 )?]);
4203 };
4204
4205 self.route_outages.mark_operator_action(&module_id);
4206 if let Err(err) = module.restart(drain_timeout_ms).await {
4207 self.route_outages
4208 .operator_action_ended_unrefused(&module_id);
4209 let (code, message) = match err {
4210 crate::supervise::SuperviseError::Disabled { .. } => {
4211 ("module_disabled", err.to_string())
4212 }
4213 crate::supervise::SuperviseError::SwapInProgress { .. } => {
4214 ("swap_in_progress", err.to_string())
4215 }
4216 _ => (
4217 "target_unavailable",
4218 format!("failed to restart module_id '{module_id}': {err}"),
4219 ),
4220 };
4221 return Ok(vec![control_error_frame(&frame, code, message)?]);
4222 }
4223
4224 let response = ClientControlResponse::SupervisorAck {
4225 module_id,
4226 applied: true,
4227 };
4228 Ok(vec![control_response_body_frame(
4229 &frame,
4230 &response,
4231 "ClientControlResponse::SupervisorAck",
4232 )?])
4233 }
4234
4235 async fn handle_supervisor_swap(
4239 &self,
4240 frame: Frame,
4241 module_id: String,
4242 ready_timeout_ms: Option<u64>,
4243 ) -> Result<Vec<Frame>, RouterError> {
4244 let module = {
4251 let operation_lock = self.supervisor.operation_lock();
4252 let _operation_guard = operation_lock.lock().await;
4253 self.supervisor.get(&module_id)
4254 };
4255 let Some(module) = module else {
4256 return Ok(vec![control_error_frame(
4257 &frame,
4258 "unknown_module",
4259 format!("module_id '{module_id}' is not supervised"),
4260 )?]);
4261 };
4262
4263 self.route_outages.mark_operator_action(&module_id);
4264 if let Err(err) = module
4265 .swap(ready_timeout_ms.map(Duration::from_millis))
4266 .await
4267 {
4268 self.route_outages
4269 .operator_action_ended_unrefused(&module_id);
4270 use crate::supervise::SuperviseError;
4271 let message = err.to_string();
4272 let error = match err {
4273 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
4274 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
4275 code: "swap_refused".to_string(),
4276 message,
4277 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
4278 },
4279 SuperviseError::SwapFailed {
4280 arm,
4281 candidate_exit,
4282 ..
4283 } => ErrorBody {
4284 code: "swap_failed".to_string(),
4285 message,
4286 detail: Some(serde_json::json!({
4287 "arm": arm.as_str(),
4288 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
4289 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
4290 })),
4291 },
4292 _ => ErrorBody::new(
4293 "target_unavailable",
4294 format!("failed to swap module_id '{module_id}': {message}"),
4295 ),
4296 };
4297 return Ok(vec![control_error_body_frame(&frame, error)?]);
4298 }
4299 self.route_outages
4303 .operator_action_ended_unrefused(&module_id);
4304
4305 let response = ClientControlResponse::SupervisorAck {
4306 module_id,
4307 applied: true,
4308 };
4309 Ok(vec![control_response_body_frame(
4310 &frame,
4311 &response,
4312 "ClientControlResponse::SupervisorAck",
4313 )?])
4314 }
4315
4316 async fn handle_supervisor_reload(
4317 &self,
4318 frame: Frame,
4319 module_id: String,
4320 ) -> Result<Vec<Frame>, RouterError> {
4321 let operation_lock = self.supervisor.operation_lock();
4322 let _operation_guard = operation_lock.lock().await;
4323 let Some(module) = self.supervisor.get(&module_id) else {
4324 return Ok(vec![control_error_frame(
4325 &frame,
4326 "unknown_module",
4327 format!("module_id '{module_id}' is not supervised"),
4328 )?]);
4329 };
4330
4331 self.route_outages.mark_operator_action(&module_id);
4332 if let Err(err) = module.reload().await {
4333 self.route_outages
4334 .operator_action_ended_unrefused(&module_id);
4335 let (code, message) = match err {
4336 crate::supervise::SuperviseError::Disabled { .. } => {
4337 ("module_disabled", err.to_string())
4338 }
4339 crate::supervise::SuperviseError::SwapInProgress { .. } => {
4340 ("swap_in_progress", err.to_string())
4341 }
4342 _ => (
4343 "reload_failed",
4344 format!("failed to reload module_id '{module_id}': {err}"),
4345 ),
4346 };
4347 return Ok(vec![control_error_frame(&frame, code, message)?]);
4348 }
4349
4350 let response = ClientControlResponse::SupervisorAck {
4351 module_id,
4352 applied: true,
4353 };
4354 Ok(vec![control_response_body_frame(
4355 &frame,
4356 &response,
4357 "ClientControlResponse::SupervisorAck",
4358 )?])
4359 }
4360
4361 async fn handle_supervisor_rescan(
4362 &self,
4363 frame: Frame,
4364 preview: bool,
4365 ) -> Result<Vec<Frame>, RouterError> {
4366 let Some(context) = self.rescan.clone() else {
4367 return Ok(vec![control_error_frame(
4368 &frame,
4369 "rescan_unavailable",
4370 "the daemon was not started with a reloadable config path".to_string(),
4371 )?]);
4372 };
4373
4374 let operation_lock = self.supervisor.operation_lock();
4375 let _operation_guard = operation_lock.lock().await;
4376 let loaded = match crate::daemon_config::load(&context.config_path) {
4377 Ok(config) => config,
4378 Err(err) => {
4379 return Ok(vec![control_error_frame(
4380 &frame,
4381 "invalid_daemon_config",
4382 format!("supervisor rescan rejected daemon config: {err}"),
4383 )?])
4384 }
4385 };
4386 let Some(config) = loaded else {
4394 return Ok(vec![control_error_frame(
4395 &frame,
4396 "invalid_daemon_config",
4397 format!(
4398 "daemon config not found at {}; refusing to rescan (an absent config would \
4399 retire every supervised module)",
4400 context.config_path.display()
4401 ),
4402 )?]);
4403 };
4404 let (
4405 configured_port,
4406 storage_config,
4407 admission_facts_carrier_module_id,
4408 admission_facts_targets,
4409 scope_authority_owners,
4410 modules,
4411 reserved_capabilities,
4412 ) = (
4413 config.port,
4414 config.storage,
4415 config.admission_facts_carrier_module_id,
4416 config.admission_facts_targets,
4417 config.scope_authority_owners,
4418 config.modules,
4419 config.reserved_capabilities,
4420 );
4421
4422 let mut restart_required = Vec::new();
4431 for section in RestartRequiredSection::ALL {
4432 let changed = match section {
4433 RestartRequiredSection::Port => configured_port != context.configured_port,
4434 RestartRequiredSection::Storage => storage_config != context.storage_config,
4435 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
4436 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
4437 }
4438 RestartRequiredSection::AdmissionFactsTargets => {
4439 admission_facts_targets != context.admission_facts_targets
4440 }
4441 RestartRequiredSection::ScopeAuthorityOwners => {
4442 scope_authority_owners != context.scope_authority_owners
4443 }
4444 };
4445 if changed {
4446 restart_required.push(section.label().to_string());
4447 }
4448 }
4449 if !restart_required.is_empty() {
4450 warn!(
4451 config_path = %context.config_path.display(),
4452 sections = %restart_required.join(", "),
4453 "daemon config changed outside the modules section; restart the daemon to apply those changes"
4454 );
4455 }
4456
4457 for configured in &modules {
4458 if let Err(err) = validate_spec(&configured.module_spec()) {
4459 return Ok(vec![control_error_frame(
4460 &frame,
4461 "invalid_daemon_config",
4462 format!("supervisor rescan rejected daemon config: {err}"),
4463 )?]);
4464 }
4465 }
4466
4467 let configured_capabilities = modules
4468 .iter()
4469 .map(|module| (module.module_id.clone(), module.enabled))
4470 .collect::<Vec<_>>();
4471 let preview_capability_warnings = if preview {
4472 let (_, registrations) = self.runtime_capability_snapshot()?;
4473 let current_modules = self
4474 .supervisor
4475 .list()
4476 .into_iter()
4477 .map(|module| module.module_id().to_string())
4478 .collect::<BTreeSet<_>>();
4479 let resulting_modules = configured_capabilities.clone();
4480 let removed = current_modules
4481 .into_iter()
4482 .filter(|module_id| {
4483 !resulting_modules
4484 .iter()
4485 .any(|(configured_id, _)| configured_id == module_id)
4486 })
4487 .collect::<Vec<_>>();
4488 self.capability_evaluator.preview_removal_warnings(
4489 resulting_modules,
4490 &removed,
4491 ®istrations,
4492 )
4493 } else {
4494 Vec::new()
4495 };
4496 let result = match self
4497 .reconcile_supervised_modules(&context.supervisor, modules, preview)
4498 .await
4499 {
4500 Ok(result) => result,
4501 Err(message) => {
4502 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
4503 }
4504 };
4505 if !preview {
4506 self.capability_evaluator
4507 .configure(configured_capabilities, reserved_capabilities);
4508 self.capability_evaluator.wake_deadline_loop();
4509 self.refresh_capability_requirements();
4510 }
4511 let mut result = result;
4512 result.restart_required = restart_required;
4513 result.capability_warnings = preview_capability_warnings;
4514 let response = ClientControlResponse::SupervisorRescan { result };
4515 Ok(vec![control_response_body_frame(
4516 &frame,
4517 &response,
4518 "ClientControlResponse::SupervisorRescan",
4519 )?])
4520 }
4521
4522 async fn handle_supervisor_release_reserved(
4523 &self,
4524 frame: Frame,
4525 module_id: String,
4526 ) -> Result<Vec<Frame>, RouterError> {
4527 let Some(context) = self.rescan.clone() else {
4528 return Ok(vec![control_error_frame(
4529 &frame,
4530 "release_unavailable",
4531 "reserved-id release requires a daemon started with a reloadable config path",
4532 )?]);
4533 };
4534 let operation_lock = self.supervisor.operation_lock();
4535 let _operation_guard = operation_lock.lock().await;
4536 let loaded = match crate::daemon_config::load(&context.config_path) {
4537 Ok(Some(config)) => config,
4538 Ok(None) => {
4539 return Ok(vec![control_error_frame(
4540 &frame,
4541 "invalid_daemon_config",
4542 format!(
4543 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
4544 context.config_path.display()
4545 ),
4546 )?])
4547 }
4548 Err(err) => {
4549 return Ok(vec![control_error_frame(
4550 &frame,
4551 "invalid_daemon_config",
4552 format!("unable to verify reserved-id release against daemon config: {err}"),
4553 )?])
4554 }
4555 };
4556 if loaded
4557 .modules
4558 .iter()
4559 .any(|configured| configured.module_id == module_id)
4560 {
4561 return Ok(vec![control_error_frame(
4562 &frame,
4563 "reserved_module_configured",
4564 format!(
4565 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
4566 ),
4567 )?]);
4568 }
4569 if !self.supervisor.release_retained_reserved_gate(&module_id) {
4570 return Ok(vec![control_error_frame(
4571 &frame,
4572 "reserved_gate_not_retained",
4573 format!(
4574 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
4575 ),
4576 )?]);
4577 }
4578
4579 let response = ClientControlResponse::SupervisorAck {
4580 module_id,
4581 applied: true,
4582 };
4583 Ok(vec![control_response_body_frame(
4584 &frame,
4585 &response,
4586 "ClientControlResponse::SupervisorAck",
4587 )?])
4588 }
4589
4590 async fn reconcile_supervised_modules(
4599 &self,
4600 supervisor: &Supervisor,
4601 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
4602 preview: bool,
4603 ) -> Result<SupervisorRescanResult, String> {
4604 let mut current = BTreeMap::new();
4605 for module in self.supervisor.list() {
4606 let (spec, health) = module.configuration().map_err(|err| {
4607 format!(
4608 "failed to read configuration for module_id '{}': {err}",
4609 module.module_id()
4610 )
4611 })?;
4612 let enabled = module
4613 .status()
4614 .map_err(|err| {
4615 format!(
4616 "failed to read status for module_id '{}': {err}",
4617 module.module_id()
4618 )
4619 })?
4620 .enabled;
4621 current.insert(
4622 module.module_id().to_string(),
4623 (module, spec, health, enabled),
4624 );
4625 }
4626 let configured = configured_modules
4627 .into_iter()
4628 .map(|module| (module.module_id.clone(), module))
4629 .collect::<BTreeMap<_, _>>();
4630
4631 let added = configured
4632 .keys()
4633 .filter(|module_id| !current.contains_key(*module_id))
4634 .cloned()
4635 .collect::<Vec<_>>();
4636 let removed = current
4637 .keys()
4638 .filter(|module_id| !configured.contains_key(*module_id))
4639 .cloned()
4640 .collect::<Vec<_>>();
4641 let mut changed_pending_reload = Vec::new();
4642 let mut configuration_changes = BTreeSet::new();
4643 let mut enabled_changes = BTreeSet::new();
4644 let mut unchanged = 0_u32;
4645
4646 for (module_id, configured_module) in &configured {
4647 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
4648 else {
4649 continue;
4650 };
4651 let launch_changed = *current_spec != configured_module.module_spec();
4655 let configuration_changed =
4656 launch_changed || *current_health != configured_module.health;
4657 let enabled_changed = *current_enabled != configured_module.enabled;
4658 if configuration_changed {
4659 configuration_changes.insert(module_id.clone());
4660 }
4661 if launch_changed {
4662 changed_pending_reload.push(module_id.clone());
4663 }
4664 if enabled_changed {
4665 enabled_changes.insert(module_id.clone());
4666 }
4667 if !configuration_changed && !enabled_changed {
4668 unchanged = unchanged.saturating_add(1);
4669 }
4670 }
4671
4672 if preview {
4678 return Ok(SupervisorRescanResult {
4679 added,
4680 removed,
4681 changed_pending_reload,
4682 enabled_changes: enabled_changes.iter().cloned().collect(),
4683 unchanged,
4684 preview: true,
4685 restart_required: Vec::new(),
4689 capability_warnings: Vec::new(),
4690 });
4691 }
4692
4693 for module_id in &removed {
4694 let module = ¤t
4695 .get(module_id)
4696 .expect("removed module came from current supervisor state")
4697 .0;
4698 module.retire().await.map_err(|err| {
4699 format!("failed to retire module_id '{module_id}' during rescan: {err}")
4700 })?;
4701 self.supervisor.record_rescan_removal(module_id);
4735 self.supervisor.retire(module_id);
4736 self.route_outages.forget(module_id);
4737 }
4738
4739 for module_id in configured.keys() {
4740 let Some((module, _, _, _)) = current.get(module_id) else {
4741 continue;
4742 };
4743 let configured_module = configured
4744 .get(module_id)
4745 .expect("configured module id came from configured map");
4746 if configuration_changes.contains(module_id) {
4747 module
4748 .update_configuration(
4749 configured_module.module_spec(),
4750 configured_module.health.clone(),
4751 configured_module.drain_timeout_ms,
4752 )
4753 .await
4754 .map_err(|err| {
4755 format!(
4756 "failed to update module_id '{module_id}' configuration during rescan: {err}"
4757 )
4758 })?;
4759 }
4760 if enabled_changes.contains(module_id) {
4761 self.route_outages.mark_operator_action(module_id);
4764 module
4765 .set_enabled(configured_module.enabled)
4766 .await
4767 .map_err(|err| {
4768 self.route_outages.operator_action_ended_unrefused(module_id);
4769 format!(
4770 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
4771 configured_module.enabled
4772 )
4773 })?;
4774 }
4775 }
4776
4777 for module_id in &added {
4778 let configured_module = configured
4779 .get(module_id)
4780 .expect("added module id came from configured map");
4781 supervisor
4782 .supervise_configured_with_health(
4783 configured_module.module_spec(),
4784 configured_module.enabled,
4785 configured_module.health.clone(),
4786 configured_module.drain_timeout_ms,
4787 configured_module.restart,
4788 )
4789 .map_err(|err| {
4790 format!("failed to add module_id '{module_id}' during rescan: {err}")
4791 })?;
4792 }
4793
4794 Ok(SupervisorRescanResult {
4795 added,
4796 removed,
4797 changed_pending_reload,
4798 enabled_changes: enabled_changes.iter().cloned().collect(),
4799 unchanged,
4800 preview: false,
4801 restart_required: Vec::new(),
4804 capability_warnings: Vec::new(),
4805 })
4806 }
4807
4808 async fn handle_supervisor_set_enabled(
4809 &self,
4810 frame: Frame,
4811 module_id: String,
4812 enabled: bool,
4813 ) -> Result<Vec<Frame>, RouterError> {
4814 let operation_lock = self.supervisor.operation_lock();
4815 let _operation_guard = operation_lock.lock().await;
4816 let Some(module) = self.supervisor.get(&module_id) else {
4817 return Ok(vec![control_error_frame(
4818 &frame,
4819 "unknown_module",
4820 format!("module_id '{module_id}' is not supervised"),
4821 )?]);
4822 };
4823
4824 self.route_outages.mark_operator_action(&module_id);
4827 let applied = match module.set_enabled(enabled).await {
4828 Ok(applied) => applied,
4829 Err(err) => {
4830 self.route_outages
4831 .operator_action_ended_unrefused(&module_id);
4832 return Ok(vec![control_error_frame(
4833 &frame,
4834 "target_unavailable",
4835 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
4836 )?]);
4837 }
4838 };
4839 if !applied {
4840 self.route_outages
4843 .operator_action_ended_unrefused(&module_id);
4844 }
4845
4846 self.capability_evaluator.wake_deadline_loop();
4847 self.refresh_capability_requirements();
4848 let response = ClientControlResponse::SupervisorAck { module_id, applied };
4849 Ok(vec![control_response_body_frame(
4850 &frame,
4851 &response,
4852 "ClientControlResponse::SupervisorAck",
4853 )?])
4854 }
4855
4856 async fn handle_supervisor_health_probe(
4857 &self,
4858 frame: Frame,
4859 module_id: String,
4860 ) -> Result<Vec<Frame>, RouterError> {
4861 self.refresh_capability_requirements();
4862 let Some(registration) = self
4863 .registry
4864 .get_module(&module_id)
4865 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4866 else {
4867 return Ok(vec![control_error_frame(
4868 &frame,
4869 "unknown_module",
4870 format!("module_id '{module_id}' is not registered"),
4871 )?]);
4872 };
4873
4874 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
4887 {
4888 return Ok(vec![control_error_frame(
4889 &frame,
4890 "health_not_advertised",
4891 format!("module_id '{module_id}' did not advertise health.check"),
4892 )?]);
4893 }
4894
4895 let deadline = Instant::now() + self.health_probe_timeout;
4896 let pending = match self.forwarding.begin_module_control_rpc_for(
4897 &module_id,
4898 MODULE_CONTROL_OP_HEALTH_CHECK,
4899 deadline,
4900 ) {
4901 Ok(pending) => pending,
4902 Err(err) => {
4903 return Ok(vec![control_error_frame(
4904 &frame,
4905 forwarding_error_code(&err),
4906 err.to_string(),
4907 )?])
4908 }
4909 };
4910
4911 let PendingModuleControlRpc {
4912 endpoint,
4913 module_sink,
4914 negotiated_ver,
4915 corr: probe_corr,
4916 receiver,
4917 } = pending;
4918 let mut guard =
4919 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
4920 let probe_body =
4921 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
4922 RouterError::backend(
4923 0,
4924 frame.header.corr,
4925 format!("failed to encode health.check request: {err}"),
4926 )
4927 })?;
4928 let probe_frame = Frame::build_with_version(
4929 negotiated_ver,
4930 FrameType::Request,
4931 control_flags(),
4932 0,
4933 0,
4934 probe_corr,
4935 probe_body,
4936 )
4937 .map_err(RouterError::FrameBuild)?;
4938
4939 if let Err(err) = module_sink.send(probe_frame).await {
4940 return Ok(vec![control_error_frame(
4941 &frame,
4942 "target_unavailable",
4943 err.to_string(),
4944 )?]);
4945 }
4946
4947 match timeout_at(deadline, receiver).await {
4948 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
4949 guard.disarm();
4950 let Some(report) = response.health_report() else {
4951 return Ok(vec![control_error_frame(
4952 &frame,
4953 "invalid_control_body",
4954 "health.check RPC returned a non-health response",
4955 )?]);
4956 };
4957 let HealthReport {
4962 status,
4963 detail,
4964 metrics,
4965 } = report;
4966 let capability_detail = self
4967 .capability_evaluator
4968 .required_problem_detail(&module_id);
4969 let response = ClientControlResponse::SupervisorHealthProbe {
4970 module_id,
4971 status,
4972 detail: append_capability_problem_detail(detail, capability_detail),
4973 metrics,
4974 };
4975 Ok(vec![control_response_body_frame(
4976 &frame,
4977 &response,
4978 "ClientControlResponse::SupervisorHealthProbe",
4979 )?])
4980 }
4981 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
4982 guard.disarm();
4983 Ok(vec![control_error_body_frame(&frame, body)?])
4984 }
4985 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
4986 guard.disarm();
4987 Ok(vec![control_error_frame(
4988 &frame,
4989 "target_unavailable",
4990 message,
4991 )?])
4992 }
4993 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
4994 guard.disarm();
4995 Ok(vec![control_error_frame(
4996 &frame,
4997 "invalid_control_body",
4998 message,
4999 )?])
5000 }
5001 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5002 guard.disarm();
5003 Ok(vec![control_error_frame(
5004 &frame,
5005 "invalid_control_body",
5006 format!("expected module-control op '{expected}', got '{actual}'"),
5007 )?])
5008 }
5009 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
5010 guard.disarm();
5011 Ok(vec![control_error_frame(
5012 &frame,
5013 "module_timeout",
5014 format!(
5015 "module_id '{module_id}' answered health.check after {:?}",
5016 self.health_probe_timeout
5017 ),
5018 )?])
5019 }
5020 Ok(Err(_)) => Ok(vec![control_error_frame(
5021 &frame,
5022 "target_unavailable",
5023 "health.check waiter was canceled before the module responded",
5024 )?]),
5025 Err(_) => Ok(vec![control_error_frame(
5026 &frame,
5027 "module_timeout",
5028 format!(
5029 "module_id '{module_id}' did not answer health.check within {:?}",
5030 self.health_probe_timeout
5031 ),
5032 )?]),
5033 }
5034 }
5035
5036 fn supervisor_status(
5037 &self,
5038 module_id: &str,
5039 corr: u64,
5040 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
5041 self.supervisor
5042 .get(module_id)
5043 .map(|module| {
5044 let warming = module.is_warming_for_control("status").map_err(|err| {
5045 RouterError::backend(
5046 0,
5047 corr,
5048 format!(
5049 "failed to read supervisor warming state for module_id '{module_id}': {err}"
5050 ),
5051 )
5052 })?;
5053 module.status_for_control("status").map_err(|err| {
5054 RouterError::backend(
5055 0,
5056 corr,
5057 format!(
5058 "failed to read supervisor status for module_id '{module_id}': {err}"
5059 ),
5060 )
5061 }).map(|status| (status, warming))
5062 })
5063 .transpose()
5064 }
5065
5066 fn guard_module_control_op(
5067 &self,
5068 frame: &Frame,
5069 module_id: &str,
5070 op: &str,
5071 ) -> Result<Option<Frame>, RouterError> {
5072 if self.module_grants_op(module_id, op, frame.header.corr)? {
5073 return Ok(None);
5074 }
5075
5076 Ok(Some(control_error_frame(
5077 frame,
5078 "op_not_allowed",
5079 format!("module_id '{module_id}' did not grant control op '{op}'"),
5080 )?))
5081 }
5082
5083 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
5084 let Some(registration) = self
5085 .registry
5086 .get_module(module_id)
5087 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
5088 else {
5089 return Ok(false);
5090 };
5091 Ok(module_registration_grants_op(®istration.control_ops, op))
5092 }
5093
5094 fn handle_status_update(
5095 &self,
5096 endpoint: ModuleEndpointId,
5097 frame: Frame,
5098 ) -> Result<Vec<Frame>, RouterError> {
5099 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
5100 Ok(update) => update,
5101 Err(err) => {
5102 if is_known_module_push_op(&frame.body) {
5107 return Ok(vec![control_error_frame(
5108 &frame,
5109 "invalid_control_body",
5110 format!("malformed module control push body: {err}"),
5111 )?]);
5112 }
5113 return Ok(Vec::new());
5114 }
5115 };
5116
5117 match update {
5118 ModuleControlPush::RouteStatus {
5119 route_channel,
5120 route_epoch,
5121 status,
5122 } => {
5123 self.forwarding
5124 .cache_status(endpoint, route_channel, route_epoch, status)
5125 .map_err(RouterError::Forwarding)?;
5126 }
5127 }
5128 Ok(Vec::new())
5129 }
5130
5131 fn handle_route_poll(
5132 &self,
5133 ctx: &RouteCtx,
5134 frame: Frame,
5135 route_channel: u16,
5136 route_epoch: u32,
5137 kind: PollKind,
5138 ) -> Result<Vec<Frame>, RouterError> {
5139 let snapshot = self
5140 .forwarding
5141 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
5142 .map_err(RouterError::Forwarding)?;
5143 let response = match (kind, snapshot) {
5144 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
5145 ClientControlResponse::RoutePoll {
5146 route_channel,
5147 route_epoch,
5148 status,
5149 live: None,
5150 }
5151 }
5152 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
5153 route_channel,
5154 route_epoch,
5155 status: None,
5156 live: None,
5157 },
5158 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
5159 let live = self
5185 .process_liveness
5186 .as_ref()
5187 .and_then(|source| source.process_live(&module_id))
5188 .unwrap_or(true);
5189 ClientControlResponse::RoutePoll {
5190 route_channel,
5191 route_epoch,
5192 status: None,
5193 live: Some(live),
5194 }
5195 }
5196 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
5197 route_channel,
5198 route_epoch,
5199 status: None,
5200 live: Some(false),
5201 },
5202 };
5203
5204 Ok(vec![control_response_body_frame(
5205 &frame,
5206 &response,
5207 "ClientControlResponse::RoutePoll",
5208 )?])
5209 }
5210
5211 pub(crate) fn observe_module_control_completion(
5212 &self,
5213 completion: ModuleControlRpcCompletion,
5214 ) -> bool {
5215 match completion {
5216 ModuleControlRpcCompletion::Unknown => false,
5217 ModuleControlRpcCompletion::Settled => true,
5218 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
5219 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
5220 info!(
5221 module_id = %module_id,
5222 latency_ms,
5223 "late health.check answer proves the module is alive"
5224 );
5225 match self
5226 .supervisor
5227 .record_late_health_answer(&module_id, latency_ms)
5228 {
5229 Ok(true) => {}
5230 Ok(false) => debug!(
5231 module_id = %module_id,
5232 latency_ms,
5233 "late health.check answer has no active supervisor snapshot"
5234 ),
5235 Err(err) => warn!(
5236 module_id = %module_id,
5237 latency_ms,
5238 error = %err,
5239 "failed to record late health.check answer"
5240 ),
5241 }
5242 true
5243 }
5244 }
5245 }
5246
5247 fn refuse_to_end_module_connection_for_a_client(
5268 &self,
5269 module_connection_id: ConnectionId,
5270 corr: u64,
5271 err: ForwardingError,
5272 ) -> Result<(), RouterError> {
5273 if let ForwardingError::ConnectionClosing { connection_id } = err {
5274 if connection_id != module_connection_id {
5275 warn!(
5276 module_connection_id = module_connection_id.get(),
5277 client_connection_id = connection_id.get(),
5278 corr,
5279 "dropping a route.bind response for a closing client; the module connection keeps serving"
5280 );
5281 return Ok(());
5282 }
5283 }
5284 Err(RouterError::Forwarding(err))
5285 }
5286
5287 fn handle_module_relay_response(
5288 &self,
5289 connection_id: ConnectionId,
5290 frame: Frame,
5291 ) -> Result<Vec<Frame>, RouterError> {
5292 let mut secondary_error = None;
5293 let outcome = match frame.header.ty {
5294 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
5295 Ok(probe) if probe.op == "route.bind" => {
5296 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
5297 Ok(ModuleControlResponse::RouteBindAck {}) => {
5298 RouteBindRelayOutcome::Accepted
5299 }
5300 Ok(other) => {
5301 let message =
5302 format!("route.bind response carried unexpected body: {other:?}");
5303 secondary_error = Some(control_error_frame(
5304 &frame,
5305 "invalid_control_body",
5306 message.clone(),
5307 )?);
5308 RouteBindRelayOutcome::ModuleGone(message)
5309 }
5310 Err(err) => {
5311 let message = format!("malformed route.bind response body: {err}");
5312 secondary_error = Some(control_error_frame(
5313 &frame,
5314 "invalid_control_body",
5315 message.clone(),
5316 )?);
5317 RouteBindRelayOutcome::ModuleGone(message)
5318 }
5319 }
5320 }
5321 Ok(probe) => {
5322 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
5323 {
5324 Ok(response) => ModuleControlRpcOutcome::Response(response),
5325 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
5326 "malformed {} response body: {err}",
5327 probe.op
5328 )),
5329 };
5330 let completion = self
5331 .forwarding
5332 .complete_module_control_rpc(
5333 connection_id,
5334 frame.header.corr,
5335 Some(&probe.op),
5336 outcome,
5337 )
5338 .map_err(RouterError::Forwarding)?;
5339 if !self.observe_module_control_completion(completion) {
5340 debug!(
5341 connection_id = connection_id.get(),
5342 corr = frame.header.corr,
5343 op = %probe.op,
5344 "dropping late or unknown module-control RPC response"
5345 );
5346 }
5347 return Ok(Vec::new());
5348 }
5349 Err(err) => {
5350 if let Some(expected_op) = self
5351 .forwarding
5352 .pending_module_control_op(connection_id, frame.header.corr)
5353 .map_err(RouterError::Forwarding)?
5354 {
5355 let completion = self
5356 .forwarding
5357 .complete_module_control_rpc(
5358 connection_id,
5359 frame.header.corr,
5360 None,
5361 ModuleControlRpcOutcome::MalformedResponse(format!(
5362 "malformed {expected_op} response body: {err}"
5363 )),
5364 )
5365 .map_err(RouterError::Forwarding)?;
5366 if !self.observe_module_control_completion(completion) {
5367 debug!(
5368 connection_id = connection_id.get(),
5369 corr = frame.header.corr,
5370 "dropping late malformed module-control RPC response"
5371 );
5372 }
5373 return Ok(Vec::new());
5374 }
5375 let message = format!("malformed route.bind response body: {err}");
5376 secondary_error = Some(control_error_frame(
5377 &frame,
5378 "invalid_control_body",
5379 message.clone(),
5380 )?);
5381 RouteBindRelayOutcome::ModuleGone(message)
5382 }
5383 },
5384 FrameType::Error => {
5385 if self
5386 .forwarding
5387 .pending_module_control_op(connection_id, frame.header.corr)
5388 .map_err(RouterError::Forwarding)?
5389 .is_some()
5390 {
5391 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
5392 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
5393 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
5394 "malformed module-control ERROR body: {err}"
5395 )),
5396 };
5397 let completion = self
5398 .forwarding
5399 .complete_module_control_rpc(
5400 connection_id,
5401 frame.header.corr,
5402 None,
5403 outcome,
5404 )
5405 .map_err(RouterError::Forwarding)?;
5406 if !self.observe_module_control_completion(completion) {
5407 debug!(
5408 connection_id = connection_id.get(),
5409 corr = frame.header.corr,
5410 "dropping late or unknown module-control RPC error"
5411 );
5412 }
5413 return Ok(Vec::new());
5414 }
5415 match serde_json::from_slice::<ErrorBody>(&frame.body) {
5416 Ok(body) => RouteBindRelayOutcome::Rejected(body),
5417 Err(err) => {
5418 let message = format!("malformed route.bind ERROR body: {err}");
5419 secondary_error = Some(control_error_frame(
5420 &frame,
5421 "invalid_control_body",
5422 message.clone(),
5423 )?);
5424 RouteBindRelayOutcome::ModuleGone(message)
5425 }
5426 }
5427 }
5428 ty => {
5429 return Ok(vec![control_error_frame(
5430 &frame,
5431 "unsupported_control_frame",
5432 format!("unsupported module channel-0 frame {ty:?}"),
5433 )?])
5434 }
5435 };
5436
5437 let settled =
5438 self.forwarding
5439 .complete_pending_relay(connection_id, frame.header.corr, outcome);
5440 let completion = match settled {
5441 Ok(completion) => completion,
5442 Err(err) => {
5443 self.refuse_to_end_module_connection_for_a_client(
5444 connection_id,
5445 frame.header.corr,
5446 err,
5447 )?;
5448 return Ok(secondary_error.into_iter().collect());
5449 }
5450 };
5451 if let Some(target) = completion.abandoned.as_ref() {
5452 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
5453 }
5454 if !completion.settled {
5455 debug!(
5456 connection_id = connection_id.get(),
5457 corr = frame.header.corr,
5458 frame_type = ?frame.header.ty,
5459 "dropping late or unknown route.bind relay response"
5460 );
5461 }
5462 Ok(secondary_error.into_iter().collect())
5463 }
5464
5465 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
5466 debug!(connection_id = connection_id.get(), "handling GOODBYE");
5467 self.cleanup_connection(connection_id)
5471 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
5472 Ok(Vec::new())
5473 }
5474}
5475
5476impl Default for ControlHandler {
5477 fn default() -> Self {
5478 Self::new(Arc::new(Registry::default()))
5479 }
5480}
5481
5482impl crate::supervise::SwapPromotionObserver for ControlHandler {
5483 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
5484 self.apply_registration_capabilities(registration);
5485 }
5486}
5487
5488fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
5489 CapabilityRequirementStatus {
5490 consumer: status.consumer,
5491 capability: status.capability,
5492 need: match status.need {
5493 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
5494 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
5495 },
5496 verdict: status.verdict.as_str().to_string(),
5497 episode_seq: status.episode_seq,
5498 config_satisfiable: status.config_satisfiable,
5499 runtime_available: status.runtime_available,
5500 detail: status.detail,
5501 }
5502}
5503
5504fn append_capability_problem_detail(
5505 detail: Option<String>,
5506 capability_detail: Option<String>,
5507) -> Option<String> {
5508 match (detail, capability_detail) {
5509 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
5510 (Some(detail), None) => Some(detail),
5511 (None, Some(capability_detail)) => Some(capability_detail),
5512 (None, None) => None,
5513 }
5514}
5515
5516fn subc_ops() -> Vec<String> {
5517 SUBC_CONTROL_OPS
5518 .iter()
5519 .map(|op| (*op).to_string())
5520 .collect()
5521}
5522
5523fn module_subc_ops() -> Vec<String> {
5524 SUBC_CONTROL_OPS
5525 .iter()
5526 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
5527 .map(|op| (*op).to_string())
5528 .collect()
5529}
5530
5531#[cfg(test)]
5532fn module_baseline_control_ops() -> Vec<String> {
5533 MODULE_BASELINE_CONTROL_OPS
5534 .iter()
5535 .map(|op| (*op).to_string())
5536 .collect()
5537}
5538
5539fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
5540 let mut seen = HashSet::new();
5541 let mut effective = Vec::new();
5542 for op in MODULE_BASELINE_CONTROL_OPS {
5543 if seen.insert((*op).to_string()) {
5544 effective.push((*op).to_string());
5545 }
5546 }
5547 for op in declared.unwrap_or_default() {
5548 if seen.insert(op.clone()) {
5549 effective.push(op);
5550 }
5551 }
5552 effective
5553}
5554
5555fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
5556 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
5557}
5558
5559fn target_module_id(target: &RouteTarget) -> &str {
5560 match target {
5561 RouteTarget::ToolProvider { module_id }
5562 | RouteTarget::ManagementSurface { module_id }
5563 | RouteTarget::InternalService { module_id, .. } => module_id,
5564 }
5565}
5566
5567fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
5568 roles.iter().any(|role| match (target, role) {
5569 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
5570 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
5571 (
5572 RouteTarget::InternalService { service_id, .. },
5573 ProviderRole::InternalService {
5574 service_id: provided,
5575 ..
5576 },
5577 ) => service_id == provided,
5578 _ => false,
5579 })
5580}
5581
5582fn is_routable_role(role: &ProviderRole) -> bool {
5583 matches!(
5584 role,
5585 ProviderRole::ToolProvider { .. }
5586 | ProviderRole::ManagementSurface { .. }
5587 | ProviderRole::InternalService { .. }
5588 )
5589}
5590
5591#[derive(Debug, Clone, Copy, PartialEq, Eq)]
5592enum ControlRequestBodyError {
5593 UnknownOp,
5594 InvalidBody,
5595}
5596
5597#[derive(Debug, Deserialize)]
5598struct ControlOpProbe {
5599 op: String,
5600}
5601
5602const MODULE_PUSH_OPS: &[&str] = &["route.status"];
5605
5606fn is_known_module_push_op(body: &[u8]) -> bool {
5607 serde_json::from_slice::<ControlOpProbe>(body)
5608 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
5609 .unwrap_or(false)
5610}
5611
5612fn is_known_module_request_op(body: &[u8]) -> bool {
5613 serde_json::from_slice::<ControlOpProbe>(body)
5614 .map(|probe| is_module_to_subc_op(&probe.op))
5615 .unwrap_or(false)
5616}
5617
5618fn is_module_to_subc_op(op: &str) -> bool {
5619 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
5620}
5621
5622fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
5623 debug!(
5624 op = %op,
5625 connection_id = connection_id.get(),
5626 corr,
5627 "control dispatch"
5628 );
5629}
5630
5631fn log_slow_control_dispatch(
5632 dispatch_started_at: Option<StdInstant>,
5633 op: &'static str,
5634 connection_id: ConnectionId,
5635 corr: u64,
5636) {
5637 let Some(dispatch_started_at) = dispatch_started_at else {
5638 return;
5639 };
5640 let elapsed = dispatch_started_at.elapsed();
5641 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
5642 warn!(
5643 op = %op,
5644 connection_id = connection_id.get(),
5645 corr,
5646 elapsed_ms = elapsed.as_millis() as u64,
5647 "slow control dispatch"
5648 );
5649 }
5650}
5651
5652fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
5653 match request {
5654 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
5655 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
5656 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
5657 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
5658 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
5659 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
5660 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
5661 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
5662 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
5663 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
5664 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
5665 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
5666 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
5667 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
5668 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
5669 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
5670 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
5671 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
5672 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
5673 }
5674}
5675
5676fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
5677 match request {
5678 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
5679 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
5680 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
5681 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
5682 }
5683}
5684
5685fn parse_client_control_request(
5686 body: &[u8],
5687) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
5688 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
5689 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5690 Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
5691 ControlRequestBodyError::InvalidBody
5692 }
5693 Ok(_) => ControlRequestBodyError::UnknownOp,
5694 Err(_) => ControlRequestBodyError::InvalidBody,
5695 };
5696 (err, classification)
5697 })
5698}
5699
5700fn parse_module_control_request_from_module(
5701 body: &[u8],
5702) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
5703 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
5704 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5705 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
5706 Ok(_) => ControlRequestBodyError::UnknownOp,
5707 Err(_) => ControlRequestBodyError::InvalidBody,
5708 };
5709 (err, classification)
5710 })
5711}
5712
5713#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
5714enum ProviderRoleKind {
5715 ToolProvider,
5716 PipelineStage,
5717 ManagementSurface,
5718 InternalService,
5719}
5720
5721fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
5722 match role {
5723 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
5724 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
5725 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
5726 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
5727 }
5728}
5729
5730fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
5731 roles.iter().map(provider_role_kind).collect()
5732}
5733
5734const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
5737
5738#[derive(Debug, Default, PartialEq, Eq)]
5740struct ScopeOutcomeCounts {
5741 created: usize,
5742 replaced: usize,
5743 updated: usize,
5744 unchanged: usize,
5745 refused: usize,
5746}
5747
5748impl ScopeOutcomeCounts {
5749 fn of(results: &[ScopeRecordResult]) -> Self {
5750 let mut counts = Self::default();
5751 for result in results {
5752 let slot = match result.outcome {
5753 ScopeRecordOutcome::Created => &mut counts.created,
5754 ScopeRecordOutcome::Replaced => &mut counts.replaced,
5755 ScopeRecordOutcome::Updated => &mut counts.updated,
5756 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
5757 ScopeRecordOutcome::Refused => &mut counts.refused,
5758 };
5759 *slot += 1;
5760 }
5761 counts
5762 }
5763}
5764
5765#[cfg(test)]
5766mod scope_outcome_count_tests {
5767 use super::*;
5768
5769 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
5770 ScopeRecordResult {
5771 scope_ref: "r".to_string(),
5772 scope_epoch: 1,
5773 outcome,
5774 code: None,
5775 message: None,
5776 version: None,
5777 parent_state: None,
5778 }
5779 }
5780
5781 #[test]
5784 fn every_outcome_is_counted_in_its_own_field() {
5785 let results = [
5786 result(ScopeRecordOutcome::Created),
5787 result(ScopeRecordOutcome::Created),
5788 result(ScopeRecordOutcome::Replaced),
5789 result(ScopeRecordOutcome::Updated),
5790 result(ScopeRecordOutcome::Unchanged),
5791 result(ScopeRecordOutcome::Refused),
5792 result(ScopeRecordOutcome::Refused),
5793 result(ScopeRecordOutcome::Refused),
5794 ];
5795 assert_eq!(
5796 ScopeOutcomeCounts::of(&results),
5797 ScopeOutcomeCounts {
5798 created: 2,
5799 replaced: 1,
5800 updated: 1,
5801 unchanged: 1,
5802 refused: 3,
5803 }
5804 );
5805 }
5806}
5807
5808fn capability_census_trigger(
5812 old: Option<&CapabilityDeclarations>,
5813 new: Option<&CapabilityDeclarations>,
5814) -> bool {
5815 let old_provides = old
5816 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
5817 .unwrap_or_default();
5818 let old_denies = old
5819 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
5820 .unwrap_or_default();
5821 let new = new.cloned().unwrap_or(CapabilityDeclarations {
5822 provides: Vec::new(),
5823 requires: Vec::new(),
5824 must_never_reach: Vec::new(),
5825 });
5826
5827 new.provides
5828 .iter()
5829 .any(|capability| !old_provides.contains(capability))
5830 || new
5831 .must_never_reach
5832 .iter()
5833 .any(|capability| !old_denies.contains(capability))
5834}
5835
5836fn denied_capability<'a>(
5839 opening_manifest: &'a ModuleManifest,
5840 target_manifest: &ModuleManifest,
5841) -> Option<&'a str> {
5842 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
5843 let target_capabilities = target_manifest.capabilities.as_ref()?;
5844 opening_capabilities
5845 .must_never_reach
5846 .iter()
5847 .find(|denied| {
5848 target_capabilities
5849 .provides
5850 .iter()
5851 .any(|provided| provided == *denied)
5852 })
5853 .map(String::as_str)
5854}
5855
5856fn catalog_update_frozen_field_message(
5857 registered: &ModuleManifest,
5858 provides: &[ProviderRole],
5859) -> Option<String> {
5860 let old_has_provides = !registered.provides.is_empty();
5861 let new_has_provides = !provides.is_empty();
5862 if old_has_provides != new_has_provides {
5863 return Some(format!(
5864 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
5865 registered.module_id
5866 ));
5867 }
5868
5869 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
5870 return Some(format!(
5871 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
5872 registered.module_id
5873 ));
5874 }
5875
5876 let registered_concurrency = manifest_concurrency(registered);
5877 let mut candidate = registered.clone();
5878 candidate.provides = provides.to_vec();
5879 let candidate_concurrency = manifest_concurrency(&candidate);
5880 if candidate_concurrency != registered_concurrency {
5881 return Some(format!(
5882 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
5883 registered.module_id, registered_concurrency, candidate_concurrency
5884 ));
5885 }
5886
5887 None
5890}
5891
5892fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
5893 manifest.provides.iter().any(is_routable_role)
5894}
5895
5896fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
5902 manifest
5903 .provides
5904 .iter()
5905 .find_map(|provider| match provider {
5906 ProviderRole::ToolProvider { concurrency, .. }
5907 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
5908 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
5909 })
5910 .unwrap_or(Concurrency::ModuleManaged)
5911}
5912
5913fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
5920 let has_management_surface = manifest
5921 .provides
5922 .iter()
5923 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
5924 if !has_management_surface {
5925 return false;
5926 }
5927 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
5928 return false;
5929 };
5930 let Some(provides) = raw
5931 .get("manifest")
5932 .and_then(|manifest| manifest.get("provides"))
5933 .and_then(serde_json::Value::as_array)
5934 else {
5935 return false;
5936 };
5937 provides.iter().any(|role| {
5942 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
5943 && role.get("concurrency").is_none()
5944 })
5945}
5946
5947fn negotiate_version(peer_version: u8) -> Result<u8, String> {
5948 if peer_version != PROTOCOL_VERSION {
5949 return Err(format!(
5950 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
5951 ));
5952 }
5953 Ok(PROTOCOL_VERSION)
5954}
5955
5956fn pong(frame: &Frame) -> Result<Frame, RouterError> {
5957 Frame::build_with_version(
5958 response_version(frame),
5959 FrameType::Pong,
5960 frame.header.flags,
5961 0,
5962 0,
5963 frame.header.corr,
5964 Vec::new(),
5965 )
5966 .map_err(RouterError::FrameBuild)
5967}
5968
5969fn control_error_frame(
5970 frame: &Frame,
5971 code: &'static str,
5972 message: impl Into<String>,
5973) -> Result<Frame, RouterError> {
5974 control_error_body_frame(
5975 frame,
5976 ErrorBody {
5977 code: code.to_string(),
5978 message: message.into(),
5979 detail: None,
5980 },
5981 )
5982}
5983
5984fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
5985 let body = serde_json::to_vec(&error).map_err(|err| {
5986 RouterError::backend(
5987 0,
5988 frame.header.corr,
5989 format!("failed to encode control ERROR: {err}"),
5990 )
5991 })?;
5992
5993 Frame::build_with_version(
5994 response_version(frame),
5995 FrameType::Error,
5996 control_flags(),
5997 0,
5998 0,
5999 frame.header.corr,
6000 body,
6001 )
6002 .map_err(RouterError::FrameBuild)
6003}
6004
6005fn control_response_body_frame<T: Serialize>(
6006 frame: &Frame,
6007 reply: &T,
6008 label: &'static str,
6009) -> Result<Frame, RouterError> {
6010 let body = serde_json::to_vec(reply).map_err(|err| {
6011 RouterError::backend(
6012 0,
6013 frame.header.corr,
6014 format!("failed to encode {label}: {err}"),
6015 )
6016 })?;
6017
6018 Frame::build_with_version(
6019 response_version(frame),
6020 FrameType::Response,
6021 control_flags(),
6022 0,
6023 0,
6024 frame.header.corr,
6025 body,
6026 )
6027 .map_err(RouterError::FrameBuild)
6028}
6029
6030fn forwarding_error_code(err: &ForwardingError) -> &'static str {
6060 match err {
6061 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
6062 ForwardingError::NoModuleConnection => "target_unavailable",
6063 ForwardingError::ModuleReloading { .. } => "module_reloading",
6064 ForwardingError::ClientRouteChannelExhausted { .. }
6065 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
6066 ForwardingError::StaleModuleEndpoint
6067 | ForwardingError::UnknownReservation { .. }
6068 | ForwardingError::ConnectionClosing { .. }
6069 | ForwardingError::ClientEgressClosed { .. }
6070 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
6071 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
6074 ForwardingError::RelayCorrelationExhausted
6075 | ForwardingError::RouteOpenBuild(_)
6076 | ForwardingError::Poisoned => "forwarding_error",
6077 }
6078}
6079
6080fn response_version(frame: &Frame) -> u8 {
6081 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
6082 frame.header.ver
6083 } else {
6084 PROTOCOL_VERSION
6085 }
6086}
6087
6088fn control_flags() -> Flags {
6089 Flags::new(false, Priority::Passive, false)
6090}
6091
6092fn send_goodbye_target_best_effort(
6097 counters: &DaemonCounters,
6098 target: &GoodbyeTarget,
6099 context: &'static str,
6100) {
6101 let Ok(frame) = Frame::build_with_version(
6102 target.negotiated_ver,
6103 FrameType::Goodbye,
6104 control_flags(),
6105 target.channel,
6106 target.epoch,
6107 0,
6108 Vec::new(),
6109 ) else {
6110 return;
6111 };
6112 crate::forwarding::send_module_route_goodbye(
6113 counters,
6114 &target.sink,
6115 frame,
6116 target.module_id.as_deref(),
6117 context,
6118 );
6119}
6120
6121pub(crate) fn send_route_control_pushes(
6122 forwarding: &ForwardingTable,
6123 routes: Vec<EndpointRoute>,
6124 push: ClientControlPush,
6125) {
6126 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
6127 for route in routes {
6128 let target = route.goodbye_target;
6129 if let Some((existing, channels)) = targets
6130 .iter_mut()
6131 .find(|(existing, _)| existing.connection_id == target.connection_id)
6132 {
6133 debug_assert_eq!(
6134 existing.negotiated_ver, target.negotiated_ver,
6135 "one connection cannot negotiate multiple frame versions"
6136 );
6137 if !channels.contains(&target.channel) {
6138 channels.push(target.channel);
6139 }
6140 continue;
6141 }
6142 let channel = target.channel;
6143 targets.push((target, vec![channel]));
6144 }
6145 for (target, mut channels) in targets {
6146 channels.sort_unstable();
6147 let mut push = push.clone();
6148 match &mut push {
6149 ClientControlPush::RouteClosing {
6150 channels: covered, ..
6151 }
6152 | ClientControlPush::RouteClosed {
6153 channels: covered, ..
6154 } => *covered = channels,
6155 }
6156 let body = match serde_json::to_vec(&push) {
6157 Ok(body) => body,
6158 Err(err) => {
6159 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
6160 continue;
6161 }
6162 };
6163 let frame = match Frame::build_with_version(
6164 target.negotiated_ver,
6165 FrameType::Push,
6166 control_flags(),
6167 0,
6168 0,
6169 0,
6170 body.clone(),
6171 ) {
6172 Ok(frame) => frame,
6173 Err(err) => {
6174 warn!(
6175 route_channel = target.channel,
6176 error = %err,
6177 "failed to build route lifecycle control PUSH frame"
6178 );
6179 continue;
6180 }
6181 };
6182 if let Err(err) = target.sink.try_send(frame) {
6183 if target.close_on_delivery_failure() {
6184 warn!(
6185 target_connection_id = target.connection_id.get(),
6186 route_channel = target.channel,
6187 error = %err,
6188 "route lifecycle control PUSH was not delivered to client; closing target connection"
6189 );
6190 let _ = forwarding.escalate_client_delivery_failure(
6191 target.connection_id,
6192 target.channel,
6193 target.epoch,
6194 CloseReason::new(
6195 "route_lifecycle_push_delivery_failed",
6196 format!(
6197 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
6198 target.channel
6199 ),
6200 ),
6201 crate::forwarding::UndeliveredFrame {
6202 module_id: target.module_id.as_deref(),
6203 sink: &target.sink,
6204 },
6205 );
6206 }
6207 }
6208 }
6209}
6210
6211#[cfg(test)]
6212mod tests {
6213 #[cfg(unix)]
6214 #[tokio::test]
6215 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
6216 let dir = subc_test_support::TestTempDir::new("rescan-live-health");
6217 let path = dir.join("subc.jsonc");
6218 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
6219 "program":"/bin/sleep","args":["60"],"protocol":"none",
6220 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
6221 }}}).to_string()).unwrap();
6222 let mut configured = crate::daemon_config::load(&path)
6223 .unwrap()
6224 .unwrap()
6225 .modules
6226 .pop()
6227 .unwrap();
6228 let registry = std::sync::Arc::new(crate::Registry::default());
6229 let handle = crate::SupervisorHandle::new();
6230 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
6231 .with_handle(handle.clone());
6232 let module = supervisor
6233 .supervise_configured_with_health(
6234 configured.module_spec(),
6235 true,
6236 configured.health.clone(),
6237 None,
6238 configured.restart,
6239 )
6240 .unwrap();
6241 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
6242 let before = module.status().unwrap().pid;
6243 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
6244 configured.health.cadence = std::time::Duration::from_secs(3600);
6245 let health_only = handler
6246 .reconcile_supervised_modules(&supervisor, vec![configured.clone()], false)
6247 .await
6248 .unwrap();
6249 assert!(
6250 health_only.changed_pending_reload.is_empty(),
6251 "health policy is already applied live"
6252 );
6253 assert_eq!(module.status().unwrap().pid, before);
6254 assert_eq!(
6255 module.configuration().unwrap().1.http,
6256 configured.health.http
6257 );
6258 configured.args = vec!["61".into()];
6259 let launch = handler
6260 .reconcile_supervised_modules(&supervisor, vec![configured], false)
6261 .await
6262 .unwrap();
6263 assert_eq!(launch.changed_pending_reload, ["stock"]);
6264 assert_eq!(
6265 module.status().unwrap().pid,
6266 before,
6267 "a launch edit is stored until reload"
6268 );
6269 module.drain().await.unwrap();
6270 }
6271 use std::{
6272 collections::BTreeMap,
6273 fmt,
6274 path::PathBuf,
6275 sync::{Arc, Mutex},
6276 time::Duration,
6277 };
6278 use subc_test_support::TestTempDir;
6279
6280 use serde_json::{json, Value};
6281 use subc_protocol::{
6282 manifest::{
6283 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
6284 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
6285 },
6286 session::HealthStatus,
6287 FrameType,
6288 };
6289
6290 use super::*;
6291 use crate::{
6292 forwarding::{DataRoute, DataRouteState},
6293 registry::ChannelState,
6294 router::FrameSink,
6295 stderr_tail::DEFAULT_MAX_LINE_BYTES,
6296 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
6297 RouteCtx, Router,
6298 };
6299 use tokio::{
6300 sync::mpsc,
6301 time::{sleep, Instant},
6302 };
6303 use tracing::{
6304 field::{Field, Visit},
6305 Event, Subscriber,
6306 };
6307 use tracing_subscriber::{layer::Context, prelude::*, Layer};
6308
6309 fn fake_aft_stub_path() -> PathBuf {
6328 let mut path = std::env::current_exe().expect("current_exe available in tests");
6329 path.pop(); path.pop(); path.push(if cfg!(windows) {
6332 "fake-aft-stub.exe"
6333 } else {
6334 "fake-aft-stub"
6335 });
6336 assert!(
6337 path.exists(),
6338 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
6339 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
6340 path.display()
6341 );
6342 path
6343 }
6344
6345 fn client_retries(code: &str) -> bool {
6350 subc_protocol::error_codes::is_retryable_route_open(code)
6351 }
6352
6353 #[test]
6364 fn retryability_of_forwarding_codes_matches_the_failure() {
6365 let transient = [
6368 ForwardingError::NoModuleConnection,
6369 ForwardingError::ModuleReloading {
6370 module_id: "m".into(),
6371 },
6372 ForwardingError::StaleModuleEndpoint,
6373 ForwardingError::UnknownReservation {
6374 client_channel: 1,
6375 module_channel: 1,
6376 },
6377 ForwardingError::ConnectionClosing {
6378 connection_id: ConnectionId::new(1),
6379 },
6380 ForwardingError::ClientEgressClosed {
6381 connection_id: ConnectionId::new(1),
6382 },
6383 ForwardingError::ModuleEgressUnavailable {
6384 connection_id: ConnectionId::new(1),
6385 },
6386 ];
6387 for err in transient {
6388 let code = forwarding_error_code(&err);
6389 assert!(
6390 client_retries(code),
6391 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
6392 );
6393 }
6394
6395 let permanent = [
6400 ForwardingError::ConnectionRoleConflict {
6401 connection_id: ConnectionId::new(1),
6402 },
6403 ForwardingError::ClientRouteChannelExhausted {
6404 connection_id: ConnectionId::new(1),
6405 },
6406 ForwardingError::ModuleRouteChannelExhausted {
6407 endpoint: ModuleEndpointId {
6408 connection_id: ConnectionId::new(1),
6409 generation: 1,
6410 },
6411 },
6412 ForwardingError::RelayCorrelationExhausted,
6413 ForwardingError::RouteOpenBuild("x".into()),
6414 ForwardingError::Poisoned,
6415 ];
6416 for err in permanent {
6417 let code = forwarding_error_code(&err);
6418 assert!(
6419 !client_retries(code),
6420 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
6421 );
6422 }
6423 }
6424
6425 #[tokio::test]
6453 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
6454 let handler = ControlHandler::default();
6455 let frame =
6456 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
6457
6458 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
6462 assert_eq!(
6463 stamped,
6464 Principal::Direct,
6465 "a caller that proved nothing must not be stamped as a supervised module"
6466 );
6467
6468 let forged = handler
6473 .route_open_principal(
6474 &frame,
6475 Some(ConsumerIdentity {
6476 module_id: "aft".to_string(),
6477 launch_nonce: "not-a-real-nonce".to_string(),
6478 }),
6479 )
6480 .unwrap();
6481 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
6482 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
6483 }
6484
6485 #[test]
6502 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
6503 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
6504 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
6505 let ClientControlRequest::RouteOpen {
6506 consumer_identity, ..
6507 } = parsed
6508 else {
6509 panic!("route.open body must parse as RouteOpen");
6510 };
6511 assert_eq!(
6512 consumer_identity,
6513 Some(ConsumerIdentity {
6514 module_id: "aft".to_string(),
6515 launch_nonce: "n".to_string(),
6516 }),
6517 "the wire field name must reach the value route_open_principal reads"
6518 );
6519 }
6520
6521 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
6522 ModuleManifest::builder(module_id, "0.1.0")
6523 .protocol_ver(protocol_ver)
6524 .provides(vec![ProviderRole::ToolProvider {
6525 tools: vec![Tool {
6526 name: "read".to_string(),
6527 description: None,
6528 execution_mode: ExecutionMode::Pure,
6529 schema: json!({"type": "object"}),
6530 }],
6531 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
6532 concurrency: Concurrency::ModuleManaged,
6533 emits_push: true,
6534 sub_supervises: true,
6535 }])
6536 .build()
6537 }
6538
6539 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
6540 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
6541 }
6542
6543 fn hello_frame_with_control_ops(
6544 module_id: &str,
6545 protocol_ver: u8,
6546 corr: u64,
6547 control_ops: Option<Vec<String>>,
6548 ) -> Frame {
6549 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
6550 }
6551
6552 fn hello_frame_with_nonce(
6553 module_id: &str,
6554 protocol_ver: u8,
6555 corr: u64,
6556 launch_nonce: Option<&str>,
6557 ) -> Frame {
6558 hello_frame_full(
6559 module_id,
6560 protocol_ver,
6561 corr,
6562 None,
6563 launch_nonce.map(ToOwned::to_owned),
6564 )
6565 }
6566
6567 fn hello_frame_full(
6568 module_id: &str,
6569 protocol_ver: u8,
6570 corr: u64,
6571 control_ops: Option<Vec<String>>,
6572 launch_nonce: Option<String>,
6573 ) -> Frame {
6574 let body = serde_json::to_vec(&ModuleHelloBody {
6575 manifest: manifest(module_id, protocol_ver),
6576 protocol_ver,
6577 control_ops,
6578 launch_nonce,
6579 })
6580 .unwrap();
6581 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
6582 }
6583
6584 fn non_routable_hello_frame_with_control_ops(
6585 module_id: &str,
6586 corr: u64,
6587 control_ops: Option<Vec<String>>,
6588 ) -> Frame {
6589 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
6590 manifest.provides.clear();
6591 let body = serde_json::to_vec(&ModuleHelloBody {
6592 manifest,
6593 protocol_ver: PROTOCOL_VERSION,
6594 control_ops,
6595 launch_nonce: None,
6596 })
6597 .unwrap();
6598 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
6599 }
6600
6601 fn capability_grammar_hello_frame(
6602 capabilities: Value,
6603 runtime_computed: Option<Value>,
6604 corr: u64,
6605 ) -> Frame {
6606 let mut body = serde_json::to_value(ModuleHelloBody {
6607 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
6608 protocol_ver: PROTOCOL_VERSION,
6609 control_ops: None,
6610 launch_nonce: None,
6611 })
6612 .expect("HELLO body serializes");
6613 body["manifest"]["capabilities"] = capabilities;
6614 if let Some(runtime_computed) = runtime_computed {
6615 body["runtime_computed"] = runtime_computed;
6616 }
6617 Frame::build(
6618 FrameType::Hello,
6619 control_flags(),
6620 0,
6621 0,
6622 corr,
6623 serde_json::to_vec(&body).expect("HELLO body reserializes"),
6624 )
6625 .expect("HELLO frame builds")
6626 }
6627
6628 fn channel_request(channel: u16, corr: u64) -> Frame {
6629 Frame::build(
6630 FrameType::Request,
6631 Flags::new(true, Priority::Interactive, false),
6632 channel,
6633 0,
6634 corr,
6635 b"opaque".to_vec(),
6636 )
6637 .unwrap()
6638 }
6639
6640 fn route_ctx(
6641 connection_id: ConnectionId,
6642 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
6643 let (tx, rx) = mpsc::channel(8);
6644 (
6645 RouteCtx {
6646 connection_id,
6647 egress: FrameSink::new(tx),
6648 },
6649 rx,
6650 )
6651 }
6652
6653 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
6654 serde_json::from_slice(&frame.body).unwrap()
6655 }
6656
6657 async fn hello_via_sink(
6662 handler: &ControlHandler,
6663 ctx: &RouteCtx,
6664 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
6665 hello: Frame,
6666 ) -> Frame {
6667 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
6668 assert!(
6669 replies.is_empty(),
6670 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
6671 );
6672 let ack = rx
6673 .try_recv()
6674 .expect("HELLO_ACK is queued on the module sink")
6675 .frame;
6676 assert_eq!(ack.header.ty, FrameType::HelloAck);
6677 ack
6678 }
6679
6680 fn parse_error(frame: &Frame) -> Value {
6681 serde_json::from_slice(&frame.body).unwrap()
6682 }
6683
6684 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
6685 serde_json::from_slice(&frame.body).unwrap()
6686 }
6687
6688 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
6689 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
6690 route_channel,
6691 route_epoch: 0,
6692 kind,
6693 })
6694 .unwrap();
6695 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6696 }
6697
6698 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
6699 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
6700 module_id: module_id.to_string(),
6701 })
6702 .unwrap();
6703 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6704 }
6705
6706 fn route_open_frame(corr: u64, module_id: &str, project_root: TestTempDir) -> Frame {
6707 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
6708 }
6709
6710 fn route_open_frame_with_consumer_capabilities(
6711 corr: u64,
6712 module_id: &str,
6713 project_root: TestTempDir,
6714 consumer_capabilities: Option<Vec<String>>,
6715 ) -> Frame {
6716 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6717 target: RouteTarget::ToolProvider {
6718 module_id: module_id.to_string(),
6719 },
6720 identity: BindIdentity::new(
6721 project_root.path().to_path_buf(),
6722 "unit".to_string(),
6723 "session".to_string(),
6724 ),
6725 consumer_identity: None,
6726 consumer_capabilities,
6727 role_versions: None,
6728 admission_facts: None,
6729 scope: None,
6730 })
6731 .unwrap();
6732 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6733 }
6734
6735 fn route_open_frame_with_role_versions(
6736 corr: u64,
6737 module_id: &str,
6738 project_root: TestTempDir,
6739 role_versions: Option<BTreeMap<String, String>>,
6740 ) -> Frame {
6741 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6742 target: RouteTarget::ToolProvider {
6743 module_id: module_id.to_string(),
6744 },
6745 identity: BindIdentity::new(
6746 project_root.path().to_path_buf(),
6747 "unit".to_string(),
6748 format!("session-{corr}"),
6749 ),
6750 consumer_identity: None,
6751 consumer_capabilities: None,
6752 role_versions,
6753 admission_facts: None,
6754 scope: None,
6755 })
6756 .unwrap();
6757 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6758 }
6759
6760 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
6761 entries
6762 .iter()
6763 .map(|(role, version)| (role.to_string(), version.to_string()))
6764 .collect()
6765 }
6766
6767 fn route_open_frame_with_admission_facts(
6768 corr: u64,
6769 module_id: &str,
6770 project_root: TestTempDir,
6771 consumer_identity: Option<subc_control::ConsumerIdentity>,
6772 facts: Option<Value>,
6773 ) -> Frame {
6774 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6775 target: RouteTarget::ToolProvider {
6776 module_id: module_id.to_string(),
6777 },
6778 identity: BindIdentity::new(
6779 project_root.path().to_path_buf(),
6780 "unit".to_string(),
6781 format!("session-{corr}"),
6782 ),
6783 consumer_identity,
6784 consumer_capabilities: None,
6785 role_versions: None,
6786 admission_facts: facts,
6787 scope: None,
6788 })
6789 .unwrap();
6790 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6791 }
6792
6793 #[derive(Clone, Default)]
6794 struct EventCapture {
6795 events: Arc<Mutex<Vec<CapturedEvent>>>,
6796 }
6797
6798 #[derive(Clone, Debug)]
6799 struct CapturedEvent {
6800 target: String,
6801 level: tracing::Level,
6802 fields: BTreeMap<String, String>,
6803 }
6804
6805 impl EventCapture {
6806 fn events(&self) -> Vec<CapturedEvent> {
6807 self.events.lock().unwrap().clone()
6808 }
6809 }
6810
6811 impl<S> Layer<S> for EventCapture
6812 where
6813 S: Subscriber,
6814 {
6815 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
6816 let mut visitor = EventFieldVisitor::default();
6817 event.record(&mut visitor);
6818 self.events.lock().unwrap().push(CapturedEvent {
6819 target: event.metadata().target().to_string(),
6820 level: *event.metadata().level(),
6821 fields: visitor.fields,
6822 });
6823 }
6824 }
6825
6826 #[derive(Default)]
6827 struct EventFieldVisitor {
6828 fields: BTreeMap<String, String>,
6829 }
6830
6831 impl Visit for EventFieldVisitor {
6832 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
6833 self.fields
6834 .insert(field.name().to_string(), format!("{value:?}"));
6835 }
6836 }
6837
6838 fn health_response(corr: u64, status: HealthStatus) -> Frame {
6839 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
6840 status,
6841 detail: Some("warming".to_string()),
6842 metrics: Some(json!({"queue_depth": 3})),
6843 })
6844 .unwrap();
6845 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
6846 }
6847
6848 fn route_bind_ack(corr: u64) -> Frame {
6849 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
6850 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
6851 }
6852
6853 fn unique_project_root(label: &str) -> TestTempDir {
6854 TestTempDir::new(label)
6855 }
6856
6857 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
6858 match parse_route_poll(frame) {
6859 ClientControlResponse::RoutePoll {
6860 status: None,
6861 live: Some(live),
6862 ..
6863 } => assert_eq!(live, expected_live),
6864 other => panic!("unexpected route.poll response: {other:?}"),
6865 }
6866 }
6867
6868 fn bind_liveness_route(
6869 registry: &Registry,
6870 forwarding: &ForwardingTable,
6871 module_id: &str,
6872 ) -> (RouteCtx, u16, u32) {
6873 let module_connection = ConnectionId::new(101);
6874 let client_connection = ConnectionId::new(202);
6875 let registration = registry
6876 .register_with_control_ops(
6877 manifest(module_id, PROTOCOL_VERSION),
6878 PROTOCOL_VERSION,
6879 module_connection,
6880 module_baseline_control_ops(),
6881 )
6882 .unwrap();
6883 let (module_tx, _module_rx) = mpsc::channel(8);
6884 let endpoint = forwarding
6885 .register_module_connection(
6886 module_connection,
6887 module_id.to_string(),
6888 PROTOCOL_VERSION,
6889 manifest_concurrency(®istration.manifest),
6890 FrameSink::new(module_tx),
6891 )
6892 .unwrap();
6893 let (client_ctx, _client_rx) = route_ctx(client_connection);
6894 let pending = forwarding
6895 .begin_route_bind_relay_for_test(
6896 client_connection,
6897 client_ctx.egress.clone(),
6898 1,
6899 module_id,
6900 )
6901 .unwrap();
6902 assert_eq!(pending.endpoint, endpoint);
6903 let route_channel = pending.client_channel;
6904 let route_epoch = pending.client_epoch;
6905 forwarding
6906 .complete_pending_relay(
6907 module_connection,
6908 pending.corr,
6909 RouteBindRelayOutcome::Accepted,
6910 )
6911 .unwrap();
6912 (client_ctx, route_channel, route_epoch)
6913 }
6914
6915 struct FakeProcessLiveness {
6916 live: Option<bool>,
6917 }
6918
6919 impl ModuleProcessLiveness for FakeProcessLiveness {
6920 fn process_live(&self, _module_id: &str) -> Option<bool> {
6921 self.live
6922 }
6923 }
6924
6925 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6926 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
6927 {
6928 let registry = Arc::new(Registry::default());
6929 let supervisor_handle = SupervisorHandle::new();
6930 let supervisor = Supervisor::new(
6931 Arc::clone(®istry),
6932 RestartPolicy::new(1, Duration::from_millis(10)),
6933 )
6934 .with_handle(supervisor_handle.clone());
6935 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
6936 let module = supervisor
6937 .spawn(ModuleSpec {
6938 module_id: "stderr-tail-wire".to_string(),
6939 program: fake_aft_stub_path(),
6940 args: Vec::new(),
6941 env: vec![
6942 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
6943 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
6944 ],
6945 reserved: false,
6946 reserved_prefixes: Vec::new(),
6947 protocol: ModuleProtocol::Subc,
6948 overlap: Default::default(),
6949 })
6950 .unwrap();
6951
6952 let deadline = Instant::now() + Duration::from_secs(5);
6953 loop {
6954 let tail = module.stderr_tail(None, None);
6955 if tail
6956 .entries
6957 .iter()
6958 .any(|entry| matches!(entry, TailEntry::ProcessStart))
6959 && tail.entries.iter().any(|entry| {
6960 matches!(
6961 entry,
6962 TailEntry::Line {
6963 truncated: true,
6964 ..
6965 }
6966 )
6967 })
6968 {
6969 break;
6970 }
6971 assert!(
6972 Instant::now() < deadline,
6973 "module did not produce a truncated line and restart boundary: {tail:?}"
6974 );
6975 sleep(Duration::from_millis(10)).await;
6976 }
6977
6978 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
6979 let request = ClientControlRequest::SupervisorStderrTail {
6980 module_id: "stderr-tail-wire".to_string(),
6981 max_lines: None,
6982 max_bytes: None,
6983 };
6984 let frame = Frame::build(
6985 FrameType::Request,
6986 control_flags(),
6987 0,
6988 0,
6989 1,
6990 serde_json::to_vec(&request).unwrap(),
6991 )
6992 .unwrap();
6993 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6994 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
6995 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
6996 serde_json::from_slice(&responses[0].body).unwrap()
6997 else {
6998 panic!("expected supervisor.stderr_tail response");
6999 };
7000
7001 assert!(
7002 tail.entries
7003 .iter()
7004 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
7005 "the control response lost the restart boundary"
7006 );
7007 let Some(StderrTailEntry::Line {
7008 text,
7009 truncated,
7010 at_ms,
7011 }) = tail.entries.iter().find(|entry| {
7012 matches!(
7013 entry,
7014 StderrTailEntry::Line {
7015 truncated: true,
7016 ..
7017 }
7018 )
7019 })
7020 else {
7021 panic!("the control response lost the truncated line");
7022 };
7023 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
7024 assert!(*truncated);
7025 assert!(
7026 at_ms.is_some(),
7027 "the control response lost the line's capture time"
7028 );
7029 }
7030
7031 #[tokio::test(flavor = "current_thread")]
7036 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
7037 let dir = TestTempDir::new("terminals-off-worker");
7038 let journal_path = dir.join("terminals.jsonl");
7039 let registry = Arc::new(Registry::default());
7040 let supervisor_handle = SupervisorHandle::new();
7041 let supervisor =
7042 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
7043 .with_handle(supervisor_handle.clone())
7044 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
7045 let module = supervisor
7046 .spawn(ModuleSpec {
7047 module_id: "terminal-off-worker".to_string(),
7048 program: fake_aft_stub_path(),
7049 args: Vec::new(),
7050 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
7051 reserved: false,
7052 reserved_prefixes: Vec::new(),
7053 protocol: ModuleProtocol::Subc,
7054 overlap: Default::default(),
7055 })
7056 .unwrap();
7057 let deadline = Instant::now() + Duration::from_secs(5);
7058 while module.terminal_history().entries.len() != 2 {
7059 assert!(Instant::now() < deadline, "module did not record two exits");
7060 sleep(Duration::from_millis(10)).await;
7061 }
7062
7063 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
7064 let handler =
7065 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
7066 let frame = Frame::build(
7067 FrameType::Request,
7068 control_flags(),
7069 0,
7070 0,
7071 1,
7072 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
7073 module_id: "terminal-off-worker".to_string(),
7074 })
7075 .unwrap(),
7076 )
7077 .unwrap();
7078 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7079 let spawned_at = std::time::Instant::now();
7080 let read = tokio::spawn({
7081 let handler = Arc::clone(&handler);
7082 async move { handler.handle_control_frame(&ctx, frame).await }
7083 });
7084 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
7087 .await
7088 .unwrap()
7089 .expect("the history read reached its pause");
7090 let elapsed = spawned_at.elapsed();
7091 assert!(
7092 elapsed < Duration::from_secs(2) && !read.is_finished(),
7093 "this task could not run while the history read was paused \
7094 (resumed after {elapsed:?}, read finished: {})",
7095 read.is_finished()
7096 );
7097
7098 drop(release);
7099 let responses = read.await.unwrap().unwrap();
7100 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
7101 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
7102 panic!("expected supervisor.terminals response");
7103 };
7104 assert_eq!(terminals.entries.len(), 2);
7105 assert_eq!(terminals.journal_skipped_lines, 0);
7106 assert_eq!(terminals.journal_read_errors, 0);
7107 }
7108
7109 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7110 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
7111 let registry = Arc::new(Registry::default());
7112 let supervisor_handle = SupervisorHandle::new();
7113 let supervisor =
7114 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
7115 .with_handle(supervisor_handle.clone());
7116 let module = supervisor
7117 .spawn(ModuleSpec {
7118 module_id: "terminal-golden".to_string(),
7119 program: fake_aft_stub_path(),
7120 args: Vec::new(),
7121 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
7122 reserved: false,
7123 reserved_prefixes: Vec::new(),
7124 protocol: ModuleProtocol::Subc,
7125 overlap: Default::default(),
7126 })
7127 .unwrap();
7128
7129 let deadline = Instant::now() + Duration::from_secs(5);
7130 while module.terminal_history().entries.len() != 2 {
7131 assert!(
7132 Instant::now() < deadline,
7133 "module did not retain two terminal exits: {:?}",
7134 module.terminal_history()
7135 );
7136 sleep(Duration::from_millis(10)).await;
7137 }
7138
7139 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
7140 let request = ClientControlRequest::SupervisorTerminals {
7141 module_id: "terminal-golden".to_string(),
7142 };
7143 let frame = Frame::build(
7144 FrameType::Request,
7145 control_flags(),
7146 0,
7147 0,
7148 1,
7149 serde_json::to_vec(&request).unwrap(),
7150 )
7151 .unwrap();
7152 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7153 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
7154 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
7155 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
7156 panic!("expected supervisor.terminals response");
7157 };
7158 assert_eq!(terminals.entries.len(), 2);
7159 assert_eq!(terminals.dropped, 0);
7160
7161 let mut rendered = serde_json::to_value(response).unwrap();
7162 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
7165 for (index, entry) in rendered["entries"]
7166 .as_array_mut()
7167 .expect("terminal response entries array")
7168 .iter_mut()
7169 .enumerate()
7170 {
7171 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
7172 }
7173
7174 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
7175 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
7176 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
7177 if std::env::var_os("UPDATE_GOLDEN").is_some() {
7178 std::fs::write(&golden_path, &serialized).unwrap();
7179 }
7180 let expected: Value =
7181 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
7182 assert_eq!(rendered, expected);
7183 }
7184
7185 #[test]
7186 fn hello_registers_manifest_and_returns_ack() {
7187 let registry = Arc::new(Registry::default());
7188 let handler = ControlHandler::new(Arc::clone(®istry));
7189 let conn = ConnectionId::new(1);
7190
7191 let responses = handler
7192 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
7193 .unwrap();
7194
7195 assert_eq!(responses.len(), 1);
7196 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
7197 assert_eq!(responses[0].header.channel, 0);
7198 assert_eq!(responses[0].header.corr, 7);
7199 let ack = parse_ack(&responses[0]);
7200 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
7201 assert!(ack
7202 .subc_capabilities
7203 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
7204 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
7205 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
7206 assert!(ack
7207 .subc_ops
7208 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
7209 assert!(ack
7210 .subc_ops
7211 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
7212
7213 let registration = registry.get_module("aft").unwrap().unwrap();
7214 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
7215 assert_eq!(registration.state, ChannelState::Active);
7216 assert_eq!(registration.connection_id, conn);
7217 assert_eq!(registration.control_ops, module_baseline_control_ops());
7218 }
7219
7220 #[test]
7221 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
7222 let invalid_identifiers = [
7223 ("case_change", "credentials-Provider/v1"),
7224 ("leading_zero", "credentials-provider/v01"),
7225 ("trailing_hyphen", "credentials-provider-/v1"),
7226 ("consecutive_hyphens", "credentials--provider/v1"),
7227 ("uppercase", "Credentials-provider/v1"),
7228 ("missing_v", "credentials-provider/1"),
7229 ("whitespace", "credentials provider/v1"),
7230 ("zero_version", "credentials-provider/v0"),
7231 ("out_of_range_version", "credentials-provider/v4294967296"),
7232 (
7233 "overlength_name",
7234 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
7235 ),
7236 ];
7237 let mut cases = invalid_identifiers
7238 .into_iter()
7239 .map(|(name, identifier)| {
7240 (
7241 format!("identifier_{name}"),
7242 "capabilities.provides[0]".to_string(),
7243 identifier.to_string(),
7244 json!({ "provides": [identifier] }),
7245 None,
7246 )
7247 })
7248 .collect::<Vec<_>>();
7249 cases.extend([
7250 (
7251 "unknown_need".to_string(),
7252 "capabilities.requires[0].need".to_string(),
7253 "deferred".to_string(),
7254 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
7255 None,
7256 ),
7257 (
7258 "duplicate_provides".to_string(),
7259 "capabilities.provides[1]".to_string(),
7260 "credentials-provider/v1".to_string(),
7261 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
7262 None,
7263 ),
7264 (
7265 "duplicate_must_never_reach".to_string(),
7266 "capabilities.must_never_reach[1]".to_string(),
7267 "credentials-provider/v1".to_string(),
7268 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
7269 None,
7270 ),
7271 (
7272 "duplicate_requires_same_need".to_string(),
7273 "capabilities.requires[1]".to_string(),
7274 "credentials-provider/v1".to_string(),
7275 json!({ "requires": [
7276 { "capability": "credentials-provider/v1", "need": "required" },
7277 { "capability": "credentials-provider/v1", "need": "required" }
7278 ] }),
7279 None,
7280 ),
7281 (
7282 "duplicate_requires_conflicting_need".to_string(),
7283 "capabilities.requires[1]".to_string(),
7284 "credentials-provider/v1".to_string(),
7285 json!({ "requires": [
7286 { "capability": "credentials-provider/v1", "need": "required" },
7287 { "capability": "credentials-provider/v1", "need": "optional" }
7288 ] }),
7289 None,
7290 ),
7291 (
7292 "capabilities_root_pointer".to_string(),
7293 "runtime_computed[0]".to_string(),
7294 "/capabilities".to_string(),
7295 json!({}),
7296 Some(json!(["/capabilities"])),
7297 ),
7298 (
7299 "capabilities_descendant_pointer".to_string(),
7300 "runtime_computed[0]".to_string(),
7301 "/capabilities/provides".to_string(),
7302 json!({}),
7303 Some(json!(["/capabilities/provides"])),
7304 ),
7305 (
7306 "malformed_pointer_without_leading_slash".to_string(),
7307 "runtime_computed[0]".to_string(),
7308 "capabilities".to_string(),
7309 json!({}),
7310 Some(json!(["capabilities"])),
7311 ),
7312 (
7313 "malformed_pointer_escape".to_string(),
7314 "runtime_computed[0]".to_string(),
7315 "/roles/~2/tools".to_string(),
7316 json!({}),
7317 Some(json!(["/roles/~2/tools"])),
7318 ),
7319 (
7320 "unknown_capabilities_field".to_string(),
7321 "capabilities.future".to_string(),
7322 "<array>".to_string(),
7323 json!({ "future": [] }),
7324 None,
7325 ),
7326 ]);
7327
7328 for (index, (name, field, value, capabilities, runtime_computed)) in
7329 cases.into_iter().enumerate()
7330 {
7331 let registry = Arc::new(Registry::default());
7332 let handler = ControlHandler::new(Arc::clone(®istry));
7333 let response = handler
7334 .handle_control(
7335 ConnectionId::new((index + 1) as u64),
7336 capability_grammar_hello_frame(
7337 capabilities,
7338 runtime_computed,
7339 index as u64 + 1,
7340 ),
7341 )
7342 .expect("invalid HELLO returns a refusal");
7343
7344 assert_eq!(response.len(), 1, "{name} must emit one refusal");
7345 let error = parse_error(&response[0]);
7346 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
7347 let message = error["message"]
7348 .as_str()
7349 .expect("error message is a string");
7350 assert!(
7351 message.contains(&field),
7352 "{name}: field missing from {message}"
7353 );
7354 assert!(
7355 message.contains(&value),
7356 "{name}: value missing from {message}"
7357 );
7358 assert_eq!(
7359 registry
7360 .active_registration_count()
7361 .expect("registry reads"),
7362 0,
7363 "{name}: refused HELLO must not create a catalog entry"
7364 );
7365 }
7366 }
7367
7368 #[test]
7369 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
7370 let registry = Arc::new(Registry::default());
7371 let handler = ControlHandler::new(Arc::clone(®istry));
7372 let capabilities = json!({
7373 "provides": ["credentials-provider/v1"],
7374 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
7375 "must_never_reach": ["federation-transport/v1"]
7376 });
7377 let response = handler
7378 .handle_control(
7379 ConnectionId::new(99),
7380 capability_grammar_hello_frame(
7381 capabilities.clone(),
7382 Some(json!(["/roles/0/tools"])),
7383 99,
7384 ),
7385 )
7386 .expect("valid HELLO registers");
7387 assert_eq!(response[0].header.ty, FrameType::HelloAck);
7388
7389 let request = Frame::build(
7390 FrameType::Request,
7391 control_flags(),
7392 0,
7393 0,
7394 100,
7395 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7396 .expect("catalog request serializes"),
7397 )
7398 .expect("catalog request frame builds");
7399 let response = handler
7400 .handle_catalog_list(request, None)
7401 .expect("catalog list succeeds");
7402 let ClientControlResponse::CatalogList { modules, .. } =
7403 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
7404 else {
7405 panic!("catalog request must return catalog.list");
7406 };
7407 assert_eq!(modules.len(), 1);
7408 assert_eq!(
7409 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
7410 capabilities
7411 );
7412 }
7413
7414 #[test]
7415 fn catalog_list_mirrors_management_operation_description() {
7416 let registry = Arc::new(Registry::default());
7417 let handler = ControlHandler::new(Arc::clone(®istry));
7418 let description = "List managed records and return their identifiers and metadata.";
7419 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
7420 manifest.provides = vec![ProviderRole::ManagementSurface {
7421 operations: vec![ManagementOperation {
7422 name: "records.list".to_string(),
7423 kind: ManagementOperationKind::Query,
7424 description: Some(description.to_string()),
7425 }],
7426 config_schema: json!({"type": "object"}),
7427 observability: vec![ObservabilitySurface {
7428 name: "records.stats".to_string(),
7429 kind: ObservabilityKind::Snapshot,
7430 }],
7431 identity_scope: vec![IdentityScope::Project],
7432 concurrency: Concurrency::ModuleManaged,
7433 }];
7434 registry
7435 .register_with_control_ops(
7436 manifest,
7437 PROTOCOL_VERSION,
7438 ConnectionId::new(99),
7439 Vec::new(),
7440 )
7441 .expect("described management manifest registers");
7442
7443 let request = Frame::build(
7444 FrameType::Request,
7445 control_flags(),
7446 0,
7447 0,
7448 100,
7449 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7450 .expect("catalog request serializes"),
7451 )
7452 .expect("catalog request frame builds");
7453 let response = handler
7454 .handle_catalog_list(request, None)
7455 .expect("catalog list succeeds");
7456 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
7457 assert_eq!(
7458 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
7459 "catalog.list must preserve the declared operation description verbatim"
7460 );
7461 }
7462
7463 #[test]
7464 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
7465 let registry = Arc::new(Registry::default());
7466 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
7467 [("vault".to_string(), true), ("squatter".to_string(), true)],
7468 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
7469 );
7470 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
7471 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
7472 provides: vec!["credentials-provider/v1".to_string()],
7473 requires: Vec::new(),
7474 must_never_reach: Vec::new(),
7475 });
7476 let frame = Frame::build(
7477 FrameType::Hello,
7478 control_flags(),
7479 0,
7480 0,
7481 77,
7482 serde_json::to_vec(&ModuleHelloBody {
7483 manifest: squatter,
7484 protocol_ver: PROTOCOL_VERSION,
7485 control_ops: None,
7486 launch_nonce: None,
7487 })
7488 .expect("HELLO serializes"),
7489 )
7490 .expect("HELLO frame builds");
7491 let response = handler
7492 .handle_control(ConnectionId::new(77), frame)
7493 .expect("reserved claim receives a typed refusal");
7494 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
7495 assert_eq!(
7496 registry
7497 .active_registration_count()
7498 .expect("registry reads"),
7499 0,
7500 "a reserved capability refusal must not leave a catalog entry"
7501 );
7502 }
7503
7504 #[test]
7505 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
7506 for settlement in ["timeout", "inconclusive", "drop"] {
7507 let breakers = RouteBindBreakers::default();
7508 let RouteBindAdmission::Admitted {
7509 guard: mut old,
7510 probe: false,
7511 } = breakers.admit("prov")
7512 else {
7513 panic!("ordinary relay admitted")
7514 };
7515 let RouteBindAdmission::Admitted {
7516 guard: mut opener, ..
7517 } = breakers.admit("prov")
7518 else {
7519 panic!("second relay admitted")
7520 };
7521 assert!(
7522 !opener
7523 .record_timeout(1, Duration::ZERO)
7524 .unwrap()
7525 .reopened_after_probe
7526 );
7527 let RouteBindAdmission::Admitted {
7528 guard: mut probe,
7529 probe: true,
7530 } = breakers.admit("prov")
7531 else {
7532 panic!("one cooldown probe admitted")
7533 };
7534 match settlement {
7535 "timeout" => assert!(
7536 !old.record_timeout(1, Duration::ZERO)
7537 .unwrap()
7538 .reopened_after_probe
7539 ),
7540 "inconclusive" => old.record_inconclusive(),
7541 "drop" => drop(old),
7542 _ => unreachable!(),
7543 }
7544 assert!(
7545 matches!(
7546 breakers.admit("prov"),
7547 RouteBindAdmission::Refused {
7548 probe_in_flight: true,
7549 ..
7550 }
7551 ),
7552 "{settlement} of a pre-open relay cannot release the real probe"
7553 );
7554 assert!(
7555 probe
7556 .record_timeout(1, Duration::ZERO)
7557 .unwrap()
7558 .reopened_after_probe
7559 );
7560 assert!(matches!(
7561 breakers.admit("prov"),
7562 RouteBindAdmission::Admitted { probe: true, .. }
7563 ));
7564 }
7565 let breakers = RouteBindBreakers::default();
7566 let admit = || match breakers.admit("prov") {
7567 RouteBindAdmission::Admitted { guard, .. } => guard,
7568 _ => panic!("relay admitted"),
7569 };
7570 admit().record_timeout(1, Duration::ZERO);
7571 let mut old_probe = admit();
7572 breakers.reset_for_new_module_connection("prov");
7573 admit().record_timeout(1, Duration::ZERO);
7574 let _new_probe = admit();
7575 old_probe.record_inconclusive();
7576 assert!(matches!(
7577 breakers.admit("prov"),
7578 RouteBindAdmission::Refused {
7579 probe_in_flight: true,
7580 ..
7581 }
7582 ));
7583 }
7584
7585 #[tokio::test]
7586 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
7587 for candidate in [false, true] {
7588 let registry = Arc::new(Registry::default());
7589 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
7590 [("vault".to_string(), true), ("squatter".to_string(), true)],
7591 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
7592 );
7593 let conn = ConnectionId::new(77);
7594 let (ctx, mut rx) = route_ctx(conn);
7595 let initial = capability_manifest("squatter", &[], &[]);
7596 if candidate {
7597 registry
7598 .register_candidate_with_control_ops(
7599 initial.clone(),
7600 PROTOCOL_VERSION,
7601 conn,
7602 module_baseline_control_ops(),
7603 )
7604 .unwrap();
7605 handler
7606 .forwarding
7607 .register_candidate_module_connection(
7608 conn,
7609 "squatter".to_string(),
7610 PROTOCOL_VERSION,
7611 manifest_concurrency(&initial),
7612 ctx.egress.clone(),
7613 )
7614 .unwrap();
7615 } else {
7616 hello_via_sink(
7617 &handler,
7618 &ctx,
7619 &mut rx,
7620 hello_frame_with_manifest(initial.clone(), 1),
7621 )
7622 .await;
7623 }
7624 let response = handler
7625 .handle_control_frame(
7626 &ctx,
7627 catalog_update_with_capabilities_frame(
7628 2,
7629 capability_manifest("squatter", &["credentials-provider/v1"], &[])
7630 .capabilities
7631 .unwrap(),
7632 ),
7633 )
7634 .await
7635 .unwrap();
7636 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
7637 assert_eq!(
7638 registry
7639 .get_module_by_connection(conn)
7640 .unwrap()
7641 .unwrap()
7642 .manifest,
7643 initial
7644 );
7645 }
7646 }
7647
7648 #[test]
7649 fn server_describe_surfaces_required_capability_verdict_fields() {
7650 let registry = Arc::new(Registry::default());
7651 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
7652 [
7653 ("consumer".to_string(), true),
7654 ("provider".to_string(), false),
7655 ],
7656 BTreeMap::new(),
7657 );
7658 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
7659 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
7660 provides: Vec::new(),
7661 requires: vec![subc_protocol::manifest::CapabilityRequirement {
7662 capability: "credentials-provider/v1".to_string(),
7663 need: subc_protocol::manifest::CapabilityNeed::Required,
7664 }],
7665 must_never_reach: Vec::new(),
7666 });
7667 let hello = Frame::build(
7668 FrameType::Hello,
7669 control_flags(),
7670 0,
7671 0,
7672 78,
7673 serde_json::to_vec(&ModuleHelloBody {
7674 manifest: consumer,
7675 protocol_ver: PROTOCOL_VERSION,
7676 control_ops: None,
7677 launch_nonce: None,
7678 })
7679 .expect("HELLO serializes"),
7680 )
7681 .expect("HELLO frame builds");
7682 handler
7683 .handle_control(ConnectionId::new(78), hello)
7684 .expect("consumer registers");
7685 let describe = Frame::build(
7686 FrameType::Request,
7687 control_flags(),
7688 0,
7689 0,
7690 79,
7691 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
7692 .expect("request serializes"),
7693 )
7694 .expect("describe frame builds");
7695 let response = handler
7696 .handle_server_describe(describe)
7697 .expect("server.describe succeeds");
7698 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
7699 let requirement = &rendered["capability_requirements"][0];
7700 assert_eq!(requirement["consumer"], "consumer");
7701 assert_eq!(requirement["verdict"], "never_provided");
7702 assert_eq!(requirement["episode_seq"], 1);
7703 assert_eq!(requirement["config_satisfiable"], false);
7704 assert_eq!(requirement["runtime_available"], false);
7705 assert!(requirement["detail"]
7706 .as_str()
7707 .expect("detail string")
7708 .contains("credentials-provider/v1"));
7709 }
7710
7711 #[test]
7712 fn catalog_list_omits_capabilities_for_legacy_manifest() {
7713 let registry = Arc::new(Registry::default());
7714 let handler = ControlHandler::new(Arc::clone(®istry));
7715 let hello = handler
7716 .handle_control(
7717 ConnectionId::new(101),
7718 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
7719 )
7720 .expect("legacy HELLO registers");
7721 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
7722
7723 let request = Frame::build(
7724 FrameType::Request,
7725 control_flags(),
7726 0,
7727 0,
7728 102,
7729 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7730 .expect("catalog request serializes"),
7731 )
7732 .expect("catalog request frame builds");
7733 let response = handler
7734 .handle_catalog_list(request, None)
7735 .expect("catalog list succeeds");
7736 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
7737 assert!(
7738 body["modules"][0].get("capabilities").is_none(),
7739 "legacy manifest must retain an absent capabilities field on catalog.list"
7740 );
7741 }
7742
7743 #[test]
7744 fn hello_ack_omits_storage_when_no_storage_config() {
7745 let registry = Arc::new(Registry::default());
7746 let handler = ControlHandler::new(Arc::clone(®istry));
7747 let responses = handler
7748 .handle_control(
7749 ConnectionId::new(1),
7750 hello_frame("aft", PROTOCOL_VERSION, 7),
7751 )
7752 .unwrap();
7753 let ack = parse_ack(&responses[0]);
7754 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
7755 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
7756 }
7757
7758 #[tokio::test]
7759 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
7760 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
7761 let registry = Arc::new(Registry::default());
7762 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
7763 let responses = handler
7764 .handle_control(
7765 ConnectionId::new(1),
7766 hello_frame("aft", PROTOCOL_VERSION, 7),
7767 )
7768 .unwrap();
7769 let ack = parse_ack(&responses[0]);
7770 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
7771
7772 let described = handler
7773 .handle_control_frame(
7774 &route_ctx(ConnectionId::new(2)).0,
7775 Frame::build(
7776 FrameType::Request,
7777 control_flags(),
7778 0,
7779 0,
7780 9,
7781 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
7782 )
7783 .unwrap(),
7784 )
7785 .await
7786 .unwrap();
7787 let ClientControlResponse::ServerDescribe { machine_id, .. } =
7788 serde_json::from_slice(&described[0].body).unwrap()
7789 else {
7790 panic!("server.describe answered with another shape");
7791 };
7792 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
7793 }
7794
7795 #[test]
7796 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
7797 let registry = Arc::new(Registry::default());
7800 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
7801 crate::daemon_config::StorageConfig::Sqlite {
7802 data_home: std::path::PathBuf::from("/data"),
7803 },
7804 ));
7805
7806 let responses = handler
7807 .handle_control(
7808 ConnectionId::new(1),
7809 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
7810 )
7811 .unwrap();
7812 let ack = parse_ack(&responses[0]);
7813 assert_eq!(
7814 ack.storage,
7815 Some(serde_json::json!({
7816 "module_id": "alfonso-routing",
7817 "storage_namespace": "default",
7818 "isolation": { "kind": "module" },
7819 "backend": {
7820 "backend": "sqlite",
7821 "path": "/data/cortexkit/alfonso-routing/store.db"
7822 }
7823 })),
7824 "the delivered descriptor is the module's own sqlite store path"
7825 );
7826 }
7827
7828 #[test]
7829 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
7830 let registry = Arc::new(Registry::default());
7831 let handler = ControlHandler::new(Arc::clone(®istry));
7832 let conn = ConnectionId::new(1);
7833 let responses = handler
7834 .handle_control(
7835 conn,
7836 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
7837 )
7838 .unwrap();
7839 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
7840 let registration = registry.get_module("aft").unwrap().unwrap();
7841 assert_eq!(registration.control_ops, module_baseline_control_ops());
7842
7843 let frame =
7844 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
7845 assert!(handler
7846 .guard_module_control_op(&frame, "aft", "route.bind")
7847 .unwrap()
7848 .is_none());
7849 let error = handler
7850 .guard_module_control_op(&frame, "aft", "test.synthetic")
7851 .unwrap()
7852 .expect("synthetic ungranted op should be rejected");
7853 assert_eq!(error.header.ty, FrameType::Error);
7854 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
7855 }
7856
7857 #[test]
7858 fn hello_control_ops_some_adds_optional_grants() {
7859 let registry = Arc::new(Registry::default());
7860 let handler = ControlHandler::new(Arc::clone(®istry));
7861 handler
7862 .handle_control(
7863 ConnectionId::new(1),
7864 hello_frame_with_control_ops(
7865 "aft",
7866 PROTOCOL_VERSION,
7867 7,
7868 Some(vec![
7869 "future.synthetic".to_string(),
7870 "route.bind".to_string(),
7871 ]),
7872 ),
7873 )
7874 .unwrap();
7875 let registration = registry.get_module("aft").unwrap().unwrap();
7876 assert_eq!(
7877 registration.control_ops,
7878 vec![
7879 "route.bind".to_string(),
7880 "route.status".to_string(),
7881 "future.synthetic".to_string(),
7882 ]
7883 );
7884 let frame =
7885 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
7886 assert!(handler
7887 .guard_module_control_op(&frame, "aft", "future.synthetic")
7888 .unwrap()
7889 .is_none());
7890 }
7891
7892 #[tokio::test]
7893 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
7894 let registry = Arc::new(Registry::default());
7895 let forwarding = Arc::new(ForwardingTable::default());
7896 let handler =
7897 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
7898 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
7899 hello_via_sink(
7900 &handler,
7901 &module_ctx,
7902 &mut module_rx,
7903 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
7904 )
7905 .await;
7906
7907 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
7908 let responses = handler
7909 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
7910 .await
7911 .unwrap();
7912 assert_eq!(responses.len(), 1);
7913 assert_eq!(responses[0].header.ty, FrameType::Error);
7914 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
7915 assert!(module_rx.try_recv().is_err());
7916 }
7917
7918 #[tokio::test]
7919 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
7920 let registry = Arc::new(Registry::default());
7921 let forwarding = Arc::new(ForwardingTable::default());
7922 let handler =
7923 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
7924 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
7925 hello_via_sink(
7926 &handler,
7927 &module_ctx,
7928 &mut module_rx,
7929 hello_frame_with_control_ops(
7930 "aft",
7931 PROTOCOL_VERSION,
7932 7,
7933 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
7934 ),
7935 )
7936 .await;
7937
7938 let project_root = unique_project_root("demux");
7939 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
7940 let route_handler = handler.clone();
7941 let route_task = tokio::spawn(async move {
7942 route_handler
7943 .handle_control_frame(
7944 &route_client_ctx,
7945 route_open_frame(100, "aft", project_root),
7946 )
7947 .await
7948 .unwrap()
7949 });
7950 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7951 .await
7952 .unwrap()
7953 .unwrap();
7954 assert!(matches!(
7955 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
7956 ModuleControlRequest::RouteBind { .. }
7957 ));
7958
7959 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
7960 let health_handler = handler.clone();
7961 let health_task = tokio::spawn(async move {
7962 health_handler
7963 .handle_control_frame(
7964 &health_client_ctx,
7965 supervisor_health_probe_frame(101, "aft"),
7966 )
7967 .await
7968 .unwrap()
7969 });
7970 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7971 .await
7972 .unwrap()
7973 .unwrap();
7974 assert_eq!(
7975 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
7976 ModuleControlRequest::HealthCheck {}
7977 );
7978
7979 handler
7980 .handle_control_frame(
7981 &module_ctx,
7982 health_response(health_frame.header.corr, HealthStatus::Degraded),
7983 )
7984 .await
7985 .unwrap();
7986 let health_response = health_task.await.unwrap();
7987 assert_eq!(health_response.len(), 1);
7988 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
7989 ClientControlResponse::SupervisorHealthProbe {
7990 module_id,
7991 status,
7992 detail,
7993 metrics,
7994 } => {
7995 assert_eq!(module_id, "aft");
7996 assert_eq!(status, HealthStatus::Degraded);
7997 assert_eq!(detail.as_deref(), Some("warming"));
7998 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
7999 }
8000 other => panic!("unexpected health response: {other:?}"),
8001 }
8002
8003 handler
8004 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8005 .await
8006 .unwrap();
8007 let route_response = route_task.await.unwrap();
8008 assert!(route_response.is_empty());
8009 let published = route_client_rx.recv().await.unwrap();
8010 assert!(matches!(
8011 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8012 ClientControlResponse::RouteOpen { .. }
8013 ));
8014 }
8015
8016 async fn relay_route_open(
8021 handler: &ControlHandler,
8022 client_connection: ConnectionId,
8023 client_egress: &FrameSink,
8024 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
8025 corr: u64,
8026 module_id: &str,
8027 project_root_label: &str,
8028 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
8029 let ctx = RouteCtx {
8030 connection_id: client_connection,
8031 egress: client_egress.clone(),
8032 };
8033 let handler = handler.clone();
8034 let project_root = unique_project_root(project_root_label);
8035 let module_id = module_id.to_string();
8036 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
8037 let task = tokio::spawn(async move {
8038 let _guard = tracing::dispatcher::set_default(&dispatch);
8039 handler
8040 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
8041 .await
8042 .unwrap()
8043 });
8044 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
8045 .await
8046 .expect("module receives the relayed route.bind")
8047 .expect("module egress is open");
8048 (task, bind.frame)
8049 }
8050
8051 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
8052 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
8053 ModuleControlRequest::RouteBind {
8054 route_channel,
8055 epoch,
8056 ..
8057 } => (route_channel, epoch),
8058 other => panic!("expected a route.bind request, got {other:?}"),
8059 }
8060 }
8061
8062 fn published_route(frame: &Frame) -> (u16, u32) {
8063 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
8064 ClientControlResponse::RouteOpen {
8065 route_channel,
8066 route_epoch,
8067 } => (route_channel, route_epoch),
8068 other => panic!("expected a route.open response, got {other:?}"),
8069 }
8070 }
8071
8072 #[tokio::test]
8092 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
8093 let registry = Arc::new(Registry::default());
8094 let forwarding = Arc::new(ForwardingTable::default());
8095 let handler =
8096 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8097
8098 let module_connection = ConnectionId::new(30);
8099 let (module_ctx, mut module_rx) = route_ctx(module_connection);
8100 hello_via_sink(
8101 &handler,
8102 &module_ctx,
8103 &mut module_rx,
8104 hello_frame("aft", PROTOCOL_VERSION, 7),
8105 )
8106 .await;
8107
8108 let dying_client = ConnectionId::new(31);
8109 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
8110
8111 let (first_task, first_bind) = relay_route_open(
8114 &handler,
8115 dying_client,
8116 &dying_ctx.egress,
8117 &mut module_rx,
8118 100,
8119 "aft",
8120 "closing-first",
8121 )
8122 .await;
8123 handler
8124 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
8125 .await
8126 .unwrap();
8127 assert!(first_task.await.unwrap().is_empty());
8128 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
8129
8130 let (second_task, second_bind) = relay_route_open(
8132 &handler,
8133 dying_client,
8134 &dying_ctx.egress,
8135 &mut module_rx,
8136 101,
8137 "aft",
8138 "closing-second",
8139 )
8140 .await;
8141 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
8142
8143 assert!(forwarding
8146 .escalate_client_delivery_failure(
8147 dying_client,
8148 first_channel,
8149 first_epoch,
8150 CloseReason::new(
8151 "module_to_client_delivery_failed",
8152 "client egress refused a module frame",
8153 ),
8154 crate::forwarding::UndeliveredFrame {
8155 module_id: None,
8156 sink: &dying_ctx.egress,
8157 },
8158 )
8159 .unwrap());
8160 assert!(!dying_ctx.egress.is_closed());
8161
8162 let ack = handler
8164 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
8165 .await;
8166 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
8167 if module_loop_error.is_some() {
8168 handler.cleanup_connection(module_connection).unwrap();
8172 }
8173 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8177 .await
8178 .ok()
8179 .flatten();
8180
8181 assert!(
8183 registry
8184 .get_module_by_connection(module_connection)
8185 .unwrap()
8186 .is_some(),
8187 "one client's closing connection ended the shared module connection: \
8188 {module_loop_error:?}"
8189 );
8190 let cotenant = ConnectionId::new(32);
8192 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
8193 let (cotenant_task, cotenant_bind) = relay_route_open(
8194 &handler,
8195 cotenant,
8196 &cotenant_ctx.egress,
8197 &mut module_rx,
8198 102,
8199 "aft",
8200 "closing-cotenant",
8201 )
8202 .await;
8203 handler
8204 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
8205 .await
8206 .unwrap();
8207 assert!(cotenant_task.await.unwrap().is_empty());
8208 let (cotenant_channel, cotenant_epoch) =
8209 published_route(&cotenant_rx.recv().await.unwrap());
8210 assert!(matches!(
8211 forwarding
8212 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
8213 .unwrap(),
8214 DataRoute::Client(DataRouteState::Bound(_))
8215 ));
8216
8217 let goodbye = post_ack_module_frame
8220 .expect("module receives a GOODBYE for the abandoned route channel");
8221 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
8222 assert_eq!(goodbye.header.channel, abandoned_channel);
8223 assert_eq!(goodbye.header.epoch, abandoned_epoch);
8224
8225 assert!(dying_rx.try_recv().is_err());
8229 let second_response = second_task.await.unwrap();
8230 assert_eq!(second_response.len(), 1);
8231 assert_eq!(
8232 parse_error(&second_response[0])["code"],
8233 "target_unavailable"
8234 );
8235 }
8236
8237 #[test]
8244 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
8245 let handler = ControlHandler::default();
8246 let module_connection = ConnectionId::new(30);
8247 let client_connection = ConnectionId::new(31);
8248
8249 handler
8250 .refuse_to_end_module_connection_for_a_client(
8251 module_connection,
8252 77,
8253 ForwardingError::ConnectionClosing {
8254 connection_id: client_connection,
8255 },
8256 )
8257 .expect("a closing client must never end the module connection");
8258
8259 assert!(matches!(
8260 handler.refuse_to_end_module_connection_for_a_client(
8261 module_connection,
8262 78,
8263 ForwardingError::ConnectionClosing {
8264 connection_id: module_connection,
8265 },
8266 ),
8267 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
8268 connection_id
8269 })) if connection_id == module_connection
8270 ));
8271 assert!(matches!(
8272 handler.refuse_to_end_module_connection_for_a_client(
8273 module_connection,
8274 79,
8275 ForwardingError::Poisoned,
8276 ),
8277 Err(RouterError::Forwarding(ForwardingError::Poisoned))
8278 ));
8279 assert!(matches!(
8280 handler.refuse_to_end_module_connection_for_a_client(
8281 module_connection,
8282 80,
8283 ForwardingError::StaleModuleEndpoint,
8284 ),
8285 Err(RouterError::Forwarding(
8286 ForwardingError::StaleModuleEndpoint
8287 ))
8288 ));
8289 }
8290
8291 #[tokio::test]
8304 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
8305 let registry = Arc::new(Registry::default());
8306 let forwarding = Arc::new(ForwardingTable::default());
8307 let supervisor = SupervisorHandle::new();
8308 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8309 let handler =
8310 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8311 .with_supervisor(supervisor);
8312
8313 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
8314 hello_via_sink(
8315 &handler,
8316 &target_ctx,
8317 &mut target_rx,
8318 hello_frame("target", PROTOCOL_VERSION, 1),
8319 )
8320 .await;
8321
8322 let wrong_nonce = handler
8326 .handle_control_frame(
8327 &route_ctx(ConnectionId::new(91)).0,
8328 route_open_frame_with_admission_facts(
8329 20,
8330 "target",
8331 unique_project_root("admission-facts"),
8332 Some(subc_control::ConsumerIdentity {
8333 module_id: "fed".to_string(),
8334 launch_nonce: "not-the-real-nonce".to_string(),
8335 }),
8336 None,
8337 ),
8338 )
8339 .await
8340 .unwrap();
8341 assert_eq!(
8342 parse_error(&wrong_nonce[0])["code"],
8343 "bad_consumer_identity",
8344 "a mismatched launch nonce must be refused, not stamped Reserved"
8345 );
8346
8347 let never_spawned = handler
8351 .handle_control_frame(
8352 &route_ctx(ConnectionId::new(92)).0,
8353 route_open_frame_with_admission_facts(
8354 21,
8355 "target",
8356 unique_project_root("admission-facts"),
8357 Some(subc_control::ConsumerIdentity {
8358 module_id: "never-spawned".to_string(),
8359 launch_nonce: "any-nonce".to_string(),
8360 }),
8361 None,
8362 ),
8363 )
8364 .await
8365 .unwrap();
8366 assert_eq!(
8367 parse_error(&never_spawned[0])["code"],
8368 "bad_consumer_identity",
8369 "an unspawned module_id must be refused rather than accepted for lack of a record"
8370 );
8371 }
8372
8373 #[tokio::test]
8396 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
8397 let registry = Arc::new(Registry::default());
8398 let forwarding = Arc::new(ForwardingTable::default());
8399 let supervisor = SupervisorHandle::new();
8400 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8401 let handler =
8402 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8403 .with_supervisor(supervisor);
8404
8405 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
8406 hello_via_sink(
8407 &handler,
8408 &target_ctx,
8409 &mut target_rx,
8410 hello_frame("target", PROTOCOL_VERSION, 1),
8411 )
8412 .await;
8413
8414 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
8415 let route_handler = handler.clone();
8416 let route_task = tokio::spawn(async move {
8417 route_handler
8418 .handle_control_frame(
8419 &client_ctx,
8420 route_open_frame_with_admission_facts(
8421 30,
8422 "target",
8423 unique_project_root("admission-facts"),
8424 Some(subc_control::ConsumerIdentity {
8425 module_id: "fed".to_string(),
8426 launch_nonce: "fed-nonce".to_string(),
8427 }),
8428 None,
8429 ),
8430 )
8431 .await
8432 .unwrap()
8433 });
8434
8435 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8442 .await
8443 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
8444 .expect("module control channel closed before route.bind");
8445 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8446 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
8447 panic!("expected route.bind")
8448 };
8449 assert_eq!(
8450 principal,
8451 Some(Principal::Reserved {
8452 module_id: "fed".to_string()
8453 }),
8454 "a correctly attested consumer must be stamped Reserved for its own id"
8455 );
8456
8457 handler
8458 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8459 .await
8460 .unwrap();
8461 assert!(route_task.await.unwrap().is_empty());
8462 assert!(
8463 matches!(
8464 serde_json::from_slice::<ClientControlResponse>(
8465 &client_rx.recv().await.unwrap().body
8466 )
8467 .unwrap(),
8468 ClientControlResponse::RouteOpen { .. }
8469 ),
8470 "the route must actually open, not merely avoid an error"
8471 );
8472 }
8473
8474 #[tokio::test(start_paused = true)]
8475 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
8476 let registry = Arc::new(Registry::default());
8477 let forwarding = Arc::new(ForwardingTable::default());
8478 let supervisor = SupervisorHandle::new();
8479 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8480 let handler =
8481 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8482 .with_supervisor(supervisor);
8483
8484 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
8485 hello_via_sink(
8486 &handler,
8487 &target_ctx,
8488 &mut target_rx,
8489 hello_frame("target", PROTOCOL_VERSION, 1),
8490 )
8491 .await;
8492
8493 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
8494 let direct_handler = handler.clone();
8495 let direct_open = tokio::spawn(async move {
8496 direct_handler
8497 .handle_control_frame(
8498 &direct_ctx,
8499 route_open_frame(2, "target", unique_project_root("route-census-direct")),
8500 )
8501 .await
8502 .unwrap()
8503 });
8504 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8505 .await
8506 .expect("no direct route.bind within 5s")
8507 .expect("target control channel closed before direct route.bind");
8508 handler
8509 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
8510 .await
8511 .unwrap();
8512 assert!(direct_open.await.unwrap().is_empty());
8513 let _ = direct_rx.recv().await.unwrap();
8514
8515 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
8516 let reserved_handler = handler.clone();
8517 let reserved_open = tokio::spawn(async move {
8518 reserved_handler
8519 .handle_control_frame(
8520 &reserved_ctx,
8521 route_open_frame_with_admission_facts(
8522 3,
8523 "target",
8524 unique_project_root("admission-facts"),
8525 Some(ConsumerIdentity {
8526 module_id: "fed".to_string(),
8527 launch_nonce: "fed-nonce".to_string(),
8528 }),
8529 None,
8530 ),
8531 )
8532 .await
8533 .unwrap()
8534 });
8535 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8536 .await
8537 .expect("no reserved route.bind within 5s")
8538 .expect("target control channel closed before reserved route.bind");
8539 handler
8540 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
8541 .await
8542 .unwrap();
8543 assert!(reserved_open.await.unwrap().is_empty());
8544 let _ = reserved_rx.recv().await.unwrap();
8545
8546 forwarding
8547 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
8548 .unwrap();
8549 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
8550 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
8551 module_id: Some("target".to_string()),
8552 })
8553 .unwrap();
8554 let census_frame =
8555 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
8556 let response = handler
8557 .handle_control_frame(&census_ctx, census_frame)
8558 .await
8559 .unwrap()
8560 .pop()
8561 .unwrap();
8562 let actual: Value = serde_json::from_slice(&response.body).unwrap();
8563 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
8564 assert!(matches!(
8565 decoded,
8566 ClientControlResponse::SupervisorRoutes { .. }
8567 ));
8568 let routes = actual["modules"][0]["routes"].as_array().unwrap();
8569 assert_eq!(routes.len(), 2);
8570 assert!(routes.iter().all(|route| route["draining"] == true));
8571 assert!(
8574 routes.iter().all(|route| route["drain_reason"] == "reload"),
8575 "draining routes must name the drain's reason: {routes:?}"
8576 );
8577 assert!(routes.iter().any(|route| {
8578 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
8579 }));
8580 assert!(routes.iter().any(|route| {
8581 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
8582 }));
8583
8584 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8585 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
8586 if std::env::var_os("UPDATE_GOLDEN").is_some() {
8587 std::fs::write(
8588 &golden_path,
8589 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
8590 )
8591 .unwrap();
8592 }
8593 let expected: Value =
8594 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
8595 assert_eq!(actual, expected);
8596 }
8597
8598 async fn query_live_roots(
8599 handler: &ControlHandler,
8600 module_ctx: &RouteCtx,
8601 ) -> ModuleControlResponseToModule {
8602 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
8603 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
8604 let response = handler
8605 .handle_control_frame(module_ctx, frame)
8606 .await
8607 .unwrap()
8608 .pop()
8609 .unwrap();
8610 serde_json::from_slice(&response.body).unwrap()
8611 }
8612
8613 #[tokio::test(start_paused = true)]
8614 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
8615 let registry = Arc::new(Registry::default());
8616 let forwarding = Arc::new(ForwardingTable::default());
8617 let handler = ControlHandler::with_forwarding(registry, forwarding);
8618 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
8619 hello_via_sink(
8620 &handler,
8621 &target_ctx,
8622 &mut target_rx,
8623 hello_frame("target", PROTOCOL_VERSION, 1),
8624 )
8625 .await;
8626 let root = unique_project_root("live-roots-known");
8627 let path = ProjectRootId::from_path_allowing_missing(root.path())
8628 .unwrap()
8629 .as_path()
8630 .to_path_buf();
8631 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
8632 let open_handler = handler.clone();
8633 let opened = tokio::spawn(async move {
8634 open_handler
8635 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
8636 .await
8637 .unwrap()
8638 });
8639 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8640 .await
8641 .unwrap()
8642 .unwrap();
8643 handler
8644 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
8645 .await
8646 .unwrap();
8647 assert!(opened.await.unwrap().is_empty());
8648 let _ = client_rx.recv().await.unwrap();
8649
8650 let root = unique_project_root("live-roots-pending");
8651 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
8652 .unwrap()
8653 .as_path()
8654 .to_path_buf();
8655 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
8656 let open_handler = handler.clone();
8657 let pending = tokio::spawn(async move {
8658 open_handler
8659 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
8660 .await
8661 .unwrap()
8662 });
8663 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8664 .await
8665 .unwrap()
8666 .unwrap();
8667 let actual = query_live_roots(&handler, &target_ctx).await;
8668 let ModuleControlResponseToModule::LiveRoots {
8669 roots,
8670 unknown_root_bindings,
8671 total_bindings,
8672 } = actual
8673 else {
8674 panic!("expected live roots")
8675 };
8676 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
8677 assert_eq!(unknown_root_bindings, 0);
8678 assert_eq!(
8679 roots.len(),
8680 2,
8681 "root-known arm must retain each canonical root"
8682 );
8683 assert_eq!(
8684 total_bindings,
8685 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
8686 );
8687 let counts = roots
8688 .iter()
8689 .map(|root| (root.project_root.clone(), root.bound, root.pending))
8690 .collect::<Vec<_>>();
8691 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
8692 expected.sort_by(|a, b| a.0.cmp(&b.0));
8693 assert_eq!(
8694 counts, expected,
8695 "roots must sort by path and count pending separately"
8696 );
8697 handler
8698 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
8699 .await
8700 .unwrap();
8701 assert!(pending.await.unwrap().is_empty());
8702 }
8703
8704 #[tokio::test(start_paused = true)]
8705 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
8706 let forwarding = Arc::new(ForwardingTable::default());
8707 let handler =
8708 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
8709 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
8710 hello_via_sink(
8711 &handler,
8712 &target_ctx,
8713 &mut target_rx,
8714 hello_frame("target", PROTOCOL_VERSION, 1),
8715 )
8716 .await;
8717 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
8718 let pending = forwarding
8719 .begin_route_bind_relay_for_test(
8720 client_ctx.connection_id,
8721 client_ctx.egress.clone(),
8722 2,
8723 "target",
8724 )
8725 .unwrap();
8726 forwarding
8727 .complete_pending_relay(
8728 target_ctx.connection_id,
8729 pending.corr,
8730 RouteBindRelayOutcome::Accepted,
8731 )
8732 .unwrap();
8733 let actual = query_live_roots(&handler, &target_ctx).await;
8734 let ModuleControlResponseToModule::LiveRoots {
8735 roots,
8736 unknown_root_bindings,
8737 total_bindings,
8738 } = actual
8739 else {
8740 panic!("expected live roots")
8741 };
8742 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
8743 assert_eq!(
8744 unknown_root_bindings, 1,
8745 "unknown-root arm must not read as no bindings"
8746 );
8747 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
8748 assert_eq!(
8749 total_bindings,
8750 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
8751 );
8752 }
8753
8754 #[tokio::test(start_paused = true)]
8762 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
8763 let forwarding = Arc::new(ForwardingTable::default());
8764 let handler =
8765 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
8766 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
8767 let replies = handler
8768 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
8769 .await
8770 .unwrap();
8771 let queued_by_hello = module_rx.len();
8772
8773 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
8774 let open_handler = handler.clone();
8775 let open = tokio::spawn(async move {
8776 open_handler
8777 .handle_control_frame(
8778 &client_ctx,
8779 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
8780 )
8781 .await
8782 .unwrap()
8783 });
8784 let mut spins = 0;
8786 while module_rx.len() == queued_by_hello {
8787 spins += 1;
8788 assert!(spins < 10_000, "route.open never queued a route.bind");
8789 tokio::task::yield_now().await;
8790 }
8791
8792 for reply in replies {
8794 module_ctx.egress.send(reply).await.unwrap();
8795 }
8796
8797 let first = module_rx.recv().await.unwrap().frame;
8798 assert_eq!(
8799 first.header.ty,
8800 FrameType::HelloAck,
8801 "the first frame a registering module reads must be its HELLO_ACK"
8802 );
8803 assert_eq!(first.header.corr, 7);
8804 let second = module_rx.recv().await.unwrap().frame;
8805 assert_eq!(second.header.ty, FrameType::Request);
8806 assert!(
8807 matches!(
8808 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
8809 ModuleControlRequest::RouteBind { .. }
8810 ),
8811 "the route.bind follows the ack"
8812 );
8813 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
8814
8815 handler
8816 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
8817 .await
8818 .unwrap();
8819 assert!(open.await.unwrap().is_empty());
8820 let _ = client_rx.recv().await.unwrap();
8821 }
8822
8823 #[tokio::test(start_paused = true)]
8824 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
8825 let handler = ControlHandler::with_forwarding(
8826 Arc::new(Registry::default()),
8827 Arc::new(ForwardingTable::default()),
8828 );
8829 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
8830 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
8831 hello_via_sink(
8832 &handler,
8833 &first_ctx,
8834 &mut first_rx,
8835 hello_frame("first", PROTOCOL_VERSION, 1),
8836 )
8837 .await;
8838 hello_via_sink(
8839 &handler,
8840 &second_ctx,
8841 &mut second_rx,
8842 hello_frame("second", PROTOCOL_VERSION, 2),
8843 )
8844 .await;
8845 let root = unique_project_root("second-only");
8846 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
8847 let cloned = handler.clone();
8848 let open = tokio::spawn(async move {
8849 cloned
8850 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
8851 .await
8852 .unwrap()
8853 });
8854 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
8855 .await
8856 .unwrap()
8857 .unwrap();
8858 let first = query_live_roots(&handler, &first_ctx).await;
8859 let second = query_live_roots(&handler, &second_ctx).await;
8860 assert!(
8861 matches!(
8862 first,
8863 ModuleControlResponseToModule::LiveRoots {
8864 total_bindings: 0,
8865 ..
8866 }
8867 ),
8868 "cross-module scope must not expose another module's roots"
8869 );
8870 assert!(
8871 matches!(
8872 second,
8873 ModuleControlResponseToModule::LiveRoots {
8874 total_bindings: 1,
8875 ..
8876 }
8877 ),
8878 "second module must see its pending route"
8879 );
8880 handler
8881 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
8882 .await
8883 .unwrap();
8884 assert!(open.await.unwrap().is_empty());
8885 }
8886
8887 #[tokio::test(start_paused = true)]
8888 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
8889 let handler = ControlHandler::with_forwarding(
8890 Arc::new(Registry::default()),
8891 Arc::new(ForwardingTable::default()),
8892 );
8893 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
8894 hello_via_sink(
8895 &handler,
8896 &target_ctx,
8897 &mut target_rx,
8898 hello_frame("target", PROTOCOL_VERSION, 1),
8899 )
8900 .await;
8901 let actual = query_live_roots(&handler, &target_ctx).await;
8902 let ModuleControlResponseToModule::LiveRoots {
8903 roots,
8904 unknown_root_bindings,
8905 total_bindings,
8906 } = actual
8907 else {
8908 panic!("expected live roots")
8909 };
8910 assert!(roots.is_empty());
8911 assert_eq!(unknown_root_bindings, 0);
8912 assert_eq!(total_bindings, 0);
8913 assert_eq!(
8914 total_bindings,
8915 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
8916 );
8917 }
8918
8919 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
8926 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8927 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
8928 let text = std::fs::read_to_string(&path)
8929 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
8930 let vectors: Vec<(String, Value)> = text
8931 .lines()
8932 .filter(|line| !line.trim().is_empty())
8933 .map(|line| {
8934 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
8935 let id = entry["corpus_id"]
8936 .as_str()
8937 .expect("every vector carries a corpus_id")
8938 .to_string();
8939 (id, entry["package"].clone())
8940 })
8941 .collect();
8942 assert_eq!(
8946 vectors.len(),
8947 3,
8948 "vendored fed corpus changed size; re-sync from subc-federation"
8949 );
8950
8951 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
8966 let richest = vectors
8967 .iter()
8968 .filter_map(|(_, package)| package.as_object())
8969 .map(|object| {
8970 object
8971 .keys()
8972 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
8973 .count()
8974 })
8975 .max()
8976 .unwrap_or(0);
8977 assert!(
8978 richest >= 2,
8979 "vendored corpus no longer carries a package with unmodelled fields, \
8980 so the relay test can no longer distinguish a verbatim relay from a lossy one"
8981 );
8982
8983 vectors
8984 }
8985
8986 #[tokio::test]
8994 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
8995 for (corpus_id, package) in fed_admission_facts_vectors() {
8996 let registry = Arc::new(Registry::default());
8997 let forwarding = Arc::new(ForwardingTable::default());
8998 let supervisor = SupervisorHandle::new();
8999 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9000 let handler =
9001 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9002 .with_supervisor(supervisor)
9003 .with_admission_facts_config(
9004 Some("fed".to_string()),
9005 Some(vec!["target".to_string()]),
9006 );
9007
9008 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
9009 hello_via_sink(
9010 &handler,
9011 &target_ctx,
9012 &mut target_rx,
9013 hello_frame("target", PROTOCOL_VERSION, 1),
9014 )
9015 .await;
9016
9017 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
9018 let route_handler = handler.clone();
9019 let expected = package.clone();
9020 let route_task = tokio::spawn(async move {
9021 route_handler
9022 .handle_control_frame(
9023 &client_ctx,
9024 route_open_frame_with_admission_facts(
9025 20,
9026 "target",
9027 unique_project_root("admission-facts"),
9028 Some(subc_control::ConsumerIdentity {
9029 module_id: "fed".to_string(),
9030 launch_nonce: "fed-nonce".to_string(),
9031 }),
9032 Some(package),
9033 ),
9034 )
9035 .await
9036 .unwrap()
9037 });
9038
9039 let bind_frame = target_rx.recv().await.unwrap();
9040 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9041 let ModuleControlRequest::RouteBind {
9042 admission_facts, ..
9043 } = bind
9044 else {
9045 panic!("{corpus_id}: expected route.bind")
9046 };
9047 assert_eq!(
9048 admission_facts,
9049 Some(expected),
9050 "{corpus_id}: relay must not add, drop or reshape any field"
9051 );
9052
9053 handler
9054 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
9055 .await
9056 .unwrap();
9057 route_task.await.unwrap();
9058 }
9059 }
9060
9061 #[tokio::test]
9062 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
9063 let registry = Arc::new(Registry::default());
9064 let forwarding = Arc::new(ForwardingTable::default());
9065 let supervisor = SupervisorHandle::new();
9066 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9067 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
9068 let handler =
9069 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9070 .with_supervisor(supervisor)
9071 .with_admission_facts_config(
9072 Some("fed".to_string()),
9073 Some(vec!["target".to_string()]),
9074 );
9075
9076 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
9077 hello_via_sink(
9078 &handler,
9079 &target_ctx,
9080 &mut target_rx,
9081 hello_frame("target", PROTOCOL_VERSION, 1),
9082 )
9083 .await;
9084 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
9085 hello_via_sink(
9086 &handler,
9087 &other_ctx,
9088 &mut other_rx,
9089 hello_frame("other", PROTOCOL_VERSION, 2),
9090 )
9091 .await;
9092
9093 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
9094 let expected_facts = facts.clone();
9095 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
9096 let route_handler = handler.clone();
9097 let route_task = tokio::spawn(async move {
9098 route_handler
9099 .handle_control_frame(
9100 &client_ctx,
9101 route_open_frame_with_admission_facts(
9102 10,
9103 "target",
9104 unique_project_root("admission-facts"),
9105 Some(subc_control::ConsumerIdentity {
9106 module_id: "fed".to_string(),
9107 launch_nonce: "fed-nonce".to_string(),
9108 }),
9109 Some(facts.clone()),
9110 ),
9111 )
9112 .await
9113 .unwrap()
9114 });
9115 let bind_frame = target_rx.recv().await.unwrap();
9116 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9117 let ModuleControlRequest::RouteBind {
9118 admission_facts, ..
9119 } = bind
9120 else {
9121 panic!("expected route.bind")
9122 };
9123 assert_eq!(admission_facts, Some(expected_facts));
9124 handler
9125 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
9126 .await
9127 .unwrap();
9128 assert!(route_task.await.unwrap().is_empty());
9129 assert!(matches!(
9130 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
9131 .unwrap(),
9132 ClientControlResponse::RouteOpen { .. }
9133 ));
9134
9135 let direct = handler
9136 .handle_control_frame(
9137 &route_ctx(ConnectionId::new(73)).0,
9138 route_open_frame_with_admission_facts(
9139 11,
9140 "target",
9141 unique_project_root("admission-facts"),
9142 None,
9143 Some(json!({"x": 1})),
9144 ),
9145 )
9146 .await
9147 .unwrap();
9148 assert_eq!(
9149 parse_error(&direct[0])["code"],
9150 "admission_facts_not_permitted"
9151 );
9152
9153 let different_reserved = handler
9154 .handle_control_frame(
9155 &route_ctx(ConnectionId::new(77)).0,
9156 route_open_frame_with_admission_facts(
9157 15,
9158 "target",
9159 unique_project_root("admission-facts"),
9160 Some(subc_control::ConsumerIdentity {
9161 module_id: "other".to_string(),
9162 launch_nonce: "other-nonce".to_string(),
9163 }),
9164 Some(json!({"x": 1})),
9165 ),
9166 )
9167 .await
9168 .unwrap();
9169 assert_eq!(
9170 parse_error(&different_reserved[0])["code"],
9171 "admission_facts_not_permitted"
9172 );
9173
9174 let other_target = handler
9175 .handle_control_frame(
9176 &route_ctx(ConnectionId::new(74)).0,
9177 route_open_frame_with_admission_facts(
9178 12,
9179 "other",
9180 unique_project_root("admission-facts"),
9181 Some(subc_control::ConsumerIdentity {
9182 module_id: "fed".to_string(),
9183 launch_nonce: "fed-nonce".to_string(),
9184 }),
9185 Some(json!({"x": 1})),
9186 ),
9187 )
9188 .await
9189 .unwrap();
9190 assert_eq!(
9191 parse_error(&other_target[0])["code"],
9192 "admission_facts_target_not_allowed"
9193 );
9194
9195 let nonexistent = handler
9196 .handle_control_frame(
9197 &route_ctx(ConnectionId::new(75)).0,
9198 route_open_frame_with_admission_facts(
9199 13,
9200 "missing",
9201 unique_project_root("admission-facts"),
9202 None,
9203 Some(json!({"x": 1})),
9204 ),
9205 )
9206 .await
9207 .unwrap();
9208 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
9209
9210 let described = handler
9211 .handle_control_frame(
9212 &route_ctx(ConnectionId::new(76)).0,
9213 Frame::build(
9214 FrameType::Request,
9215 control_flags(),
9216 0,
9217 0,
9218 14,
9219 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
9220 )
9221 .unwrap(),
9222 )
9223 .await
9224 .unwrap();
9225 let ClientControlResponse::ServerDescribe { capabilities, .. } =
9226 serde_json::from_slice(&described[0].body).unwrap()
9227 else {
9228 panic!("expected server.describe response")
9229 };
9230 assert!(capabilities
9231 .iter()
9232 .any(|cap| cap == "admission_facts_relay_v1"));
9233 }
9234
9235 #[tokio::test]
9236 async fn admission_facts_without_configured_carrier_are_rejected() {
9237 let registry = Arc::new(Registry::default());
9238 let forwarding = Arc::new(ForwardingTable::default());
9239 let handler = ControlHandler::with_forwarding(registry, forwarding);
9240 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
9241 hello_via_sink(
9242 &handler,
9243 &target_ctx,
9244 &mut target_rx,
9245 hello_frame("target", PROTOCOL_VERSION, 1),
9246 )
9247 .await;
9248
9249 let responses = handler
9250 .handle_control_frame(
9251 &route_ctx(ConnectionId::new(79)).0,
9252 route_open_frame_with_admission_facts(
9253 16,
9254 "target",
9255 unique_project_root("admission-facts"),
9256 None,
9257 Some(json!({"x": 1})),
9258 ),
9259 )
9260 .await
9261 .unwrap();
9262 assert_eq!(
9263 parse_error(&responses[0])["code"],
9264 "admission_facts_not_permitted"
9265 );
9266 }
9267
9268 #[tokio::test]
9269 async fn route_open_relays_consumer_capabilities_verbatim() {
9270 let registry = Arc::new(Registry::default());
9271 let forwarding = Arc::new(ForwardingTable::default());
9272 let handler =
9273 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9274 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
9275 hello_via_sink(
9276 &handler,
9277 &module_ctx,
9278 &mut module_rx,
9279 hello_frame("aft", PROTOCOL_VERSION, 7),
9280 )
9281 .await;
9282
9283 let expected = vec!["elicitation".to_string(), "roots".to_string()];
9284 let expected_for_request = expected.clone();
9285 let project_root = unique_project_root("consumer-capabilities-present");
9286 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
9287 let route_handler = handler.clone();
9288 let route_task = tokio::spawn(async move {
9289 route_handler
9290 .handle_control_frame(
9291 &client_ctx,
9292 route_open_frame_with_consumer_capabilities(
9293 401,
9294 "aft",
9295 project_root,
9296 Some(expected_for_request),
9297 ),
9298 )
9299 .await
9300 .unwrap()
9301 });
9302 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9303 .await
9304 .unwrap()
9305 .unwrap();
9306 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9307 let ModuleControlRequest::RouteBind {
9308 consumer_capabilities,
9309 ..
9310 } = bind
9311 else {
9312 panic!("expected route.bind request, got {bind:?}");
9313 };
9314 assert_eq!(consumer_capabilities, Some(expected.clone()));
9315
9316 handler
9317 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9318 .await
9319 .unwrap();
9320 let route_response = route_task.await.unwrap();
9321 assert!(route_response.is_empty());
9322 let published = client_rx.recv().await.unwrap();
9323 assert!(matches!(
9324 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9325 ClientControlResponse::RouteOpen { .. }
9326 ));
9327 }
9328
9329 #[tokio::test]
9330 async fn route_open_without_consumer_capabilities_relays_none() {
9331 let registry = Arc::new(Registry::default());
9332 let forwarding = Arc::new(ForwardingTable::default());
9333 let handler =
9334 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9335 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
9336 hello_via_sink(
9337 &handler,
9338 &module_ctx,
9339 &mut module_rx,
9340 hello_frame("aft", PROTOCOL_VERSION, 7),
9341 )
9342 .await;
9343
9344 let project_root = unique_project_root("consumer-capabilities-absent");
9345 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
9346 let route_handler = handler.clone();
9347 let route_task = tokio::spawn(async move {
9348 route_handler
9349 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
9350 .await
9351 .unwrap()
9352 });
9353 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9354 .await
9355 .unwrap()
9356 .unwrap();
9357 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9358 let ModuleControlRequest::RouteBind {
9359 consumer_capabilities,
9360 ..
9361 } = bind
9362 else {
9363 panic!("expected route.bind request, got {bind:?}");
9364 };
9365 assert_eq!(consumer_capabilities, None);
9366
9367 handler
9368 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9369 .await
9370 .unwrap();
9371 let route_response = route_task.await.unwrap();
9372 assert!(route_response.is_empty());
9373 let published = client_rx.recv().await.unwrap();
9374 assert!(matches!(
9375 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9376 ClientControlResponse::RouteOpen { .. }
9377 ));
9378 }
9379
9380 async fn bind_role_versions_for(
9384 sent: Option<BTreeMap<String, String>>,
9385 connection: u64,
9386 ) -> Option<BTreeMap<String, String>> {
9387 let registry = Arc::new(Registry::default());
9388 let forwarding = Arc::new(ForwardingTable::default());
9389 let handler =
9390 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9391 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
9392 hello_via_sink(
9393 &handler,
9394 &module_ctx,
9395 &mut module_rx,
9396 hello_frame("aft", PROTOCOL_VERSION, 7),
9397 )
9398 .await;
9399 let project_root = unique_project_root("role-versions");
9400 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
9401 let route_handler = handler.clone();
9402 let route_task = tokio::spawn(async move {
9403 route_handler
9404 .handle_control_frame(
9405 &client_ctx,
9406 route_open_frame_with_role_versions(403, "aft", project_root, sent),
9407 )
9408 .await
9409 .unwrap()
9410 });
9411 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9412 .await
9413 .expect("a well-formed route.open reaches the module as a bind")
9414 .unwrap();
9415 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9416 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
9417 panic!("expected route.bind request, got {bind:?}");
9418 };
9419 handler
9420 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9421 .await
9422 .unwrap();
9423 assert!(route_task.await.unwrap().is_empty());
9424 let published = client_rx.recv().await.unwrap();
9425 assert!(matches!(
9426 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9427 ClientControlResponse::RouteOpen { .. }
9428 ));
9429 role_versions
9430 }
9431
9432 #[tokio::test]
9433 async fn route_open_relays_role_versions_verbatim() {
9434 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
9435 assert_eq!(
9436 bind_role_versions_for(Some(sent.clone()), 141).await,
9437 Some(sent)
9438 );
9439 }
9440
9441 #[tokio::test]
9444 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
9445 assert_eq!(bind_role_versions_for(None, 143).await, None);
9446 assert_eq!(
9447 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
9448 None
9449 );
9450 }
9451
9452 #[tokio::test]
9453 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
9454 let registry = Arc::new(Registry::default());
9455 let forwarding = Arc::new(ForwardingTable::default());
9456 let handler =
9457 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9458 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
9459 hello_via_sink(
9460 &handler,
9461 &module_ctx,
9462 &mut module_rx,
9463 hello_frame("aft", PROTOCOL_VERSION, 7),
9464 )
9465 .await;
9466
9467 let nine: BTreeMap<String, String> = (0..9)
9468 .map(|index| (format!("role-{index}"), "v1".to_string()))
9469 .collect();
9470 for (label, malformed) in [
9471 (
9472 "invalid role name",
9473 role_versions(&[("Tool_Provider", "v1")]),
9474 ),
9475 ("invalid version", role_versions(&[("tool-provider", "v0")])),
9476 ("nine entries", nine),
9477 ] {
9478 let responses = tokio::time::timeout(
9481 Duration::from_secs(1),
9482 handler.handle_control_frame(
9483 &route_ctx(ConnectionId::new(148)).0,
9484 route_open_frame_with_role_versions(
9485 404,
9486 "aft",
9487 unique_project_root("role-versions-malformed"),
9488 Some(malformed),
9489 ),
9490 ),
9491 )
9492 .await
9493 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
9494 .unwrap();
9495 assert_eq!(responses.len(), 1, "{label}");
9496 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
9497 let error = parse_error(&responses[0]);
9498 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
9499 assert_eq!(
9500 error["detail"]["field"], "role_versions",
9501 "{label}: {error}"
9502 );
9503 assert!(
9504 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
9505 "{label}: a malformed declaration is terminal"
9506 );
9507 assert!(
9508 module_rx.try_recv().is_err(),
9509 "{label}: the module must never see a bind"
9510 );
9511 }
9512 }
9513
9514 #[tokio::test]
9518 async fn route_role_versions_capability_is_advertised() {
9519 let handler = ControlHandler::new(Arc::new(Registry::default()));
9520 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
9521 let ack = hello_via_sink(
9522 &handler,
9523 &ctx,
9524 &mut rx,
9525 hello_frame("m", PROTOCOL_VERSION, 1),
9526 )
9527 .await;
9528 let ack = parse_ack(&ack);
9529 assert!(
9530 ack.subc_capabilities
9531 .iter()
9532 .any(|c| c == "route-role-versions/v1"),
9533 "{:?}",
9534 ack.subc_capabilities
9535 );
9536
9537 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
9538 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
9539 let reply = handler
9540 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
9541 .await
9542 .unwrap()
9543 .pop()
9544 .unwrap();
9545 let ClientControlResponse::ServerDescribe { capabilities, .. } =
9546 serde_json::from_slice(&reply.body).unwrap()
9547 else {
9548 panic!("not a server.describe reply");
9549 };
9550 assert!(
9551 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
9552 "{capabilities:?}"
9553 );
9554 }
9555
9556 #[tokio::test]
9557 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
9558 let registry = Arc::new(Registry::default());
9559 let forwarding = Arc::new(ForwardingTable::default());
9560 let handler =
9561 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9562 .with_health_probe_timeout(Duration::from_secs(5));
9563 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
9564 hello_via_sink(
9565 &handler,
9566 &module_ctx,
9567 &mut module_rx,
9568 non_routable_hello_frame_with_control_ops(
9569 "mcp",
9570 300,
9571 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9572 ),
9573 )
9574 .await;
9575 assert!(registry
9576 .get_module("mcp")
9577 .unwrap()
9578 .unwrap()
9579 .manifest
9580 .provides
9581 .is_empty());
9582
9583 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
9584 let route_response = handler
9585 .handle_control_frame(
9586 &route_client_ctx,
9587 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
9588 )
9589 .await
9590 .unwrap();
9591 assert_eq!(route_response[0].header.ty, FrameType::Error);
9592 assert_eq!(
9593 parse_error(&route_response[0])["code"],
9594 "target_unavailable"
9595 );
9596 assert!(parse_error(&route_response[0])["message"]
9597 .as_str()
9598 .unwrap()
9599 .contains("does not provide the requested target"));
9600 assert!(module_rx.try_recv().is_err());
9601
9602 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
9603 let health_handler = handler.clone();
9604 let health_task = tokio::spawn(async move {
9605 health_handler
9606 .handle_control_frame(
9607 &health_client_ctx,
9608 supervisor_health_probe_frame(302, "mcp"),
9609 )
9610 .await
9611 .unwrap()
9612 });
9613 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9614 .await
9615 .unwrap()
9616 .unwrap();
9617 assert_eq!(
9618 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
9619 ModuleControlRequest::HealthCheck {}
9620 );
9621 handler
9622 .handle_control_frame(
9623 &module_ctx,
9624 health_response(health_frame.header.corr, HealthStatus::Ok),
9625 )
9626 .await
9627 .unwrap();
9628 let health_response = health_task.await.unwrap();
9629 assert_eq!(health_response[0].header.ty, FrameType::Response);
9630 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
9631 ClientControlResponse::SupervisorHealthProbe {
9632 module_id, status, ..
9633 } => {
9634 assert_eq!(module_id, "mcp");
9635 assert_eq!(status, HealthStatus::Ok);
9636 }
9637 other => panic!("unexpected health response: {other:?}"),
9638 }
9639
9640 forwarding
9645 .cleanup_connection(module_ctx.connection_id)
9646 .unwrap();
9647 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
9648 let cleanup_response = tokio::time::timeout(
9649 Duration::from_millis(200),
9650 handler.handle_control_frame(
9651 &cleanup_probe_ctx,
9652 supervisor_health_probe_frame(303, "mcp"),
9653 ),
9654 )
9655 .await
9656 .expect("probe should fail immediately when the control lane is gone")
9657 .unwrap();
9658 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
9659 assert_eq!(
9660 parse_error(&cleanup_response[0])["code"],
9661 "target_unavailable"
9662 );
9663 assert!(parse_error(&cleanup_response[0])["message"]
9664 .as_str()
9665 .unwrap()
9666 .contains("no module connection"));
9667
9668 handler
9669 .cleanup_connection(module_ctx.connection_id)
9670 .unwrap();
9671 }
9672
9673 #[tokio::test]
9674 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
9675 let registry = Arc::new(Registry::default());
9676 let supervisor_handle = SupervisorHandle::new();
9677 let supervisor =
9678 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
9679 .with_handle(supervisor_handle.clone())
9680 .with_connection_file_path(
9681 std::env::temp_dir()
9682 .join(format!("subc-route-open-warming-{}", std::process::id())),
9683 );
9684 let module = supervisor
9685 .supervise_configured(
9686 ModuleSpec {
9687 module_id: "warming".to_string(),
9688 program: fake_aft_stub_path(),
9689 args: Vec::new(),
9690 env: Vec::new(),
9691 reserved: false,
9692 reserved_prefixes: Vec::new(),
9693 protocol: ModuleProtocol::Subc,
9694 overlap: Default::default(),
9695 },
9696 true,
9697 )
9698 .unwrap();
9699 assert_eq!(module.state().unwrap(), ModuleState::Running);
9700
9701 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9702 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
9703 let response = handler
9704 .handle_control_frame(
9705 &ctx,
9706 route_open_frame(304, "warming", unique_project_root("warming")),
9707 )
9708 .await
9709 .unwrap();
9710 module.stop().await.unwrap();
9711
9712 assert_eq!(response[0].header.ty, FrameType::Error);
9713 let error = parse_error(&response[0]);
9714 assert_eq!(error["code"], "module_warming");
9715 assert!(error["message"]
9716 .as_str()
9717 .unwrap()
9718 .contains("state=running, enabled=true, live=false"));
9719 }
9720
9721 #[test]
9722 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
9723 let handler = ControlHandler::new(Arc::new(Registry::default()));
9724 let capture = EventCapture::default();
9725 let _subscriber =
9726 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9727 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
9728 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
9729 let pending = (0..limit).collect::<Vec<_>>();
9730 let response = handler
9731 .route_open_capacity_refusal(
9732 &ctx,
9733 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
9734 "busy",
9735 pending.len(),
9736 limit,
9737 )
9738 .unwrap();
9739 assert_eq!(parse_error(&response)["code"], "target_unavailable");
9740 let event = capture
9741 .events()
9742 .into_iter()
9743 .find(|event| {
9744 event.target == "control"
9745 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
9746 })
9747 .expect("connection admission refusal event");
9748 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
9749 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
9750 }
9751
9752 #[test]
9753 fn route_open_target_cap_logs_admission_reason_and_capacity() {
9754 let handler = ControlHandler::new(Arc::new(Registry::default()));
9755 let capture = EventCapture::default();
9756 let _subscriber =
9757 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9758 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
9759 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
9760 let guards = (0..limit)
9761 .map(|_| {
9762 handler
9763 .route_bind_concurrency
9764 .try_admit("busy", limit)
9765 .unwrap()
9766 })
9767 .collect::<Vec<_>>();
9768 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
9769 Err(in_flight) => in_flight,
9770 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
9771 };
9772 let response = handler
9773 .route_open_target_capacity_refusal(
9774 &ctx,
9775 &route_open_frame(397, "busy", unique_project_root("target-cap")),
9776 "busy",
9777 in_flight,
9778 )
9779 .unwrap();
9780 assert_eq!(parse_error(&response)["code"], "target_unavailable");
9781 let event = capture
9782 .events()
9783 .into_iter()
9784 .find(|event| {
9785 event.target == "control"
9786 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
9787 })
9788 .expect("target admission refusal event");
9789 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
9790 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
9791 drop(guards);
9792 }
9793
9794 #[tokio::test]
9799 async fn route_open_refusal_names_the_check_that_refused() {
9800 let handler = ControlHandler::new(Arc::new(Registry::default()));
9801 let capture = EventCapture::default();
9802 let _subscriber =
9803 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9804 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
9805 let response = handler
9806 .handle_control_frame(
9807 &ctx,
9808 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
9809 )
9810 .await
9811 .unwrap();
9812
9813 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
9814 let event = capture
9815 .events()
9816 .into_iter()
9817 .find(|event| {
9818 event.target == "control"
9819 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
9820 })
9821 .expect("route.open refusal event");
9822 assert_eq!(
9823 event.fields.get("reason"),
9824 Some(&"\"not_registered\"".to_string())
9825 );
9826 }
9827
9828 #[tokio::test]
9829 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
9830 let registry = Arc::new(Registry::default());
9831 let supervisor_handle = SupervisorHandle::new();
9832 let supervisor =
9833 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
9834 .with_handle(supervisor_handle.clone())
9835 .with_connection_file_path(std::env::temp_dir().join(format!(
9836 "subc-route-open-refusal-info-{}",
9837 std::process::id()
9838 )));
9839 let module = supervisor
9840 .supervise_configured(
9841 ModuleSpec {
9842 module_id: "warming".to_string(),
9843 program: fake_aft_stub_path(),
9844 args: Vec::new(),
9845 env: Vec::new(),
9846 reserved: false,
9847 reserved_prefixes: Vec::new(),
9848 protocol: ModuleProtocol::Subc,
9849 overlap: Default::default(),
9850 },
9851 true,
9852 )
9853 .unwrap();
9854 assert_eq!(module.state().unwrap(), ModuleState::Running);
9855
9856 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9857 assert!(handler
9858 .counters()
9859 .snapshot()
9860 .get("route_open_refused_by_code")
9861 .is_none());
9862 let capture = EventCapture::default();
9863 let _subscriber =
9864 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9865 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
9866 let response = handler
9867 .handle_control_frame(
9868 &ctx,
9869 route_open_frame(394, "warming", unique_project_root("refusal-info")),
9870 )
9871 .await
9872 .unwrap();
9873 module.stop().await.unwrap();
9874
9875 assert_eq!(parse_error(&response[0])["code"], "module_warming");
9876 let event = capture
9877 .events()
9878 .into_iter()
9879 .find(|event| {
9880 event.target == "control"
9881 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
9882 })
9883 .expect("route.open refusal event");
9884 assert_eq!(
9885 event.fields.get("module_id"),
9886 Some(&"\"warming\"".to_string())
9887 );
9888 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
9889 assert_eq!(
9890 event.fields.get("reason"),
9891 Some(&"\"supervised_not_registered\"".to_string())
9892 );
9893 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
9894 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
9895 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
9896 assert_eq!(
9897 handler.counters().snapshot()["route_open_refused_by_code"],
9898 json!({ "module_warming": 1 })
9899 );
9900 }
9901
9902 const OUTAGE_START: &str = "route.open refusing module: not serving";
9903 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
9904
9905 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
9906 capture
9907 .events()
9908 .into_iter()
9909 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
9910 .collect()
9911 }
9912
9913 fn supervise_stub(
9914 registry: &Arc<Registry>,
9915 module_id: &str,
9916 enabled: bool,
9917 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
9918 let supervisor_handle = SupervisorHandle::new();
9919 let supervisor =
9920 Supervisor::new(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
9921 .with_handle(supervisor_handle.clone())
9922 .with_connection_file_path(std::env::temp_dir().join(format!(
9923 "subc-route-outage-{module_id}-{}",
9924 std::process::id()
9925 )));
9926 let module = supervisor
9927 .supervise_configured(
9928 ModuleSpec {
9929 module_id: module_id.to_string(),
9930 program: fake_aft_stub_path(),
9931 args: Vec::new(),
9932 env: Vec::new(),
9933 reserved: false,
9934 reserved_prefixes: Vec::new(),
9935 protocol: ModuleProtocol::Subc,
9936 overlap: Default::default(),
9937 },
9938 enabled,
9939 )
9940 .unwrap();
9941 (supervisor_handle, module)
9942 }
9943
9944 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
9945 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
9946 module_id: module_id.to_string(),
9947 drain_timeout_ms: Some(50),
9948 })
9949 .unwrap();
9950 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9951 }
9952
9953 #[test]
9957 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
9958 let registry = Arc::new(Registry::default());
9959 let forwarding = Arc::new(ForwardingTable::default());
9960 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9961 let second = ControlHandler::with_forwarding(registry, forwarding);
9962 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
9963 }
9964
9965 #[tokio::test(flavor = "current_thread")]
9969 async fn route_open_unknown_module_refusals_add_no_outage_state() {
9970 let handler = ControlHandler::new(Arc::new(Registry::default()));
9971 let capture = EventCapture::default();
9972 let _subscriber =
9973 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9974 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
9975 for corr in 0..8 {
9976 let response = handler
9977 .handle_control_frame(
9978 &ctx,
9979 route_open_frame(
9980 380 + corr,
9981 &format!("nobody-{corr}"),
9982 unique_project_root("outage-unknown"),
9983 ),
9984 )
9985 .await
9986 .unwrap();
9987 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
9988 }
9989
9990 assert_eq!(handler.route_outages.tracked_module_count(), 0);
9991 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
9992 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
9993 }
9994
9995 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
10000 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
10001 let registry = Arc::new(Registry::default());
10002 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
10003 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10004 let capture = EventCapture::default();
10005 let _subscriber =
10006 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10007 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
10008 handler.route_outages.record_accepted("outage-restart");
10011
10012 let response = handler
10013 .handle_control_frame(
10014 &ctx,
10015 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
10016 )
10017 .await
10018 .unwrap();
10019 assert_eq!(response[0].header.ty, FrameType::Error);
10020 let starts = outage_lines(&capture, OUTAGE_START);
10021 assert_eq!(starts.len(), 1, "{starts:?}");
10022 assert_eq!(starts[0].level, tracing::Level::WARN);
10023 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
10024 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
10025 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
10026 handler.route_outages.record_accepted("outage-restart");
10027 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
10028
10029 let restart = handler
10030 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
10031 .await
10032 .unwrap();
10033 assert_eq!(
10034 restart[0].header.ty,
10035 FrameType::Response,
10036 "{:?}",
10037 parse_error(&restart[0])
10038 );
10039 handler
10040 .handle_control_frame(
10041 &ctx,
10042 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
10043 )
10044 .await
10045 .unwrap();
10046 module.stop().await.unwrap();
10047
10048 let starts = outage_lines(&capture, OUTAGE_START);
10049 assert_eq!(starts.len(), 2, "{starts:?}");
10050 assert_eq!(starts[1].level, tracing::Level::INFO);
10051 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
10052 }
10053
10054 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
10058 async fn failed_operator_restart_leaves_no_operator_mark() {
10059 let registry = Arc::new(Registry::default());
10060 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
10061 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10062 let capture = EventCapture::default();
10063 let _subscriber =
10064 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10065 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
10066 handler.route_outages.record_accepted("outage-disabled");
10067
10068 let restart = handler
10069 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
10070 .await
10071 .unwrap();
10072 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
10073 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
10074
10075 handler
10076 .handle_control_frame(
10077 &ctx,
10078 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
10079 )
10080 .await
10081 .unwrap();
10082 let starts = outage_lines(&capture, OUTAGE_START);
10083 assert_eq!(starts.len(), 1, "{starts:?}");
10084 assert_eq!(starts[0].level, tracing::Level::WARN);
10085 }
10086
10087 #[tokio::test(flavor = "current_thread")]
10088 async fn route_open_unknown_module_escapes_target_module_id() {
10089 let handler = ControlHandler::new(Arc::new(Registry::default()));
10090 let capture = EventCapture::default();
10091 let _subscriber =
10092 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10093 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
10094 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
10095 let response = handler
10096 .handle_control_frame(
10097 &ctx,
10098 route_open_frame(
10099 395,
10100 hostile_module_id,
10101 unique_project_root("hostile-target-module-id"),
10102 ),
10103 )
10104 .await
10105 .unwrap();
10106
10107 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
10108 let event = capture
10109 .events()
10110 .into_iter()
10111 .find(|event| {
10112 event.target == "control"
10113 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
10114 })
10115 .expect("route.open unknown-module refusal event");
10116 let logged = event.fields.get("module_id").expect("module_id field");
10117 assert!(!logged.bytes().any(|byte| byte < 0x20));
10118 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
10119 }
10120
10121 #[tokio::test(flavor = "current_thread")]
10122 async fn route_open_module_rejection_uses_daemon_counter_key() {
10123 let registry = Arc::new(Registry::default());
10124 let forwarding = Arc::new(ForwardingTable::default());
10125 let handler =
10126 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10127 let module_connection = ConnectionId::new(95);
10128 let (module_ctx, mut module_rx) = route_ctx(module_connection);
10129 hello_via_sink(
10130 &handler,
10131 &module_ctx,
10132 &mut module_rx,
10133 hello_frame("aft", PROTOCOL_VERSION, 395),
10134 )
10135 .await;
10136
10137 let client_connection = ConnectionId::new(96);
10138 let (client_ctx, _client_rx) = route_ctx(client_connection);
10139 let capture = EventCapture::default();
10140 let _subscriber =
10141 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10142 let (route_task, bind) = relay_route_open(
10143 &handler,
10144 client_connection,
10145 &client_ctx.egress,
10146 &mut module_rx,
10147 396,
10148 "aft",
10149 "hostile-module-code",
10150 )
10151 .await;
10152 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
10153 let rejection = Frame::build(
10154 FrameType::Error,
10155 control_flags(),
10156 0,
10157 0,
10158 bind.header.corr,
10159 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
10160 )
10161 .unwrap();
10162 handler
10163 .handle_control_frame(&module_ctx, rejection)
10164 .await
10165 .unwrap();
10166
10167 let response = route_task.await.unwrap();
10168 assert_eq!(parse_error(&response[0])["code"], hostile_code);
10169 let counters = handler.counters().snapshot();
10170 assert_eq!(
10171 counters["route_open_refused_by_code"],
10172 json!({ "module_rejected": 1 })
10173 );
10174 assert!(counters["route_open_refused_by_code"]
10175 .get(hostile_code)
10176 .is_none());
10177
10178 let event = capture
10179 .events()
10180 .into_iter()
10181 .find(|event| {
10182 event.target == "control"
10183 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
10184 })
10185 .expect("route.open module-rejection refusal event");
10186 let logged = event.fields.get("module_code").expect("module_code field");
10187 assert!(!logged.bytes().any(|byte| byte < 0x20));
10188 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
10189 }
10190
10191 #[tokio::test]
10192 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
10193 let registry = Arc::new(Registry::default());
10194 let supervisor_handle = SupervisorHandle::new();
10195 let missing_program = std::env::temp_dir().join(format!(
10196 "subc-route-open-missing-program-{}",
10197 std::process::id()
10198 ));
10199 let supervisor =
10200 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
10201 .with_handle(supervisor_handle.clone());
10202 let module = supervisor
10203 .supervise_configured(
10204 ModuleSpec {
10205 module_id: "failed".to_string(),
10206 program: missing_program,
10207 args: Vec::new(),
10208 env: Vec::new(),
10209 reserved: false,
10210 reserved_prefixes: Vec::new(),
10211 protocol: ModuleProtocol::Subc,
10212 overlap: Default::default(),
10213 },
10214 true,
10215 )
10216 .unwrap();
10217 assert_eq!(module.state().unwrap(), ModuleState::Failed);
10218
10219 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10220 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
10221 let response = handler
10222 .handle_control_frame(
10223 &ctx,
10224 route_open_frame(305, "failed", unique_project_root("failed")),
10225 )
10226 .await
10227 .unwrap();
10228
10229 assert_eq!(response[0].header.ty, FrameType::Error);
10230 let error = parse_error(&response[0]);
10231 assert_eq!(error["code"], "target_unavailable");
10232 assert!(error["message"]
10233 .as_str()
10234 .unwrap()
10235 .contains("state=failed, enabled=true, live=false"));
10236 }
10237
10238 #[tokio::test]
10239 async fn route_open_role_mismatch_remains_target_unavailable() {
10240 let registry = Arc::new(Registry::default());
10241 let handler = ControlHandler::new(Arc::clone(®istry));
10242 handler
10243 .handle_control(
10244 ConnectionId::new(41),
10245 non_routable_hello_frame_with_control_ops("health-only", 306, None),
10246 )
10247 .unwrap();
10248
10249 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
10250 let response = handler
10251 .handle_control_frame(
10252 &ctx,
10253 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
10254 )
10255 .await
10256 .unwrap();
10257
10258 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10259 assert!(parse_error(&response[0])["message"]
10260 .as_str()
10261 .unwrap()
10262 .contains("does not provide the requested target"));
10263 }
10264
10265 #[tokio::test]
10266 async fn route_open_inactive_registration_remains_target_unavailable() {
10267 let registry = Arc::new(Registry::default());
10268 let handler = ControlHandler::new(Arc::clone(®istry));
10269 handler
10270 .handle_control(
10271 ConnectionId::new(43),
10272 hello_frame("inactive", PROTOCOL_VERSION, 308),
10273 )
10274 .unwrap();
10275 assert!(registry
10276 .set_module_state_for_test("inactive", ChannelState::Closed)
10277 .unwrap());
10278
10279 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
10280 let response = handler
10281 .handle_control_frame(
10282 &ctx,
10283 route_open_frame(309, "inactive", unique_project_root("inactive")),
10284 )
10285 .await
10286 .unwrap();
10287
10288 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10289 assert!(parse_error(&response[0])["message"]
10290 .as_str()
10291 .unwrap()
10292 .contains("is not active"));
10293 }
10294
10295 #[tokio::test]
10296 async fn late_health_reply_is_recorded_through_the_module_response_path() {
10297 let registry = Arc::new(Registry::default());
10298 let forwarding = Arc::new(ForwardingTable::default());
10299 let supervisor_handle = SupervisorHandle::new();
10300 let supervisor = Supervisor::new(Arc::clone(®istry), crate::RestartPolicy::default())
10301 .with_forwarding(Arc::clone(&forwarding))
10302 .with_handle(supervisor_handle.clone());
10303 let module = supervisor
10304 .supervise_configured(
10305 crate::ModuleSpec {
10306 module_id: "late-health-response".to_string(),
10307 program: PathBuf::from("disabled-module"),
10308 args: Vec::new(),
10309 env: Vec::new(),
10310 reserved: false,
10311 reserved_prefixes: Vec::new(),
10312 protocol: ModuleProtocol::Subc,
10313 overlap: Default::default(),
10314 },
10315 false,
10316 )
10317 .unwrap();
10318 let handler =
10319 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10320 .with_supervisor(supervisor_handle);
10321 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
10322 handler
10323 .handle_control_frame(
10324 &module_ctx,
10325 hello_frame_with_control_ops(
10326 "late-health-response",
10327 PROTOCOL_VERSION,
10328 7,
10329 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10330 ),
10331 )
10332 .await
10333 .unwrap();
10334 let probe_started_at = Instant::now() - Duration::from_millis(80);
10335 let pending = forwarding
10336 .begin_health_probe_rpc_for(
10337 "late-health-response",
10338 MODULE_CONTROL_OP_HEALTH_CHECK,
10339 probe_started_at,
10340 Instant::now() - Duration::from_millis(1),
10341 )
10342 .unwrap();
10343 assert!(forwarding
10344 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
10345 .unwrap());
10346
10347 let responses = handler
10348 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
10349 .await
10350 .unwrap();
10351
10352 assert!(responses.is_empty());
10353 let health = module.status().unwrap().health;
10354 assert_eq!(health.late_answer_count, 1);
10355 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
10356 }
10357
10358 #[tokio::test]
10359 async fn health_probe_timeout_and_module_death_are_typed() {
10360 let registry = Arc::new(Registry::default());
10361 let forwarding = Arc::new(ForwardingTable::default());
10362 let handler =
10363 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10364 .with_health_probe_timeout(Duration::from_millis(50));
10365 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
10366 hello_via_sink(
10367 &handler,
10368 &module_ctx,
10369 &mut module_rx,
10370 hello_frame_with_control_ops(
10371 "aft",
10372 PROTOCOL_VERSION,
10373 7,
10374 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10375 ),
10376 )
10377 .await;
10378
10379 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10380 let responses = handler
10381 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
10382 .await
10383 .unwrap();
10384 assert_eq!(responses[0].header.ty, FrameType::Error);
10385 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
10386 let _ = module_rx.try_recv();
10387
10388 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
10389 let health_handler = handler.clone();
10390 let death_task = tokio::spawn(async move {
10391 health_handler
10392 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
10393 .await
10394 .unwrap()
10395 });
10396 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10397 .await
10398 .unwrap()
10399 .unwrap();
10400 handler
10401 .cleanup_connection(module_ctx.connection_id)
10402 .unwrap();
10403 let responses = death_task.await.unwrap();
10404 assert_eq!(responses[0].header.ty, FrameType::Error);
10405 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
10406 }
10407
10408 #[test]
10409 fn hello_requires_exact_protocol_version() {
10410 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
10411 let registry = Arc::new(Registry::default());
10412 let handler = ControlHandler::new(Arc::clone(®istry));
10413 let responses = handler
10414 .handle_control(
10415 ConnectionId::new(connection),
10416 hello_frame("aft", offered, 9),
10417 )
10418 .unwrap();
10419
10420 assert_eq!(responses.len(), 1);
10421 assert_eq!(responses[0].header.ty, FrameType::Error);
10422 let error = parse_error(&responses[0]);
10423 assert_eq!(error["code"], "version_unsupported");
10424 assert!(registry.get_module("aft").unwrap().is_none());
10425 assert_eq!(registry.active_registration_count().unwrap(), 0);
10426 }
10427 }
10428
10429 #[test]
10430 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
10431 let registry = Arc::new(Registry::default());
10432 let forwarding = Arc::new(ForwardingTable::default());
10433 let handler =
10434 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10435 let module_connection = ConnectionId::new(301);
10436 let registration = registry
10437 .register_with_control_ops(
10438 manifest("aft-push", PROTOCOL_VERSION),
10439 PROTOCOL_VERSION,
10440 module_connection,
10441 module_baseline_control_ops(),
10442 )
10443 .unwrap();
10444 let (module_tx, _module_rx) = mpsc::channel(8);
10445 let endpoint = forwarding
10446 .register_module_connection(
10447 module_connection,
10448 "aft-push".to_string(),
10449 PROTOCOL_VERSION,
10450 manifest_concurrency(®istration.manifest),
10451 FrameSink::new(module_tx),
10452 )
10453 .unwrap();
10454
10455 let unknown = Frame::build(
10457 FrameType::Push,
10458 control_flags(),
10459 0,
10460 0,
10461 5,
10462 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
10463 )
10464 .unwrap();
10465 let out = handler.handle_status_update(endpoint, unknown).unwrap();
10466 assert!(
10467 out.is_empty(),
10468 "unknown push op must be ignored, got {out:?}"
10469 );
10470
10471 let malformed = Frame::build(
10473 FrameType::Push,
10474 control_flags(),
10475 0,
10476 0,
10477 6,
10478 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
10479 )
10480 .unwrap();
10481 let out = handler.handle_status_update(endpoint, malformed).unwrap();
10482 assert_eq!(out.len(), 1);
10483 assert_eq!(out[0].header.ty, FrameType::Error);
10484 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
10485 }
10486
10487 #[test]
10488 fn hello_rejected_when_connection_already_owns_client_routes() {
10489 let registry = Arc::new(Registry::default());
10490 let forwarding = Arc::new(ForwardingTable::default());
10491 let handler =
10492 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10493 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
10495 let client_connection = ConnectionId::new(202);
10496
10497 let responses = handler
10500 .handle_control(
10501 client_connection,
10502 hello_frame("aft-second", PROTOCOL_VERSION, 9),
10503 )
10504 .unwrap();
10505 assert_eq!(responses[0].header.ty, FrameType::Error);
10506 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
10507 assert!(registry.get_module("aft-second").unwrap().is_none());
10508 }
10509
10510 #[tokio::test]
10511 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
10512 let registry = Arc::new(Registry::default());
10513 let forwarding = Arc::new(ForwardingTable::default());
10514 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
10515 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
10516 hello_via_sink(
10517 &handler,
10518 &module_ctx,
10519 &mut module_rx,
10520 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
10521 )
10522 .await;
10523 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
10524 let pending = forwarding
10525 .begin_route_bind_relay_for_test(
10526 client_ctx.connection_id,
10527 client_ctx.egress.clone(),
10528 2,
10529 "alpha",
10530 )
10531 .unwrap();
10532 forwarding
10533 .complete_pending_relay(
10534 module_ctx.connection_id,
10535 pending.corr,
10536 RouteBindRelayOutcome::Accepted,
10537 )
10538 .unwrap();
10539 client_rx.try_recv().unwrap();
10540 for module_id in ["beta", "alpha"] {
10541 let replies = handler
10542 .handle_control_frame(
10543 &module_ctx,
10544 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
10545 )
10546 .await
10547 .unwrap();
10548 assert_eq!(replies.len(), 1, "second HELLO must be refused");
10549 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
10550 }
10551 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
10552 assert!(registry.get_module("beta").unwrap().is_none());
10553 assert!(matches!(
10554 forwarding
10555 .lookup_data_route(
10556 client_ctx.connection_id,
10557 pending.client_channel,
10558 pending.client_epoch,
10559 )
10560 .unwrap(),
10561 DataRoute::Client(DataRouteState::Bound(_))
10562 ));
10563 assert!(handler
10564 .hello_launch_nonces
10565 .lock()
10566 .unwrap()
10567 .presented(module_ctx.connection_id, Some("alpha-nonce")));
10568 assert!(module_rx.try_recv().is_err());
10569 }
10570
10571 #[test]
10572 fn reserved_module_hello_requires_matching_launch_nonce() {
10573 let registry = Arc::new(Registry::default());
10574 let supervisor = SupervisorHandle::new();
10575 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
10578 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
10579
10580 let no_nonce = handler
10582 .handle_control(
10583 ConnectionId::new(1),
10584 hello_frame("vault", PROTOCOL_VERSION, 1),
10585 )
10586 .unwrap();
10587 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
10588 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
10589 assert!(registry.get_module("vault").unwrap().is_none());
10590
10591 let wrong = handler
10593 .handle_control(
10594 ConnectionId::new(2),
10595 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
10596 )
10597 .unwrap();
10598 assert_eq!(wrong[0].header.ty, FrameType::Error);
10599 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
10600 assert!(registry.get_module("vault").unwrap().is_none());
10601
10602 let ok = handler
10604 .handle_control(
10605 ConnectionId::new(3),
10606 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
10607 )
10608 .unwrap();
10609 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
10610 assert!(registry.get_module("vault").unwrap().is_some());
10611 }
10612
10613 #[test]
10614 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
10615 let registry = Arc::new(Registry::default());
10616 let supervisor = SupervisorHandle::new();
10617 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
10618 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
10619 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
10620
10621 let squat = handler
10622 .handle_control(
10623 ConnectionId::new(1),
10624 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
10625 )
10626 .unwrap();
10627 assert_eq!(squat[0].header.ty, FrameType::Error);
10628 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
10629 assert!(parse_error(&squat[0])["message"]
10630 .as_str()
10631 .unwrap()
10632 .contains("fed:"));
10633
10634 let accepted_peer = handler
10635 .handle_control(
10636 ConnectionId::new(2),
10637 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
10638 )
10639 .unwrap();
10640 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
10641
10642 let accepted_short = handler
10643 .handle_control(
10644 ConnectionId::new(3),
10645 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
10646 )
10647 .unwrap();
10648 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
10649
10650 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
10651 let response = handler
10652 .handle_control(
10653 ConnectionId::new(conn),
10654 hello_frame(module_id, PROTOCOL_VERSION, conn),
10655 )
10656 .unwrap();
10657 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
10658 }
10659 }
10660
10661 #[test]
10662 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
10663 let registry = Arc::new(Registry::default());
10664 let supervisor = SupervisorHandle::new();
10665 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
10666 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
10667 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
10668 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
10669
10670 let owner_nonce = handler
10671 .handle_control(
10672 ConnectionId::new(1),
10673 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
10674 )
10675 .unwrap();
10676 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
10677 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
10678 assert!(registry.get_module("fed:special").unwrap().is_none());
10679
10680 let exact_nonce = handler
10681 .handle_control(
10682 ConnectionId::new(2),
10683 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
10684 )
10685 .unwrap();
10686 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
10687 assert!(registry.get_module("fed:special").unwrap().is_some());
10688 }
10689
10690 #[test]
10691 fn non_reserved_module_ignores_launch_nonce() {
10692 let registry = Arc::new(Registry::default());
10693 let handler = ControlHandler::new(Arc::clone(®istry));
10696 let no_nonce = handler
10697 .handle_control(
10698 ConnectionId::new(1),
10699 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
10700 )
10701 .unwrap();
10702 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
10703 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
10704
10705 let echoed_nonce = handler
10706 .handle_control(
10707 ConnectionId::new(2),
10708 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
10709 )
10710 .unwrap();
10711 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
10712 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
10713 }
10714
10715 #[test]
10716 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
10717 let handler = ControlHandler::default();
10718 let conn = ConnectionId::new(1);
10719 let malformed = Frame::build(
10720 FrameType::Hello,
10721 control_flags(),
10722 0,
10723 0,
10724 3,
10725 b"{not json".to_vec(),
10726 )
10727 .unwrap();
10728
10729 let error = handler.handle_control(conn, malformed).unwrap();
10730 assert_eq!(error[0].header.ty, FrameType::Error);
10731 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
10732
10733 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
10734 let pong = handler.handle_control(conn, ping).unwrap();
10735 assert_eq!(pong[0].header.ty, FrameType::Pong);
10736 assert_eq!(pong[0].header.corr, 4);
10737 }
10738
10739 #[test]
10740 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
10741 let registry = Arc::new(Registry::default());
10742 let handler = ControlHandler::new(Arc::clone(®istry));
10743
10744 handler
10745 .handle_control(
10746 ConnectionId::new(1),
10747 hello_frame("aft", PROTOCOL_VERSION, 1),
10748 )
10749 .unwrap();
10750 let duplicate = handler
10751 .handle_control(
10752 ConnectionId::new(2),
10753 hello_frame("aft", PROTOCOL_VERSION, 2),
10754 )
10755 .unwrap();
10756
10757 assert_eq!(duplicate[0].header.ty, FrameType::Error);
10758 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
10759 let registration = registry.get_module("aft").unwrap().unwrap();
10760 assert_eq!(registration.connection_id, ConnectionId::new(1));
10761 }
10762
10763 #[test]
10764 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
10765 let registry = Arc::new(Registry::default());
10766 let forwarding = Arc::new(ForwardingTable::default());
10767 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
10768 let handler =
10769 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10770 .with_process_liveness(process_liveness);
10771 let (ctx, route_channel, route_epoch) =
10772 bind_liveness_route(®istry, &forwarding, "aft-dead");
10773 let responses = handler
10774 .handle_route_poll(
10775 &ctx,
10776 route_poll_frame(41, PollKind::Liveness, route_channel),
10777 route_channel,
10778 route_epoch,
10779 PollKind::Liveness,
10780 )
10781 .unwrap();
10782
10783 assert_eq!(responses.len(), 1);
10784 assert_eq!(responses[0].header.ty, FrameType::Response);
10785 assert_route_poll_liveness(&responses[0], false);
10786 }
10787
10788 #[test]
10789 fn liveness_poll_without_process_source_uses_bound_route() {
10790 let registry = Arc::new(Registry::default());
10791 let forwarding = Arc::new(ForwardingTable::default());
10792 let handler =
10793 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10794 let (ctx, route_channel, route_epoch) =
10795 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
10796 let responses = handler
10797 .handle_route_poll(
10798 &ctx,
10799 route_poll_frame(42, PollKind::Liveness, route_channel),
10800 route_channel,
10801 route_epoch,
10802 PollKind::Liveness,
10803 )
10804 .unwrap();
10805
10806 assert_route_poll_liveness(&responses[0], true);
10807 }
10808
10809 #[test]
10810 fn liveness_poll_untracked_process_source_uses_bound_route() {
10811 let registry = Arc::new(Registry::default());
10812 let forwarding = Arc::new(ForwardingTable::default());
10813 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
10814 let handler =
10815 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10816 .with_process_liveness(process_liveness);
10817 let (ctx, route_channel, route_epoch) =
10818 bind_liveness_route(®istry, &forwarding, "aft-untracked");
10819 let responses = handler
10820 .handle_route_poll(
10821 &ctx,
10822 route_poll_frame(43, PollKind::Liveness, route_channel),
10823 route_channel,
10824 route_epoch,
10825 PollKind::Liveness,
10826 )
10827 .unwrap();
10828
10829 assert_route_poll_liveness(&responses[0], true);
10830 }
10831
10832 #[tokio::test]
10833 async fn unknown_op_returns_unknown_control_op() {
10834 let handler = ControlHandler::default();
10835 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
10836 let request = Frame::build(
10837 FrameType::Request,
10838 control_flags(),
10839 0,
10840 0,
10841 55,
10842 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
10843 )
10844 .unwrap();
10845
10846 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
10847
10848 assert_eq!(response.len(), 1);
10849 assert_eq!(response[0].header.ty, FrameType::Error);
10850 assert_eq!(response[0].header.corr, 55);
10851 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
10852 }
10853
10854 #[tokio::test]
10855 async fn supervisor_provenance_rejects_unknown_exact_module() {
10856 let handler = ControlHandler::default();
10857 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
10858 let request = Frame::build(
10859 FrameType::Request,
10860 control_flags(),
10861 0,
10862 0,
10863 57,
10864 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
10865 )
10866 .unwrap();
10867
10868 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
10869
10870 assert_eq!(response.len(), 1);
10871 assert_eq!(response[0].header.ty, FrameType::Error);
10872 assert_eq!(response[0].header.corr, 57);
10873 let error = parse_error(&response[0]);
10874 assert_eq!(error["code"], "unknown_module");
10875 assert_eq!(error["message"], "module_id 'missing' is not supervised");
10876 }
10877
10878 #[test]
10879 fn provenance_probe_override_keeps_handler_tests_deterministic() {
10880 let expected = subc_control::RunningImageAgreement::Unavailable {
10881 reason: subc_control::RunningImageUnavailableReason::HashFailed,
10882 };
10883 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
10884 assert_eq!(handler.provenance_probe_override, Some(expected));
10885 }
10886
10887 #[test]
10888 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
10889 let verdict = reload_verdict(
10890 std::path::Path::new("/bin/new"),
10891 Some(std::path::Path::new("/bin/old")),
10892 subc_control::RunningImageAgreement::Unavailable {
10893 reason: subc_control::RunningImageUnavailableReason::HashFailed,
10894 },
10895 );
10896 assert!(matches!(
10897 verdict.path,
10898 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
10899 if configured == std::path::Path::new("/bin/new")
10900 && spawned_from == std::path::Path::new("/bin/old")
10901 ));
10902 }
10903
10904 #[test]
10905 fn reload_verdict_detects_replaced_image_at_same_path() {
10906 let image = subc_control::RunningImageAgreement::Mismatch {
10907 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
10908 digest: "old".into(),
10909 },
10910 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
10911 digest: "new".into(),
10912 },
10913 };
10914 let verdict = reload_verdict(
10915 std::path::Path::new("/bin/same"),
10916 Some(std::path::Path::new("/bin/same")),
10917 image.clone(),
10918 );
10919 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
10920 assert_eq!(verdict.image, image);
10921 }
10922
10923 #[test]
10924 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
10925 let image = subc_control::RunningImageAgreement::Unavailable {
10926 reason: subc_control::RunningImageUnavailableReason::NotRunning,
10927 };
10928 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
10929 assert_eq!(
10930 verdict.path,
10931 subc_control::ReloadPathAgreement::Unavailable {
10932 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
10933 }
10934 );
10935 assert_eq!(verdict.image, image);
10936
10937 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
10938 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
10939 };
10940 let verdict = reload_verdict(
10941 std::path::Path::new("/bin/same"),
10942 Some(std::path::Path::new("/bin/same")),
10943 unconfirmed.clone(),
10944 );
10945 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
10946 assert_eq!(verdict.image, unconfirmed);
10947 }
10948
10949 #[test]
10950 fn reload_verdict_preserves_each_image_unavailability_reason() {
10951 use subc_control::RunningImageUnavailableReason as Reason;
10952
10953 for reason in [
10954 Reason::NotRunning,
10955 Reason::UnsupportedPlatform,
10956 Reason::RunningExecutableUnreadable,
10957 Reason::SpawnedPathUnreadable,
10958 Reason::HashFailed,
10959 Reason::ProcessIdentityUnconfirmed,
10960 Reason::Unknown("future_probe_reason".to_string()),
10961 ] {
10962 let image = subc_control::RunningImageAgreement::Unavailable {
10963 reason: reason.clone(),
10964 };
10965 let verdict = reload_verdict(
10966 std::path::Path::new("/bin/same"),
10967 Some(std::path::Path::new("/bin/same")),
10968 image.clone(),
10969 );
10970 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
10971 assert_eq!(verdict.image, image, "{reason:?}");
10972 }
10973 }
10974
10975 #[tokio::test]
10976 async fn malformed_control_bodies_return_invalid_control_body() {
10977 let handler = ControlHandler::default();
10978 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
10979
10980 for (corr, body) in [
10981 (56, br#"{"route_channel":1}"#.as_slice()),
10982 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
10983 (
10984 58,
10985 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
10986 ),
10987 ] {
10988 let request = Frame::build(
10989 FrameType::Request,
10990 control_flags(),
10991 0,
10992 0,
10993 corr,
10994 body.to_vec(),
10995 )
10996 .unwrap();
10997 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
10998
10999 assert_eq!(response.len(), 1);
11000 assert_eq!(response[0].header.ty, FrameType::Error);
11001 assert_eq!(response[0].header.corr, corr);
11002 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
11003 }
11004 }
11005
11006 #[tokio::test]
11007 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
11008 let registry = Arc::new(Registry::default());
11009 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
11010 let router = Router::with_control_handler(Arc::clone(&control));
11011 let connection = router.begin_connection();
11012 let (ctx, mut rx) = route_ctx(connection.id());
11013
11014 router
11015 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
11016 .await
11017 .unwrap();
11018 let response = rx.recv().await.unwrap();
11019 let ack = parse_ack(&response);
11020 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
11021 let channel = 1;
11022
11023 let goodbye =
11024 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
11025 router.route_for_connection(&ctx, goodbye).await.unwrap();
11026 assert!(rx.try_recv().is_err());
11027 assert!(registry.get_module("aft").unwrap().is_none());
11028
11029 router
11030 .route_for_connection(&ctx, channel_request(channel, 13))
11031 .await
11032 .unwrap();
11033 let error_frame = rx.recv().await.unwrap();
11034 assert_eq!(error_frame.header.ty, FrameType::Error);
11035 assert_eq!(error_frame.header.channel, channel);
11036 }
11037
11038 #[tokio::test]
11039 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
11040 let registry = Arc::new(Registry::default());
11041 let handler = ControlHandler::new(registry).with_capability_config(
11042 [("prov".to_string(), true), ("cons".to_string(), true)],
11043 BTreeMap::new(),
11044 );
11045 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
11046 register_capability_manifest(
11047 &handler,
11048 &provider_ctx,
11049 &mut provider_rx,
11050 capability_manifest("prov", &["thing/v1"], &[]),
11051 1,
11052 )
11053 .await;
11054 let mut consumer = capability_manifest("cons", &[], &[]);
11055 consumer.capabilities.as_mut().unwrap().requires.push(
11056 subc_protocol::manifest::CapabilityRequirement {
11057 capability: "thing/v1".to_string(),
11058 need: subc_protocol::manifest::CapabilityNeed::Required,
11059 },
11060 );
11061 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
11062 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
11063 assert_eq!(
11064 handler.capability_evaluator.verdict("cons", "thing/v1"),
11065 Some(CapabilityVerdict::Provided)
11066 );
11067 let (mut client_rx, _) = open_route_for_capability_test(
11068 &handler,
11069 &provider_ctx,
11070 &mut provider_rx,
11071 703,
11072 3,
11073 "prov",
11074 None,
11075 )
11076 .await;
11077 let goodbye =
11078 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
11079 handler
11080 .handle_control_frame(&provider_ctx, goodbye)
11081 .await
11082 .unwrap();
11083 assert_eq!(
11084 handler.capability_evaluator.verdict("cons", "thing/v1"),
11085 Some(CapabilityVerdict::NeverProvided)
11086 );
11087 let closed = client_rx
11088 .try_recv()
11089 .expect("GOODBYE pushes route.closed before route GOODBYE");
11090 assert!(
11091 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
11092 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
11093 );
11094 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
11095 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
11096 }
11097
11098 #[tokio::test]
11099 async fn dropping_router_connection_releases_registration() {
11100 let registry = Arc::new(Registry::default());
11101 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
11102 let router = Router::with_control_handler(control);
11103 let connection = router.begin_connection();
11104 let (ctx, mut rx) = route_ctx(connection.id());
11105
11106 router
11107 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
11108 .await
11109 .unwrap();
11110 let response = rx.recv().await.unwrap();
11111 let ack = parse_ack(&response);
11112 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
11113 assert!(registry.get_module("aft").unwrap().is_some());
11114
11115 drop(connection);
11116
11117 assert!(registry.get_module("aft").unwrap().is_none());
11118 assert_eq!(registry.active_registration_count().unwrap(), 0);
11119 }
11120
11121 fn capability_manifest(
11122 module_id: &str,
11123 provides: &[&str],
11124 must_never_reach: &[&str],
11125 ) -> ModuleManifest {
11126 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
11127 manifest.capabilities = Some(CapabilityDeclarations {
11128 provides: provides
11129 .iter()
11130 .map(|capability| (*capability).to_string())
11131 .collect(),
11132 requires: Vec::new(),
11133 must_never_reach: must_never_reach
11134 .iter()
11135 .map(|capability| (*capability).to_string())
11136 .collect(),
11137 });
11138 manifest
11139 }
11140
11141 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
11142 Frame::build(
11143 FrameType::Hello,
11144 control_flags(),
11145 0,
11146 0,
11147 corr,
11148 serde_json::to_vec(&ModuleHelloBody {
11149 protocol_ver: manifest.protocol_ver,
11150 manifest,
11151 control_ops: None,
11152 launch_nonce: None,
11153 })
11154 .expect("capability test HELLO serializes"),
11155 )
11156 .expect("capability test HELLO frame builds")
11157 }
11158
11159 fn catalog_update_with_capabilities_frame(
11160 corr: u64,
11161 capabilities: CapabilityDeclarations,
11162 ) -> Frame {
11163 Frame::build(
11164 FrameType::Request,
11165 control_flags(),
11166 0,
11167 0,
11168 corr,
11169 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
11170 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
11171 capabilities: Some(capabilities),
11172 ready: None,
11173 })
11174 .expect("capability catalog.update serializes"),
11175 )
11176 .expect("capability catalog.update frame builds")
11177 }
11178
11179 async fn register_capability_manifest(
11180 handler: &ControlHandler,
11181 ctx: &RouteCtx,
11182 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
11183 manifest: ModuleManifest,
11184 corr: u64,
11185 ) {
11186 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
11187 }
11188
11189 async fn open_route_for_capability_test(
11190 handler: &ControlHandler,
11191 target_ctx: &RouteCtx,
11192 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
11193 client_connection_id: u64,
11194 corr: u64,
11195 target_module_id: &str,
11196 consumer_identity: Option<ConsumerIdentity>,
11197 ) -> (
11198 mpsc::Receiver<crate::router::OutboundFrame>,
11199 ModuleControlRequest,
11200 ) {
11201 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
11202 let route_handler = handler.clone();
11203 let target_module_id = target_module_id.to_string();
11204 let route_task = tokio::spawn(async move {
11205 route_handler
11206 .handle_control_frame(
11207 &client_ctx,
11208 route_open_frame_with_admission_facts(
11209 corr,
11210 &target_module_id,
11211 unique_project_root("admission-facts"),
11212 consumer_identity,
11213 None,
11214 ),
11215 )
11216 .await
11217 .expect("capability test route.open succeeds")
11218 });
11219 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
11220 .await
11221 .expect("capability test route.open must reach route.bind")
11222 .expect("target control receiver stays open");
11223 let bind_request: ModuleControlRequest =
11224 serde_json::from_slice(&bind.body).expect("route.bind decodes");
11225 handler
11226 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
11227 .await
11228 .expect("capability test route.bind ACK succeeds");
11229 assert!(route_task.await.expect("route.open task joins").is_empty());
11230 let opened = client_rx
11231 .recv()
11232 .await
11233 .expect("successful route.open publishes a response");
11234 assert!(matches!(
11235 serde_json::from_slice::<ClientControlResponse>(&opened.body),
11236 Ok(ClientControlResponse::RouteOpen { .. })
11237 ));
11238 (client_rx, bind_request)
11239 }
11240
11241 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
11242 assert_eq!(frame.header.ty, FrameType::Push);
11243 assert_eq!(frame.header.channel, 0);
11244 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
11245 .expect("route.closed control push decodes");
11246 let ClientControlPush::RouteClosed { channels, .. } = &push else {
11247 panic!("expected route.closed");
11248 };
11249 assert_eq!(channels.len(), 1, "exactly one violating route closed");
11250 let channels = channels.clone();
11251 assert_eq!(
11252 push,
11253 ClientControlPush::RouteClosed {
11254 module_id: target_module_id.to_string(),
11255 channels,
11256 reason: RouteCloseReason::CapabilityDenied,
11257 drained: false,
11258 abandoned: 0,
11259 excluded_subscriptions: 0,
11260 terminal: Some(false),
11261 }
11262 );
11263 }
11264
11265 #[tokio::test]
11266 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
11267 let registry = Arc::new(Registry::default());
11268 let forwarding = Arc::new(ForwardingTable::default());
11269 let supervisor = SupervisorHandle::new();
11270 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11271 let handler =
11272 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11273 .with_supervisor(supervisor);
11274 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
11275 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
11276 register_capability_manifest(
11277 &handler,
11278 &target_ctx,
11279 &mut target_rx,
11280 capability_manifest("target", &["credentials-provider/v1"], &[]),
11281 1,
11282 )
11283 .await;
11284 register_capability_manifest(
11285 &handler,
11286 &opener_ctx,
11287 &mut opener_rx,
11288 capability_manifest("opener", &[], &["credentials-provider/v1"]),
11289 2,
11290 )
11291 .await;
11292
11293 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
11294 let replies = handler
11295 .handle_control_frame(
11296 &client_ctx,
11297 route_open_frame_with_admission_facts(
11298 3,
11299 "target",
11300 unique_project_root("admission-facts"),
11301 Some(ConsumerIdentity {
11302 module_id: "opener".to_string(),
11303 launch_nonce: "opener-nonce".to_string(),
11304 }),
11305 None,
11306 ),
11307 )
11308 .await
11309 .expect("denied route.open returns a typed frame");
11310 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
11311 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
11312 assert!(
11313 target_rx.try_recv().is_err(),
11314 "forbidden route.open must not relay route.bind"
11315 );
11316 }
11317
11318 #[tokio::test]
11319 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
11320 let registry = Arc::new(Registry::default());
11321 let forwarding = Arc::new(ForwardingTable::default());
11322 let supervisor = SupervisorHandle::new();
11323 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11324 let handler =
11325 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11326 .with_supervisor(supervisor);
11327 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
11328 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
11329 register_capability_manifest(
11330 &handler,
11331 &target_ctx,
11332 &mut target_rx,
11333 capability_manifest("target", &["credentials-provider/v1"], &[]),
11334 1,
11335 )
11336 .await;
11337 register_capability_manifest(
11338 &handler,
11339 &old_opener_ctx,
11340 &mut old_opener_rx,
11341 capability_manifest("opener", &[], &[]),
11342 2,
11343 )
11344 .await;
11345 let (mut client_rx, _) = open_route_for_capability_test(
11346 &handler,
11347 &target_ctx,
11348 &mut target_rx,
11349 712,
11350 3,
11351 "target",
11352 Some(ConsumerIdentity {
11353 module_id: "opener".to_string(),
11354 launch_nonce: "opener-nonce".to_string(),
11355 }),
11356 )
11357 .await;
11358 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
11359
11360 handler
11361 .cleanup_connection(old_opener_ctx.connection_id)
11362 .expect("old opener registration cleans up");
11363 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
11364 register_capability_manifest(
11365 &handler,
11366 &new_opener_ctx,
11367 &mut new_opener_rx,
11368 capability_manifest("opener", &[], &["credentials-provider/v1"]),
11369 4,
11370 )
11371 .await;
11372
11373 assert_capability_denied_push(
11374 client_rx
11375 .try_recv()
11376 .expect("HELLO deny addition must emit route.closed")
11377 .frame,
11378 "target",
11379 );
11380 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
11381 assert!(matches!(
11382 target_rx.try_recv(),
11383 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
11384 ));
11385 }
11386
11387 #[tokio::test]
11388 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
11389 let registry = Arc::new(Registry::default());
11390 let forwarding = Arc::new(ForwardingTable::default());
11391 let supervisor = SupervisorHandle::new();
11392 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11393 let handler =
11394 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11395 .with_supervisor(supervisor);
11396 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
11397 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
11398 register_capability_manifest(
11399 &handler,
11400 &target_ctx,
11401 &mut target_rx,
11402 capability_manifest("target", &[], &[]),
11403 1,
11404 )
11405 .await;
11406 register_capability_manifest(
11407 &handler,
11408 &opener_ctx,
11409 &mut opener_rx,
11410 capability_manifest("opener", &[], &["credentials-provider/v1"]),
11411 2,
11412 )
11413 .await;
11414 let (mut client_rx, _) = open_route_for_capability_test(
11415 &handler,
11416 &target_ctx,
11417 &mut target_rx,
11418 722,
11419 3,
11420 "target",
11421 Some(ConsumerIdentity {
11422 module_id: "opener".to_string(),
11423 launch_nonce: "opener-nonce".to_string(),
11424 }),
11425 )
11426 .await;
11427 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
11428
11429 let replies = handler
11430 .handle_control_frame(
11431 &target_ctx,
11432 catalog_update_with_capabilities_frame(
11433 4,
11434 CapabilityDeclarations {
11435 provides: vec!["credentials-provider/v1".to_string()],
11436 requires: Vec::new(),
11437 must_never_reach: Vec::new(),
11438 },
11439 ),
11440 )
11441 .await
11442 .expect("claim catalog.update succeeds");
11443 assert!(matches!(
11444 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
11445 Ok(ModuleControlResponseToModule::CatalogUpdate {})
11446 ));
11447 assert_capability_denied_push(
11448 client_rx
11449 .try_recv()
11450 .expect("claim addition must emit route.closed")
11451 .frame,
11452 "target",
11453 );
11454 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
11455 assert!(matches!(
11456 target_rx.try_recv(),
11457 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
11458 ));
11459 }
11460
11461 #[tokio::test]
11462 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
11463 let registry = Arc::new(Registry::default());
11464 let forwarding = Arc::new(ForwardingTable::default());
11465 let supervisor = SupervisorHandle::new();
11466 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11467 let handler =
11468 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11469 .with_supervisor(supervisor);
11470 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
11471 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
11472 register_capability_manifest(
11473 &handler,
11474 &target_ctx,
11475 &mut target_rx,
11476 capability_manifest("target", &["credentials-provider/v1"], &[]),
11477 1,
11478 )
11479 .await;
11480 register_capability_manifest(
11481 &handler,
11482 &opener_ctx,
11483 &mut opener_rx,
11484 capability_manifest("opener", &[], &[]),
11485 2,
11486 )
11487 .await;
11488 let (mut client_rx, _) = open_route_for_capability_test(
11489 &handler,
11490 &target_ctx,
11491 &mut target_rx,
11492 732,
11493 3,
11494 "target",
11495 Some(ConsumerIdentity {
11496 module_id: "opener".to_string(),
11497 launch_nonce: "opener-nonce".to_string(),
11498 }),
11499 )
11500 .await;
11501 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
11502
11503 handler
11504 .handle_control_frame(
11505 &target_ctx,
11506 catalog_update_with_capabilities_frame(
11507 4,
11508 CapabilityDeclarations {
11509 provides: Vec::new(),
11510 requires: Vec::new(),
11511 must_never_reach: Vec::new(),
11512 },
11513 ),
11514 )
11515 .await
11516 .expect("claim removal catalog.update succeeds");
11517 assert_eq!(
11518 forwarding.active_binding_count().unwrap(),
11519 1,
11520 "removing an attested target claim must leave the route census unchanged"
11521 );
11522 assert!(
11523 client_rx.try_recv().is_err(),
11524 "claim removal must not emit route.closed capability_denied"
11525 );
11526 assert!(
11527 target_rx.try_recv().is_err(),
11528 "claim removal must not send the target a route GOODBYE"
11529 );
11530 }
11531
11532 #[tokio::test]
11535 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
11536 let registry = Arc::new(Registry::default());
11537 let forwarding = Arc::new(ForwardingTable::default());
11538 let supervisor = SupervisorHandle::new();
11539 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11540 let handler =
11541 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11542 .with_supervisor(supervisor);
11543 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
11544 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
11545 register_capability_manifest(
11546 &handler,
11547 &target_ctx,
11548 &mut target_rx,
11549 capability_manifest("target", &["credentials-provider/v1"], &[]),
11550 1,
11551 )
11552 .await;
11553 register_capability_manifest(
11554 &handler,
11555 &opener_ctx,
11556 &mut opener_rx,
11557 capability_manifest("opener", &[], &["credentials-provider/v1"]),
11558 2,
11559 )
11560 .await;
11561
11562 let (_client_rx, bind) = open_route_for_capability_test(
11563 &handler,
11564 &target_ctx,
11565 &mut target_rx,
11566 742,
11567 3,
11568 "target",
11569 None,
11570 )
11571 .await;
11572 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
11573 panic!("direct scope-honesty route must bind");
11574 };
11575 assert_eq!(principal, Some(Principal::Direct));
11576 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
11577 }
11578
11579 #[tokio::test]
11582 async fn must_never_reach_self_route_is_capability_forbidden() {
11583 let registry = Arc::new(Registry::default());
11584 let forwarding = Arc::new(ForwardingTable::default());
11585 let supervisor = SupervisorHandle::new();
11586 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
11587 let handler =
11588 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11589 .with_supervisor(supervisor);
11590 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
11591 register_capability_manifest(
11592 &handler,
11593 &self_ctx,
11594 &mut self_rx,
11595 capability_manifest(
11596 "self-provider",
11597 &["credentials-provider/v1"],
11598 &["credentials-provider/v1"],
11599 ),
11600 1,
11601 )
11602 .await;
11603
11604 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
11605 let replies = handler
11606 .handle_control_frame(
11607 &client_ctx,
11608 route_open_frame_with_admission_facts(
11609 2,
11610 "self-provider",
11611 unique_project_root("admission-facts"),
11612 Some(ConsumerIdentity {
11613 module_id: "self-provider".to_string(),
11614 launch_nonce: "self-nonce".to_string(),
11615 }),
11616 None,
11617 ),
11618 )
11619 .await
11620 .expect("self-route refusal returns a typed frame");
11621 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
11622 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
11623 assert!(
11624 self_rx.try_recv().is_err(),
11625 "self denial must not relay route.bind"
11626 );
11627 }
11628
11629 #[test]
11630 fn unsupported_channel_zero_frame_returns_error() {
11631 let handler = ControlHandler::default();
11632 let request = Frame::build(
11633 FrameType::Request,
11634 control_flags(),
11635 0,
11636 0,
11637 21,
11638 b"opaque".to_vec(),
11639 )
11640 .unwrap();
11641
11642 let response = handler
11643 .handle_control(ConnectionId::new(1), request)
11644 .unwrap();
11645
11646 assert_eq!(response[0].header.ty, FrameType::Error);
11647 assert_eq!(
11648 parse_error(&response[0])["code"],
11649 "unsupported_control_frame"
11650 );
11651 }
11652
11653 mod swap {
11658 use super::*;
11659
11660 const INCUMBENT: ConnectionId = ConnectionId::new(30);
11661 const CANDIDATE: ConnectionId = ConnectionId::new(40);
11662
11663 struct Swap {
11664 registry: Arc<Registry>,
11665 forwarding: Arc<ForwardingTable>,
11666 handler: ControlHandler,
11667 incumbent_ctx: RouteCtx,
11668 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
11669 candidate_ctx: RouteCtx,
11670 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
11671 }
11672
11673 async fn swap_with_incumbent() -> Swap {
11674 let registry = Arc::new(Registry::default());
11675 let forwarding = Arc::new(ForwardingTable::default());
11676 let handler =
11677 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11678 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
11679 hello_via_sink(
11680 &handler,
11681 &incumbent_ctx,
11682 &mut incumbent_rx,
11683 hello_frame("aft", PROTOCOL_VERSION, 7),
11684 )
11685 .await;
11686 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
11687 Swap {
11688 registry,
11689 forwarding,
11690 handler,
11691 incumbent_ctx,
11692 incumbent_rx,
11693 candidate_ctx,
11694 candidate_rx,
11695 }
11696 }
11697
11698 fn register_candidate(swap: &Swap, ready: Option<bool>) {
11699 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
11700 candidate_manifest.ready = ready;
11701 let registration = swap
11702 .registry
11703 .register_candidate_with_control_ops(
11704 candidate_manifest,
11705 PROTOCOL_VERSION,
11706 CANDIDATE,
11707 module_baseline_control_ops(),
11708 )
11709 .unwrap();
11710 swap.forwarding
11711 .register_candidate_module_connection(
11712 CANDIDATE,
11713 "aft".to_string(),
11714 PROTOCOL_VERSION,
11715 manifest_concurrency(®istration.manifest),
11716 swap.candidate_ctx.egress.clone(),
11717 )
11718 .unwrap();
11719 }
11720
11721 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
11722 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
11723 swap.registry.promote_candidate("aft").unwrap().unwrap();
11724 cutover.incumbent.unwrap()
11725 }
11726
11727 fn keyed_total(counters: &Value, key: &str) -> u64 {
11728 counters[key]
11729 .as_object()
11730 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
11731 .unwrap_or(0)
11732 }
11733
11734 #[tokio::test]
11740 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
11741 let mut swap = swap_with_incumbent().await;
11742 let handler = swap.handler.clone();
11743
11744 let cotenant = ConnectionId::new(31);
11746 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
11747 let (cotenant_task, cotenant_bind) = relay_route_open(
11748 &handler,
11749 cotenant,
11750 &cotenant_ctx.egress,
11751 &mut swap.incumbent_rx,
11752 100,
11753 "aft",
11754 "swap-cotenant",
11755 )
11756 .await;
11757 handler
11758 .handle_control_frame(
11759 &swap.incumbent_ctx,
11760 route_bind_ack(cotenant_bind.header.corr),
11761 )
11762 .await
11763 .unwrap();
11764 assert!(cotenant_task.await.unwrap().is_empty());
11765 let (cotenant_channel, cotenant_epoch) =
11766 published_route(&cotenant_rx.recv().await.unwrap());
11767
11768 let caller = ConnectionId::new(32);
11770 let (caller_ctx, mut caller_rx) = route_ctx(caller);
11771 let (caller_task, caller_bind) = relay_route_open(
11772 &handler,
11773 caller,
11774 &caller_ctx.egress,
11775 &mut swap.incumbent_rx,
11776 101,
11777 "aft",
11778 "swap-caller",
11779 )
11780 .await;
11781 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
11782
11783 register_candidate(&swap, None);
11784 cutover(&swap);
11785
11786 let ack = handler
11788 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
11789 .await;
11790 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
11791 if module_loop_error.is_some() {
11792 handler.cleanup_connection(INCUMBENT).unwrap();
11795 }
11796
11797 assert!(
11799 cotenant_rx.try_recv().is_err(),
11800 "the co-tenant route on the incumbent was torn down by one late ack: \
11801 {module_loop_error:?}"
11802 );
11803 assert!(matches!(
11804 swap.forwarding
11805 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
11806 .unwrap(),
11807 DataRoute::Client(DataRouteState::Bound(_))
11808 ));
11809 assert_eq!(module_loop_error, None);
11810 assert!(swap
11811 .registry
11812 .get_module_by_connection(INCUMBENT)
11813 .unwrap()
11814 .is_some());
11815
11816 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
11818 .await
11819 .expect("the incumbent is told to drop the abandoned binding")
11820 .unwrap()
11821 .frame;
11822 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11823 assert_eq!(goodbye.header.channel, abandoned_channel);
11824 assert_eq!(goodbye.header.epoch, abandoned_epoch);
11825 assert!(swap.incumbent_rx.try_recv().is_err());
11826
11827 let response = caller_task.await.unwrap();
11829 assert_eq!(response.len(), 1);
11830 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
11831 assert!(caller_rx.try_recv().is_err());
11832
11833 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
11837 let counters = handler.counters().snapshot();
11838 assert_eq!(
11839 keyed_total(&counters, "route_open_accepted_by_principal"),
11840 1
11841 );
11842 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
11843 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
11844 }
11845
11846 #[tokio::test]
11850 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
11851 let mut swap = swap_with_incumbent().await;
11852 register_candidate(&swap, None);
11853 let incumbent = cutover(&swap);
11854 swap.forwarding
11855 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
11856 .unwrap()
11857 .expect("the incumbent is still registered");
11858
11859 let client = ConnectionId::new(33);
11860 let (client_ctx, mut client_rx) = route_ctx(client);
11861 let route_handler = swap.handler.clone();
11862 let open_ctx = RouteCtx {
11863 connection_id: client,
11864 egress: client_ctx.egress.clone(),
11865 };
11866 let mut route_task = tokio::spawn(async move {
11867 route_handler
11868 .handle_control_frame(
11869 &open_ctx,
11870 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
11871 )
11872 .await
11873 .unwrap()
11874 });
11875 let bind = tokio::select! {
11876 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
11877 response = &mut route_task => {
11878 let response = response.unwrap();
11879 panic!(
11880 "post-cutover route.open was refused instead of relayed to the candidate: {}",
11881 parse_error(&response[0])["code"]
11882 );
11883 }
11884 };
11885 swap.handler
11886 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
11887 .await
11888 .unwrap();
11889 assert!(route_task.await.unwrap().is_empty());
11890 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
11891 match swap
11892 .forwarding
11893 .lookup_data_route(client, channel, epoch)
11894 .unwrap()
11895 {
11896 DataRoute::Client(DataRouteState::Bound(route)) => {
11897 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
11898 }
11899 other => panic!("expected a bound route on the candidate, got {other:?}"),
11900 }
11901 assert!(swap.incumbent_rx.try_recv().is_err());
11902 }
11903
11904 #[tokio::test]
11909 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
11910 let swap = swap_with_incumbent().await;
11911 register_candidate(&swap, Some(false));
11912 let update = Frame::build(
11913 FrameType::Request,
11914 control_flags(),
11915 0,
11916 0,
11917 55,
11918 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
11919 provides: manifest("aft", PROTOCOL_VERSION).provides,
11920 capabilities: None,
11921 ready: Some(true),
11922 })
11923 .unwrap(),
11924 )
11925 .unwrap();
11926
11927 let replies = swap
11928 .handler
11929 .handle_control_frame(&swap.candidate_ctx, update)
11930 .await
11931 .unwrap();
11932
11933 assert_eq!(replies.len(), 1);
11934 assert_eq!(
11935 replies[0].header.ty,
11936 FrameType::Response,
11937 "candidate catalog.update was refused: {:?}",
11938 serde_json::from_slice::<Value>(&replies[0].body).ok()
11939 );
11940 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
11941 assert_eq!(
11942 swap.registry
11943 .get_module("aft")
11944 .unwrap()
11945 .unwrap()
11946 .connection_id,
11947 INCUMBENT
11948 );
11949 }
11950 }
11951
11952 mod swap_admission {
11956 use super::*;
11957
11958 const INCUMBENT_NONCE: &str = "incumbent-nonce";
11959 const CANDIDATE_NONCE: &str = "candidate-nonce";
11960
11961 fn handler_with_incumbent(
11962 module_id: &str,
11963 reserved: bool,
11964 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
11965 let registry = Arc::new(Registry::default());
11966 let supervisor = SupervisorHandle::new();
11967 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
11968 if reserved {
11969 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
11970 }
11971 let handler =
11972 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
11973 let incumbent = handler
11974 .handle_control(
11975 ConnectionId::new(1),
11976 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
11977 )
11978 .unwrap();
11979 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
11980 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
11981 (registry, supervisor, handler)
11982 }
11983
11984 #[test]
11990 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
11991 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
11992
11993 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
11994 let replies = handler
11995 .handle_control(
11996 ConnectionId::new(connection),
11997 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
11998 )
11999 .unwrap();
12000 assert_eq!(replies[0].header.ty, FrameType::Error);
12001 assert_eq!(
12002 parse_error(&replies[0])["code"],
12003 "swap_token_invalid",
12004 "nonce {nonce:?}"
12005 );
12006 }
12007 assert!(registry.get_candidate("aft").unwrap().is_none());
12008 assert_eq!(
12009 registry.get_module("aft").unwrap().unwrap().connection_id,
12010 ConnectionId::new(1)
12011 );
12012
12013 let admitted = handler
12016 .handle_control(
12017 ConnectionId::new(4),
12018 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
12019 )
12020 .unwrap();
12021 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
12022 assert_eq!(
12023 registry
12024 .get_candidate("aft")
12025 .unwrap()
12026 .unwrap()
12027 .connection_id,
12028 ConnectionId::new(4)
12029 );
12030 assert_eq!(
12031 registry.get_module("aft").unwrap().unwrap().connection_id,
12032 ConnectionId::new(1),
12033 "the candidate must not take the active slot"
12034 );
12035 let replayed = handler
12036 .handle_control(
12037 ConnectionId::new(5),
12038 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
12039 )
12040 .unwrap();
12041 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
12042
12043 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
12047 let squatter = handler
12048 .handle_control(
12049 ConnectionId::new(6),
12050 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
12051 )
12052 .unwrap();
12053 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
12054 assert!(
12055 registry.get_module("aft").unwrap().is_none(),
12056 "a squatter took the active slot of an id being swapped"
12057 );
12058 }
12059
12060 #[test]
12065 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
12066 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
12067
12068 let replies = handler
12069 .handle_control(
12070 ConnectionId::new(2),
12071 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
12072 )
12073 .unwrap();
12074
12075 assert_eq!(
12076 replies[0].header.ty,
12077 FrameType::HelloAck,
12078 "reserved candidate refused: {:?}",
12079 serde_json::from_slice::<Value>(&replies[0].body).ok()
12080 );
12081 assert_eq!(
12082 registry
12083 .get_candidate("vault")
12084 .unwrap()
12085 .unwrap()
12086 .connection_id,
12087 ConnectionId::new(2)
12088 );
12089 }
12090
12091 #[test]
12094 fn without_an_open_swap_the_ordinary_gates_decide() {
12095 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
12096 supervisor.close_swap("vault");
12097
12098 let candidate = handler
12099 .handle_control(
12100 ConnectionId::new(2),
12101 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
12102 )
12103 .unwrap();
12104 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
12105 let duplicate = handler
12106 .handle_control(
12107 ConnectionId::new(3),
12108 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
12109 )
12110 .unwrap();
12111 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
12112 assert!(registry.get_candidate("vault").unwrap().is_none());
12113 }
12114 }
12115
12116 mod scopes {
12120 use subc_protocol::scope::{
12121 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
12122 ScopeStatus,
12123 };
12124
12125 use super::*;
12126
12127 const OWNER: &str = "prefrontal-core";
12128
12129 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
12130 ScopeRecord {
12131 scope_ref: scope_ref.to_string(),
12132 scope_epoch,
12133 kind: ScopeKind::Head,
12134 parent: None,
12135 child_owners: Vec::new(),
12136 carriers: Vec::new(),
12137 attributes: Default::default(),
12138 }
12139 }
12140
12141 async fn call(
12142 handler: &ControlHandler,
12143 ctx: &RouteCtx,
12144 request: &ModuleControlRequestFromModule,
12145 ) -> Frame {
12146 let body = serde_json::to_vec(request).unwrap();
12147 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
12148 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
12149 assert_eq!(replies.len(), 1, "{replies:?}");
12150 replies.pop().unwrap()
12151 }
12152
12153 async fn sync(
12154 handler: &ControlHandler,
12155 ctx: &RouteCtx,
12156 generation: u64,
12157 scopes: Vec<ScopeRecord>,
12158 ) -> Result<ModuleControlResponseToModule, String> {
12159 let reply = call(
12160 handler,
12161 ctx,
12162 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
12163 )
12164 .await;
12165 match reply.header.ty {
12166 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
12167 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
12168 }
12169 }
12170
12171 async fn describe(
12172 handler: &ControlHandler,
12173 ctx: &RouteCtx,
12174 owner: &str,
12175 scope_ref: &str,
12176 ) -> ModuleControlResponseToModule {
12177 let reply = call(
12178 handler,
12179 ctx,
12180 &ModuleControlRequestFromModule::ScopeDescribe {
12181 owner: Principal::Reserved {
12182 module_id: owner.to_string(),
12183 },
12184 scope_ref: scope_ref.to_string(),
12185 },
12186 )
12187 .await;
12188 assert_eq!(
12189 reply.header.ty,
12190 FrameType::Response,
12191 "{:?}",
12192 parse_error(&reply)
12193 );
12194 serde_json::from_slice(&reply.body).unwrap()
12195 }
12196
12197 async fn module(
12199 handler: &ControlHandler,
12200 connection: u64,
12201 module_id: &str,
12202 nonce: Option<&str>,
12203 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
12204 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
12205 hello_via_sink(
12206 handler,
12207 &ctx,
12208 &mut rx,
12209 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
12210 )
12211 .await;
12212 (ctx, rx)
12213 }
12214
12215 #[tokio::test]
12218 async fn a_client_connection_cannot_sync_or_describe() {
12219 let handler = ControlHandler::new(Arc::new(Registry::default()));
12220 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
12221 for request in [
12222 ModuleControlRequestFromModule::ScopeSync {
12223 generation: 1,
12224 scopes: vec![head("s", 1)],
12225 },
12226 ModuleControlRequestFromModule::ScopeDescribe {
12227 owner: Principal::Direct,
12228 scope_ref: "s".to_string(),
12229 },
12230 ] {
12231 let reply = call(&handler, &ctx, &request).await;
12232 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
12233 }
12234 assert!(
12235 !handler
12236 .scopes
12237 .read()
12238 .unwrap()
12239 .describe(
12240 &Principal::Reserved {
12241 module_id: OWNER.to_string()
12242 },
12243 "s"
12244 )
12245 .owner_synced
12246 );
12247 }
12248
12249 #[tokio::test]
12252 async fn a_module_without_a_supervised_launch_cannot_sync() {
12253 let handler = ControlHandler::new(Arc::new(Registry::default()));
12254 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
12255 assert_eq!(
12256 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
12257 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
12258 );
12259 }
12260
12261 #[tokio::test]
12262 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
12263 let supervisor = SupervisorHandle::new();
12264 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
12265 let handler = ControlHandler::new(Arc::new(Registry::default()))
12266 .with_supervisor(supervisor.clone());
12267 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
12268 sync(&handler, &incumbent, 1, vec![head("s", 1)])
12269 .await
12270 .expect("the current launch syncs");
12271
12272 supervisor.open_swap(OWNER, "n2".to_string());
12275 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
12276 assert_eq!(
12277 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
12278 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
12279 );
12280 sync(&handler, &incumbent, 2, vec![head("s", 1)])
12281 .await
12282 .expect("the serving owner syncs during the swap");
12283
12284 supervisor.close_swap(OWNER);
12287 assert_eq!(
12288 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
12289 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
12290 );
12291 sync(&handler, &incumbent, 3, vec![head("s", 1)])
12292 .await
12293 .expect("the serving owner syncs after the rollback");
12294 handler.cleanup_connection(candidate.connection_id).unwrap();
12295
12296 supervisor.open_swap(OWNER, "n3".to_string());
12301 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
12302 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
12303 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
12304 .await
12305 .expect("the promoted launch takes authority");
12306 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
12307 panic!("unexpected reply {reply:?}");
12308 };
12309 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
12310 assert_eq!(
12311 sync(&handler, &incumbent, 4, Vec::new()).await,
12312 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
12313 );
12314 }
12315
12316 #[tokio::test]
12319 async fn closing_the_authority_connection_frees_sync_authority() {
12320 let supervisor = SupervisorHandle::new();
12321 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
12322 let handler = ControlHandler::new(Arc::new(Registry::default()))
12323 .with_supervisor(supervisor.clone());
12324 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
12325 sync(&handler, &first, 10, vec![head("s", 1)])
12326 .await
12327 .unwrap();
12328 handler.cleanup_connection(first.connection_id).unwrap();
12329
12330 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
12331 sync(&handler, &second, 1, vec![head("s", 1)])
12332 .await
12333 .expect("the next connection takes the released authority");
12334 }
12335
12336 #[tokio::test]
12337 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
12338 let supervisor = SupervisorHandle::new();
12339 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
12340 let handler =
12341 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
12342 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
12343 sync(&handler, &first, 10, vec![head("s", 1)])
12344 .await
12345 .unwrap();
12346 handler
12347 .handle_control_frame(
12348 &first,
12349 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
12350 )
12351 .await
12352 .unwrap();
12353 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
12354 sync(&handler, &second, 1, vec![head("s", 1)])
12355 .await
12356 .expect("GOODBYE releases authority even if the old socket remains open");
12357 }
12358
12359 #[tokio::test]
12360 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
12361 let registry = Arc::new(Registry::default());
12362 let supervisor_handle = SupervisorHandle::new();
12363 let supervisor = Supervisor::new(Arc::clone(®istry), RestartPolicy::default())
12364 .with_handle(supervisor_handle.clone())
12365 .with_daemon_incarnation("incarnation-7".to_string());
12366 supervisor
12369 .supervise_configured(
12370 ModuleSpec {
12371 module_id: OWNER.to_string(),
12372 program: PathBuf::from("/nonexistent/prefrontal-core"),
12373 args: Vec::new(),
12374 env: Vec::new(),
12375 reserved: false,
12376 reserved_prefixes: Vec::new(),
12377 protocol: ModuleProtocol::Subc,
12378 overlap: Default::default(),
12379 },
12380 false,
12381 )
12382 .unwrap();
12383 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
12384 let handler =
12385 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12386 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
12387
12388 let ModuleControlResponseToModule::ScopeDescribe {
12390 status,
12391 daemon_incarnation,
12392 owner_synced,
12393 owner_configured,
12394 scope,
12395 ..
12396 } = describe(&handler, &reader, OWNER, "s").await
12397 else {
12398 panic!("not a describe reply");
12399 };
12400 assert_eq!(status, ScopeStatus::NotLive);
12401 assert_eq!(daemon_incarnation, "incarnation-7");
12402 assert!(!owner_synced);
12403 assert!(owner_configured);
12404 assert!(scope.is_none());
12405
12406 let ModuleControlResponseToModule::ScopeDescribe {
12409 status,
12410 owner_configured,
12411 ..
12412 } = describe(&handler, &reader, "ghost", "s").await
12413 else {
12414 panic!("not a describe reply");
12415 };
12416 assert_eq!(status, ScopeStatus::NotLive);
12417 assert!(!owner_configured);
12418
12419 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
12421 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
12422 let ModuleControlResponseToModule::ScopeDescribe {
12423 status,
12424 scope_epoch,
12425 owner_synced,
12426 scope,
12427 ..
12428 } = describe(&handler, &reader, OWNER, "s").await
12429 else {
12430 panic!("not a describe reply");
12431 };
12432 assert_eq!(status, ScopeStatus::Live);
12433 assert_eq!(scope_epoch, Some(4));
12434 assert!(owner_synced);
12435 let stamp = scope.expect("a live scope carries its stamp");
12436 assert!(
12437 stamp.owner_authorized,
12438 "prefrontal-core is the default authority"
12439 );
12440 assert_eq!(stamp.kind, ScopeKind::Head);
12441 }
12442
12443 #[tokio::test]
12444 async fn scope_authority_owners_decides_owner_authorized() {
12445 let supervisor = SupervisorHandle::new();
12446 supervisor.set_spawn_nonce("broca", "b1".to_string());
12447 let handler = ControlHandler::new(Arc::new(Registry::default()))
12448 .with_supervisor(supervisor)
12449 .with_scope_authority_owners(vec!["broca".to_string()]);
12450 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
12451 let mut gated = head("s", 1);
12452 gated.attributes.agent_id = Some("agent".to_string());
12453 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
12454 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
12455 describe(&handler, &broca, "broca", "s").await
12456 else {
12457 panic!("not a describe reply");
12458 };
12459 assert!(scope.unwrap().owner_authorized);
12460 }
12461
12462 #[tokio::test]
12466 async fn scope_ops_and_the_scopes_capability_are_advertised() {
12467 let handler = ControlHandler::new(Arc::new(Registry::default()));
12468 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
12469 let ack = hello_via_sink(
12470 &handler,
12471 &ctx,
12472 &mut rx,
12473 hello_frame("m", PROTOCOL_VERSION, 1),
12474 )
12475 .await;
12476 let ack = parse_ack(&ack);
12477 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
12478 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
12479 }
12480 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
12481
12482 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
12483 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12484 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12485 let reply = handler
12486 .handle_control_frame(&client, frame)
12487 .await
12488 .unwrap()
12489 .pop()
12490 .unwrap();
12491 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12492 serde_json::from_slice(&reply.body).unwrap()
12493 else {
12494 panic!("not a server.describe reply");
12495 };
12496 assert!(
12497 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
12498 "{capabilities:?}"
12499 );
12500 }
12501
12502 const PLEXUS: &str = "plexus";
12505 const OTHER: &str = "other";
12506 const AFT: &str = "aft";
12507 const BROCA: &str = "broca";
12508 const MAGIC: &str = "magic-context";
12509
12510 fn nonce(module_id: &str) -> String {
12511 format!("nonce-{module_id}")
12512 }
12513
12514 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
12515 let (tx, rx) = mpsc::channel(64);
12516 (
12517 RouteCtx {
12518 connection_id: ConnectionId::new(connection),
12519 egress: FrameSink::new(tx),
12520 },
12521 rx,
12522 )
12523 }
12524
12525 struct Rig {
12529 handler: ControlHandler,
12530 forwarding: Arc<ForwardingTable>,
12531 owner: RouteCtx,
12532 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12533 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
12534 generation: u64,
12535 next_connection: u64,
12536 _supervisor: Supervisor,
12537 }
12538
12539 async fn rig() -> Rig {
12540 rig_with_flow_support(true).await
12541 }
12542
12543 async fn rig_with_flow_support(flow_support: bool) -> Rig {
12544 let registry = Arc::new(Registry::default());
12545 let forwarding = Arc::new(ForwardingTable::default());
12546 let supervisor_handle = SupervisorHandle::new();
12547 let supervisor = Supervisor::new(Arc::clone(®istry), RestartPolicy::default())
12548 .with_handle(supervisor_handle.clone());
12549 supervisor
12550 .supervise_configured(
12551 ModuleSpec {
12552 module_id: OWNER.to_string(),
12553 program: PathBuf::from("/nonexistent/prefrontal-core"),
12554 args: Vec::new(),
12555 env: Vec::new(),
12556 reserved: false,
12557 reserved_prefixes: Vec::new(),
12558 protocol: ModuleProtocol::Subc,
12559 overlap: Default::default(),
12560 },
12561 false,
12562 )
12563 .unwrap();
12564 for module_id in [OWNER, AFT, BROCA, MAGIC] {
12565 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
12566 }
12567 let handler =
12568 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12569 .with_supervisor(supervisor_handle);
12570 let (owner, mut owner_rx) = wide_ctx(1);
12571 hello_via_sink(
12572 &handler,
12573 &owner,
12574 &mut owner_rx,
12575 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
12576 )
12577 .await;
12578 let mut modules = BTreeMap::new();
12579 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
12580 let (ctx, mut rx) = wide_ctx(connection);
12581 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
12582 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
12583 body["manifest"]["provenance"] =
12587 serde_json::json!({"wire_crate_version": "0.29.0"});
12588 if flow_support {
12589 body["manifest"]["capabilities"] =
12590 serde_json::json!({"provides": ["flow-scopes/v1"]});
12591 }
12592 let hello = Frame::build(
12593 FrameType::Hello,
12594 control_flags(),
12595 0,
12596 0,
12597 connection,
12598 serde_json::to_vec(&body).unwrap(),
12599 )
12600 .unwrap();
12601 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
12602 modules.insert(module_id.to_string(), (ctx, rx));
12603 }
12604 Rig {
12605 handler,
12606 forwarding,
12607 owner,
12608 _owner_rx: owner_rx,
12609 modules,
12610 generation: 0,
12611 next_connection: 100,
12612 _supervisor: supervisor,
12613 }
12614 }
12615
12616 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
12617 ScopeCarrier {
12618 principal: Principal::Reserved {
12619 module_id: module_id.to_string(),
12620 },
12621 targets: targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()),
12622 }
12623 }
12624
12625 fn session(scope_epoch: u64) -> ScopeRecord {
12628 let mut record = head("s", scope_epoch);
12629 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
12630 record.attributes.agent_id = Some("agent-1".to_string());
12631 record.attributes.delegates = true;
12632 record
12633 }
12634
12635 impl Rig {
12636 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
12637 self.generation += 1;
12638 sync(&self.handler, &self.owner, self.generation, scopes)
12639 .await
12640 .expect("the owner's sync is accepted");
12641 }
12642
12643 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
12644 ScopeSelector {
12645 owner: Principal::Reserved {
12646 module_id: OWNER.to_string(),
12647 },
12648 scope_ref: scope_ref.to_string(),
12649 scope_epoch,
12650 }
12651 }
12652
12653 fn open_frame(
12654 &mut self,
12655 opener: Option<&str>,
12656 target: &str,
12657 scope: Option<ScopeSelector>,
12658 ) -> (
12659 RouteCtx,
12660 mpsc::Receiver<crate::router::OutboundFrame>,
12661 Frame,
12662 ) {
12663 self.next_connection += 1;
12664 let (ctx, rx) = wide_ctx(self.next_connection);
12665 let root = unique_project_root("scoped-open");
12666 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
12667 target: RouteTarget::ToolProvider {
12668 module_id: target.to_string(),
12669 },
12670 identity: BindIdentity::new(
12671 root.path().to_path_buf(),
12672 "unit".to_string(),
12673 "session".to_string(),
12674 ),
12675 consumer_identity: opener.map(|module_id| ConsumerIdentity {
12676 module_id: module_id.to_string(),
12677 launch_nonce: nonce(module_id),
12678 }),
12679 consumer_capabilities: None,
12680 role_versions: None,
12681 admission_facts: None,
12682 scope,
12683 })
12684 .unwrap();
12685 let frame = Frame::build(
12686 FrameType::Request,
12687 control_flags(),
12688 0,
12689 0,
12690 self.next_connection,
12691 body,
12692 )
12693 .unwrap();
12694 (ctx, rx, frame)
12695 }
12696
12697 async fn refused(
12699 &mut self,
12700 opener: Option<&str>,
12701 target: &str,
12702 scope: Option<ScopeSelector>,
12703 ) -> String {
12704 self.refusal_body(opener, target, scope).await["code"]
12705 .as_str()
12706 .unwrap()
12707 .to_string()
12708 }
12709
12710 async fn refusal_body(
12711 &mut self,
12712 opener: Option<&str>,
12713 target: &str,
12714 scope: Option<ScopeSelector>,
12715 ) -> Value {
12716 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
12717 let replies = tokio::time::timeout(
12718 Duration::from_secs(2),
12719 self.handler.handle_control_frame(&ctx, frame),
12720 )
12721 .await
12722 .expect("the open must be refused before waiting for a bind ack")
12723 .unwrap();
12724 assert_eq!(replies.len(), 1, "{replies:?}");
12725 assert_eq!(replies[0].header.ty, FrameType::Error);
12726 let (_, module_rx) = self.modules.get_mut(target).unwrap();
12727 assert!(
12728 module_rx.try_recv().is_err(),
12729 "a refused open relays nothing"
12730 );
12731 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
12732 parse_error(&replies[0])
12733 }
12734
12735 async fn relayed(
12737 &mut self,
12738 opener: Option<&str>,
12739 target: &str,
12740 scope: Option<ScopeSelector>,
12741 ) -> Relayed {
12742 let (ctx, rx, frame) = self.open_frame(opener, target, scope);
12743 let handler = self.handler.clone();
12744 let task_ctx = ctx.clone();
12745 let task = tokio::spawn(async move {
12746 handler
12747 .handle_control_frame(&task_ctx, frame)
12748 .await
12749 .unwrap()
12750 });
12751 let (_, module_rx) = self.modules.get_mut(target).unwrap();
12752 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
12753 .await
12754 .expect("the target receives the relayed route.bind")
12755 .unwrap()
12756 .frame;
12757 Relayed {
12758 target: target.to_string(),
12759 client: ctx,
12760 client_rx: rx,
12761 task,
12762 bind,
12763 }
12764 }
12765
12766 async fn ack(&self, relayed: &Relayed) {
12767 let (module, _) = &self.modules[&relayed.target];
12768 self.handler
12769 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
12770 .await
12771 .unwrap();
12772 }
12773
12774 async fn bound(
12776 &mut self,
12777 opener: Option<&str>,
12778 target: &str,
12779 scope: Option<ScopeSelector>,
12780 ) -> Bound {
12781 let relayed = self.relayed(opener, target, scope).await;
12782 self.ack(&relayed).await;
12783 let Relayed {
12784 target,
12785 client,
12786 mut client_rx,
12787 task,
12788 bind,
12789 } = relayed;
12790 assert!(
12791 task.await.unwrap().is_empty(),
12792 "the open is answered by commit"
12793 );
12794 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
12795 Bound {
12796 target,
12797 client,
12798 client_rx,
12799 channel,
12800 epoch,
12801 bind,
12802 }
12803 }
12804
12805 fn live(&self, route: &Bound) -> bool {
12806 matches!(
12807 self.forwarding
12808 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
12809 .unwrap(),
12810 DataRoute::Client(DataRouteState::Bound(_))
12811 )
12812 }
12813 }
12814
12815 struct Relayed {
12816 target: String,
12817 client: RouteCtx,
12818 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12819 task: tokio::task::JoinHandle<Vec<Frame>>,
12820 bind: Frame,
12821 }
12822
12823 struct Bound {
12824 target: String,
12825 client: RouteCtx,
12826 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12827 channel: u16,
12828 epoch: u32,
12829 bind: Frame,
12830 }
12831
12832 impl Bound {
12833 fn closed_reason(&mut self) -> RouteCloseReason {
12836 let mut reason = None;
12837 let mut goodbye = false;
12838 while let Ok(outbound) = self.client_rx.try_recv() {
12839 let frame = outbound.frame;
12840 match frame.header.ty {
12841 FrameType::Goodbye => {
12842 assert_eq!(
12843 (frame.header.channel, frame.header.epoch),
12844 (self.channel, self.epoch)
12845 );
12846 goodbye = true;
12847 }
12848 FrameType::Push => {
12849 let ClientControlPush::RouteClosed {
12850 reason: r,
12851 module_id,
12852 ..
12853 } = serde_json::from_slice(&frame.body).unwrap()
12854 else {
12855 panic!("unexpected push");
12856 };
12857 assert_eq!(module_id, self.target);
12858 reason = Some(r);
12859 }
12860 other => panic!("unexpected frame {other:?}"),
12861 }
12862 }
12863 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
12864 reason.expect("the client is told why the route closed")
12865 }
12866
12867 fn untouched(&mut self) -> bool {
12868 self.client_rx.try_recv().is_err()
12869 }
12870
12871 fn stamp(&self) -> Option<ScopeStamp> {
12872 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
12873 ModuleControlRequest::RouteBind { scope, .. } => scope,
12874 other => panic!("expected a route.bind, got {other:?}"),
12875 }
12876 }
12877 }
12878
12879 #[tokio::test]
12880 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
12881 ) {
12882 let mut rig = rig().await;
12883 let mut record = session(1);
12884 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
12885 record.child_owners = vec![Principal::Reserved {
12886 module_id: MAGIC.to_string(),
12887 }];
12888 rig.sync(vec![record]).await;
12889 let scope = || Some(rig_selector("s", Some(1)));
12890
12891 rig.bound(Some(OWNER), PLEXUS, scope()).await;
12894 rig.bound(Some(AFT), OTHER, scope()).await;
12895 rig.bound(Some(BROCA), PLEXUS, scope()).await;
12896
12897 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
12901 assert_eq!(
12902 rig.refused(opener, target, scope()).await,
12903 error_codes::SCOPE_NOT_CARRIER,
12904 "{opener:?} -> {target}"
12905 );
12906 }
12907 }
12908
12909 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
12910 ScopeSelector {
12911 owner: Principal::Reserved {
12912 module_id: OWNER.to_string(),
12913 },
12914 scope_ref: scope_ref.to_string(),
12915 scope_epoch,
12916 }
12917 }
12918
12919 #[tokio::test]
12920 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
12921 let mut rig = rig().await;
12922 rig.sync(vec![session(1)]).await;
12923 for opener in [OWNER, AFT] {
12924 assert_eq!(
12925 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
12926 .await,
12927 error_codes::SCOPE_EPOCH_REQUIRED,
12928 "{opener}"
12929 );
12930 }
12931 }
12932
12933 #[tokio::test]
12934 async fn admission_separates_not_synced_not_live_and_ended() {
12935 let mut rig = rig().await;
12936 let code = rig
12938 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12939 .await;
12940 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
12941 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
12942
12943 let ghost = ScopeSelector {
12945 owner: Principal::Reserved {
12946 module_id: "ghost".to_string(),
12947 },
12948 scope_ref: "s".to_string(),
12949 scope_epoch: Some(1),
12950 };
12951 assert_eq!(
12952 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
12953 error_codes::SCOPE_NOT_LIVE
12954 );
12955
12956 rig.sync(vec![session(2)]).await;
12957 assert_eq!(
12958 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
12959 .await,
12960 error_codes::SCOPE_NOT_LIVE
12961 );
12962 for epoch in [1, 3] {
12963 assert_eq!(
12964 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
12965 .await,
12966 error_codes::SCOPE_ENDED,
12967 "epoch {epoch}"
12968 );
12969 }
12970 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
12972 .await;
12973 }
12974
12975 #[tokio::test]
12976 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
12977 let mut rig = rig().await;
12978 rig.sync(vec![session(1)]).await;
12979 let route = rig
12980 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12981 .await;
12982 let stamp = route.stamp().expect("a scoped bind carries the stamp");
12983 assert_eq!(stamp.scope_ref, "s");
12984 assert_eq!(stamp.scope_epoch, 1);
12985 assert_eq!(stamp.kind, ScopeKind::Head);
12986 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
12987 assert!(stamp.attributes.delegates);
12988 assert!(stamp.owner_authorized);
12989 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
12990 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
12991
12992 let (broca, mut broca_rx) = wide_ctx(50);
12995 hello_via_sink(
12996 &rig.handler,
12997 &broca,
12998 &mut broca_rx,
12999 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
13000 )
13001 .await;
13002 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
13003 .await
13004 .unwrap();
13005 let own = ScopeSelector {
13006 owner: Principal::Reserved {
13007 module_id: BROCA.to_string(),
13008 },
13009 scope_ref: "b".to_string(),
13010 scope_epoch: Some(1),
13011 };
13012 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
13013 assert!(!route.stamp().unwrap().owner_authorized);
13014 }
13015
13016 #[tokio::test]
13017 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
13018 let mut rig = rig().await;
13019 let mut record = head("s", 1);
13020 record.carriers = vec![carrier(AFT, None)];
13021 let flow_id = "Flow:run-7/step_2!~";
13022 record.attributes.flow_id = Some(flow_id.to_string());
13023 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
13024 .await
13025 .unwrap();
13026 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
13027 panic!("not a sync reply");
13028 };
13029 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
13030 let route = rig
13031 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13032 .await;
13033 assert!(rig.live(&route), "the stamped bind committed");
13034 let stamp = route.stamp().expect("a flow scope carries a stamp");
13035 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
13036 assert_eq!(stamp.attributes.agent_id, None);
13037 assert!(!stamp.attributes.delegates);
13038 assert!(stamp.owner_authorized);
13039 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
13040 assert_eq!(unscoped.stamp(), None);
13041 }
13042
13043 #[tokio::test]
13044 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
13045 let mut rig = rig_with_flow_support(false).await;
13046 let mut record = session(1);
13047 record.attributes.flow_id = Some("flow:7".to_string());
13048 rig.sync(vec![record]).await;
13049 for opener in [OWNER, AFT] {
13050 let body = rig
13051 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
13052 .await;
13053 assert_eq!(body["code"], "target_flow_unsupported");
13054 let message = body["message"].as_str().unwrap();
13055 for required in [PLEXUS, "flow-scopes/v1"] {
13056 assert!(message.contains(required), "{message}");
13057 }
13058 }
13059 }
13060
13061 #[tokio::test]
13062 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
13063 let mut rig = rig_with_flow_support(true).await;
13064 let mut record = session(1);
13065 record.attributes.flow_id = Some("flow:7".to_string());
13066 rig.sync(vec![record]).await;
13067 for opener in [OWNER, AFT] {
13068 let route = rig
13069 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
13070 .await;
13071 assert!(rig.live(&route));
13072 assert_eq!(
13073 route.stamp().unwrap().attributes.flow_id.as_deref(),
13074 Some("flow:7")
13075 );
13076 }
13077 }
13078
13079 #[tokio::test]
13080 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
13081 use std::future::Future;
13082
13083 let mut rig = rig_with_flow_support(true).await;
13084 let mut record = session(1);
13085 record.attributes.flow_id = Some("flow:7".to_string());
13086 rig.sync(vec![record]).await;
13087 let (client, mut client_rx, frame) =
13088 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
13089 for _ in 0..64 {
13092 client.egress.try_send(route_bind_ack(1)).unwrap();
13093 }
13094 let handler = rig.handler.clone();
13095 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
13096 std::future::poll_fn(|cx| {
13097 assert!(open.as_mut().poll(cx).is_pending());
13098 std::task::Poll::Ready(())
13099 })
13100 .await;
13101 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
13102
13103 let old_connection = rig.modules[PLEXUS].0.connection_id;
13104 rig.handler.cleanup_connection(old_connection).unwrap();
13105 let (replacement, mut replacement_rx) = wide_ctx(200);
13106 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
13107 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
13108 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
13109 let hello = Frame::build(
13110 FrameType::Hello,
13111 control_flags(),
13112 0,
13113 0,
13114 200,
13115 serde_json::to_vec(&body).unwrap(),
13116 )
13117 .unwrap();
13118 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
13119
13120 client_rx.try_recv().unwrap();
13121 let replies = tokio::time::timeout(Duration::from_secs(2), open)
13122 .await
13123 .expect("the replacement is refused without waiting for a bind ack")
13124 .unwrap();
13125 assert_eq!(replies.len(), 1);
13126 let body = parse_error(&replies[0]);
13127 assert_eq!(body["code"], "target_flow_unsupported");
13128 for required in [PLEXUS, "flow-scopes/v1"] {
13129 assert!(body["message"].as_str().unwrap().contains(required));
13130 }
13131 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
13132 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
13133 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
13134 assert!(rig
13135 .handler
13136 .registry
13137 .get_module_by_connection(replacement.connection_id)
13138 .unwrap()
13139 .is_some());
13140 }
13141
13142 #[tokio::test]
13143 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
13144 {
13145 let mut rig = rig_with_flow_support(false).await;
13146 rig.sync(vec![session(1)]).await;
13147 let route = rig
13148 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13149 .await;
13150 assert!(rig.live(&route));
13151 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
13152 let mut record = session(1);
13153 record.attributes.flow_id = Some("flow:7".to_string());
13154 rig.sync(vec![record]).await;
13155 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
13156 assert!(rig.live(&unscoped));
13157 assert_eq!(unscoped.stamp(), None);
13158 }
13159
13160 #[tokio::test]
13161 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
13162 let mut rig = rig().await;
13163 let mut record = session(1);
13164 record.attributes.flow_id = Some("flow:7".to_string());
13165 rig.sync(vec![record.clone()]).await;
13166 let before = rig.forwarding.published_scope_tag(OWNER, "s");
13167 let mut owner_route = rig
13168 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
13169 .await;
13170 let mut carrier_route = rig
13171 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
13172 .await;
13173 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
13174 rig.sync(vec![record.clone()]).await;
13175 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
13176 assert!(rig.live(&owner_route) && owner_route.untouched());
13177 assert!(rig.live(&carrier_route) && carrier_route.untouched());
13178
13179 record.attributes.flow_id = Some("flow:8".to_string());
13180 rig.sync(vec![record]).await;
13181 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
13182 let before = before.unwrap();
13183 assert_eq!(after.scope_epoch, before.scope_epoch);
13184 assert!(after.version > before.version);
13185 for route in [&mut owner_route, &mut carrier_route] {
13186 assert!(!rig.live(route));
13187 assert_eq!(
13188 route.closed_reason(),
13189 RouteCloseReason::ScopeDelegationChanged
13190 );
13191 }
13192 assert!(rig.live(&unscoped) && unscoped.untouched());
13193 for target in [PLEXUS, OTHER] {
13196 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
13197 let goodbye = module_rx
13198 .try_recv()
13199 .expect("the provider sees the drain")
13200 .frame;
13201 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
13202 assert!(module_rx.try_recv().is_err());
13203 }
13204 let rebound = rig
13205 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13206 .await;
13207 assert_eq!(
13208 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
13209 Some("flow:8")
13210 );
13211 }
13212
13213 #[tokio::test]
13218 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
13219 let mut rig = rig().await;
13220 rig.sync(vec![session(1)]).await;
13221 let mut cotenant = rig
13222 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
13223 .await;
13224
13225 let mut changed = session(1);
13226 changed.child_owners.push(Principal::Reserved {
13227 module_id: MAGIC.to_string(),
13228 });
13229 let mut ended = None;
13230 for (code, next) in [
13231 (error_codes::SCOPE_CHANGED, vec![changed]),
13232 (error_codes::SCOPE_ENDED, Vec::new()),
13233 ] {
13234 let relayed = rig
13235 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13236 .await;
13237 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
13238 ended = Some(next.is_empty());
13239 rig.sync(next).await;
13240 rig.ack(&relayed).await;
13241 let replies = relayed.task.await.unwrap();
13242 assert_eq!(replies.len(), 1, "{replies:?}");
13243 assert_eq!(parse_error(&replies[0])["code"], code);
13244 assert_eq!(
13245 subc_protocol::error_codes::is_retryable_route_open(code),
13246 code == error_codes::SCOPE_CHANGED
13247 );
13248 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
13249 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
13251 let mut goodbyes = Vec::new();
13254 while let Ok(outbound) = plexus_rx.try_recv() {
13255 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
13256 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
13257 }
13258 assert!(
13259 goodbyes.contains(&(bind_channel, bind_epoch)),
13260 "{goodbyes:?}"
13261 );
13262 assert!(rig
13263 .handler
13264 .registry
13265 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
13266 .unwrap()
13267 .is_some());
13268 }
13269 assert_eq!(ended, Some(true));
13270 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
13273 }
13274
13275 #[tokio::test]
13278 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
13279 struct Case {
13280 name: &'static str,
13281 change: fn(&mut ScopeRecord),
13282 closed: [Option<RouteCloseReason>; 4],
13285 }
13286 use RouteCloseReason::*;
13287 let cases = [
13288 Case {
13289 name: "a carrier entry removed",
13290 change: |r| {
13291 r.carriers.retain(|c| {
13292 c.principal
13293 != Principal::Reserved {
13294 module_id: AFT.to_string(),
13295 }
13296 })
13297 },
13298 closed: [None, Some(ScopeCarrierRemoved), None, None],
13299 },
13300 Case {
13301 name: "a target removed from a carrier",
13302 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
13303 closed: [None, None, None, Some(ScopeCarrierRemoved)],
13304 },
13305 Case {
13306 name: "a bare carrier narrowed to targets",
13307 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
13308 closed: [None, Some(ScopeCarrierRemoved), None, None],
13309 },
13310 Case {
13311 name: "delegates turned off",
13312 change: |r| r.attributes.delegates = false,
13313 closed: [Some(ScopeDelegationChanged); 4],
13314 },
13315 Case {
13316 name: "agent_id changed",
13317 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
13318 closed: [Some(ScopeDelegationChanged); 4],
13319 },
13320 Case {
13321 name: "a carrier added, child owners changed, the record re-sent",
13322 change: |r| {
13323 r.carriers.push(carrier(MAGIC, None));
13324 r.child_owners.push(Principal::Reserved {
13325 module_id: MAGIC.to_string(),
13326 });
13327 },
13328 closed: [None; 4],
13329 },
13330 Case {
13331 name: "a target added",
13332 change: |r| {
13333 r.carriers[1]
13334 .targets
13335 .as_mut()
13336 .unwrap()
13337 .push("third".to_string())
13338 },
13339 closed: [None; 4],
13340 },
13341 Case {
13342 name: "delegates turned on",
13343 change: |r| r.attributes.delegates = true,
13344 closed: [None; 4],
13345 },
13346 ];
13347 for case in cases {
13348 let mut rig = rig().await;
13349 rig.sync(vec![session(1)]).await;
13350 let scope = || Some(rig_selector("s", Some(1)));
13351 let mut routes = [
13352 rig.bound(Some(OWNER), PLEXUS, scope()).await,
13353 rig.bound(Some(AFT), PLEXUS, scope()).await,
13354 rig.bound(Some(BROCA), PLEXUS, scope()).await,
13355 rig.bound(Some(BROCA), OTHER, scope()).await,
13356 ];
13357 let mut record = session(1);
13358 (case.change)(&mut record);
13359 rig.sync(vec![record]).await;
13360 for (index, expected) in case.closed.iter().enumerate() {
13361 let route = &mut routes[index];
13362 match expected {
13363 Some(reason) => {
13364 assert!(!rig.live(route), "{}: route {index} still live", case.name);
13365 assert_eq!(
13366 route.closed_reason(),
13367 *reason,
13368 "{}: route {index}",
13369 case.name
13370 );
13371 }
13372 None => {
13373 assert!(rig.live(route), "{}: route {index} closed", case.name);
13374 assert!(
13375 route.untouched(),
13376 "{}: route {index} was told something",
13377 case.name
13378 );
13379 }
13380 }
13381 }
13382 }
13383 }
13384
13385 #[tokio::test]
13386 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
13387 for next in [Vec::new(), vec![session(2)]] {
13389 let mut rig = rig().await;
13390 rig.sync(vec![session(1)]).await;
13391 let mut route = rig
13392 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13393 .await;
13394 rig.sync(next).await;
13395 assert!(!rig.live(&route));
13396 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
13397 }
13398
13399 let mut rig = rig().await;
13402 let mut child = session(1);
13403 child.scope_ref = "child".to_string();
13404 child.kind = ScopeKind::Worker;
13405 child.parent = Some(ScopeParent {
13406 owner: Principal::Reserved {
13407 module_id: OWNER.to_string(),
13408 },
13409 scope_ref: "s".to_string(),
13410 scope_epoch: 1,
13411 });
13412 rig.sync(vec![session(1), child.clone()]).await;
13413 let mut child_route = rig
13414 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
13415 .await;
13416 assert_eq!(
13417 child_route.stamp().unwrap().parent_state,
13418 Some(ParentState::Linked)
13419 );
13420 rig.sync(vec![child]).await;
13421 assert!(!rig.live(&child_route));
13422 assert_eq!(
13423 child_route.closed_reason(),
13424 RouteCloseReason::ScopeParentEnded
13425 );
13426 }
13427
13428 #[tokio::test]
13429 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
13430 ) {
13431 let mut rig = rig().await;
13432 rig.sync(vec![session(1)]).await;
13433 let mut route = rig
13434 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13435 .await;
13436 let before = rig.forwarding.published_scope_tag(OWNER, "s");
13437 rig.sync(vec![session(1)]).await;
13438 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
13439 assert!(rig.live(&route) && route.untouched());
13440
13441 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
13445 .forwarding
13446 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
13447 .unwrap()
13448 else {
13449 panic!("the route is bound");
13450 };
13451 binding.flow.acquire_tagged(9, false).await.unwrap();
13452 let mut widened = session(1);
13453 widened.carriers.push(carrier(MAGIC, None));
13454 rig.sync(vec![widened]).await;
13455 assert!(rig.live(&route) && route.untouched());
13456 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
13457 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
13458 assert_eq!(binding.flow.in_flight(), 1);
13461 binding
13462 .flow
13463 .acquire_tagged(10, false)
13464 .await
13465 .expect("the flow is still open");
13466 }
13467
13468 #[tokio::test]
13471 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
13472 let mut rig = rig().await;
13473 rig.sync(vec![session(1)]).await;
13474 let mut on_incumbent = rig
13475 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13476 .await;
13477
13478 let (candidate, _candidate_rx) = wide_ctx(9);
13481 let registration = rig
13482 .handler
13483 .registry
13484 .register_candidate_with_control_ops(
13485 manifest(PLEXUS, PROTOCOL_VERSION),
13486 PROTOCOL_VERSION,
13487 candidate.connection_id,
13488 module_baseline_control_ops(),
13489 )
13490 .unwrap();
13491 rig.forwarding
13492 .register_candidate_module_connection(
13493 candidate.connection_id,
13494 PLEXUS.to_string(),
13495 PROTOCOL_VERSION,
13496 manifest_concurrency(®istration.manifest),
13497 candidate.egress.clone(),
13498 )
13499 .unwrap();
13500 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
13501 rig.handler
13502 .registry
13503 .promote_candidate(PLEXUS)
13504 .unwrap()
13505 .unwrap();
13506 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
13507
13508 rig.sync(Vec::new()).await;
13509 assert!(!rig.live(&on_incumbent));
13510 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
13511 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
13512 let goodbye = incumbent_rx
13513 .try_recv()
13514 .expect("the superseded endpoint is told")
13515 .frame;
13516 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
13517 }
13518 }
13519}
13520
13521#[cfg(test)]
13522mod concurrency_default_exposure_tests {
13523 use super::*;
13524
13525 fn hello_body(role_json: &str) -> Vec<u8> {
13526 format!(
13527 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
13528 )
13529 .into_bytes()
13530 }
13531
13532 fn manifest_from(body: &[u8]) -> ModuleManifest {
13533 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
13534 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
13535 .expect("manifest parses")
13536 }
13537
13538 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
13539
13540 #[test]
13541 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
13542 let body = hello_body(&format!(
13543 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
13544 ));
13545 let manifest = manifest_from(&body);
13546 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
13549 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
13550 }
13551
13552 #[test]
13553 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
13554 let body = hello_body(&format!(
13555 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
13556 ));
13557 let manifest = manifest_from(&body);
13558 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
13559 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
13560 }
13561
13562 #[test]
13563 fn non_management_roles_are_never_reported() {
13564 let body = hello_body(
13565 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
13566 );
13567 let manifest = manifest_from(&body);
13568 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
13569 }
13570}