1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 path::{Path, PathBuf},
5 sync::{Arc, Mutex, RwLock},
6 time::{Duration, Instant as StdInstant},
7};
8
9use serde::{Deserialize, Serialize};
10use subc_control::{
11 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
12 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
13 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
14 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
15 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
16 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
17 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
18};
19use subc_protocol::{
20 error_codes,
21 manifest::{
22 validate_hello_capability_grammar, validate_hello_self_signal_declarations,
23 CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
24 ProviderRole,
25 },
26 scope::{
27 ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
28 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_DESCRIBE_OP, SCOPE_SYNC_OP,
29 },
30 session::{
31 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
32 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
33 MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE, ROLE_VERSIONS_FIELD,
34 },
35 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
36 Priority, RouteTarget, PROTOCOL_VERSION,
37};
38use tokio::time::{timeout_at, Instant};
39use tracing::{debug, info, warn};
40
41use crate::{
42 capability_requirements::{
43 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
44 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
45 RuntimeModule,
46 },
47 daemon_config::RestartRequiredSection,
48 forwarding::{
49 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
50 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
51 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
52 },
53 observability::{
54 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
55 },
56 provenance::{
57 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
58 },
59 registry::{ChannelState, ConnectionId, Registry, RegistryError},
60 router::{RouteCtx, RouterError},
61 scopes::{BoundScope, HelloLaunchNonces, ScopeTable},
62 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
63 stderr_tail::{CaptureState, TailEntry},
64 supervise::{
65 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
66 SupervisorHandle, SwapHelloAdmission,
67 },
68 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
69};
70
71pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
77
78const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
79const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
80const CAP_PING_PONG: &str = "ping_pong_v1";
81const CAP_SESSION_ATTACH: &str = "session_attach_v1";
82const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
83
84const SUBC_CONTROL_OPS: &[&str] = &[
85 ops::SERVER_DESCRIBE,
86 ops::CATALOG_LIST,
87 ops::ROUTE_OPEN,
88 ops::ROUTE_POLL,
89 ops::ROUTE_CLOSING,
90 ops::ROUTE_CLOSED,
91 ops::SUPERVISOR_LIST,
92 ops::SUPERVISOR_RESTART,
93 ops::SUPERVISOR_SWAP,
94 ops::SUPERVISOR_RELOAD,
95 ops::SUPERVISOR_RESCAN,
96 ops::SUPERVISOR_RELEASE_RESERVED,
97 ops::SUPERVISOR_SET_ENABLED,
98 ops::SUPERVISOR_HEALTH_PROBE,
99 ops::SUPERVISOR_HEALTH,
100 ops::SUPERVISOR_STDERR_TAIL,
101 ops::SUPERVISOR_TERMINALS,
102 ops::SUPERVISOR_ROUTES,
103 ops::SUPERVISOR_PROVENANCE,
104 ops::SUPERVISOR_SPAWN_SNAPSHOT,
105 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
106];
107
108const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
109 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
110 "supervisor.live_roots",
111 SCOPE_SYNC_OP,
112 SCOPE_DESCRIBE_OP,
113];
114
115const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
119
120const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
121
122pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
130
131pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
146
147pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
165
166const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
167const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
168
169fn reload_verdict(
170 configured: &Path,
171 spawned_from: Option<&Path>,
172 image: subc_control::RunningImageAgreement,
173) -> PendingReloadVerdict {
174 let path = match spawned_from {
175 Some(spawned_from) if configured == spawned_from => ReloadPathAgreement::Match,
176 Some(spawned_from) => ReloadPathAgreement::Mismatch {
177 configured: configured.to_path_buf(),
178 spawned_from: spawned_from.to_path_buf(),
179 },
180 None => ReloadPathAgreement::Unavailable {
181 reason: if matches!(
182 image,
183 subc_control::RunningImageAgreement::Unavailable {
184 reason: subc_control::RunningImageUnavailableReason::NotRunning
185 }
186 ) {
187 ReloadPathUnavailableReason::NotRunning
188 } else {
189 ReloadPathUnavailableReason::SpawnedPathUnavailable
190 },
191 },
192 };
193 PendingReloadVerdict { path, image }
194}
195
196#[derive(Clone)]
197struct DaemonProvenanceFacts {
198 build: DaemonBuildProvenance,
199 pid: Option<u32>,
200 started_at_ms: Option<u64>,
201 start_clock: Option<crate::clock::StartClock>,
202 executable_path: Option<PathBuf>,
203 executable_identity: Option<SpawnedFileIdentity>,
204 process_start_time: Option<u64>,
205 probe: ExecutableIdentityProbe,
206}
207
208impl Default for DaemonProvenanceFacts {
209 fn default() -> Self {
210 Self {
211 build: DaemonBuildProvenance {
212 build_git_sha: None,
213 build_lock_digest: None,
214 },
215 pid: None,
216 started_at_ms: None,
217 start_clock: None,
218 executable_path: None,
219 executable_identity: None,
220 process_start_time: None,
221 probe: ExecutableIdentityProbe::default(),
222 }
223 }
224}
225
226#[derive(Debug, Clone)]
227struct SupervisorRescanContext {
228 supervisor: Supervisor,
229 config_path: PathBuf,
230 configured_port: Option<u16>,
231 storage_config: Option<crate::daemon_config::StorageConfig>,
232 admission_facts_carrier_module_id: Option<String>,
233 admission_facts_targets: Option<Vec<String>>,
234 scope_authority_owners: Vec<String>,
235}
236
237const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
258 "reloading",
259 "supervisor_not_live",
260 "registration_not_active",
261 "no_forwarding_connection",
262 "relay_send_failed",
263];
264
265#[derive(Clone)]
267pub struct ControlHandler {
268 registry: Arc<Registry>,
269 forwarding: Arc<ForwardingTable>,
270 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
271 supervisor: SupervisorHandle,
272 subc_capabilities: Arc<[String]>,
273 route_bind_relay_timeout: Duration,
277 route_bind_relay_timeouts: BTreeMap<String, Duration>,
281 route_bind_breakers: RouteBindBreakers,
284 route_bind_concurrency: RouteBindConcurrency,
287 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
292 route_bind_breaker_threshold: u32,
294 route_bind_breaker_cooldown: Duration,
296 health_probe_timeout: Duration,
297 storage_config: Option<crate::daemon_config::StorageConfig>,
300 machine_id: Option<crate::machine_id::MachineId>,
304 admission_facts_carrier_module_id: Option<String>,
305 admission_facts_targets: Option<Vec<String>>,
306 scopes: Arc<RwLock<ScopeTable>>,
310 scope_authority_owners: Vec<String>,
313 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
316 rescan: Option<SupervisorRescanContext>,
317 connected_clients: ConnectedClients,
318 counters: DaemonCounters,
319 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
320 daemon_provenance: DaemonProvenanceFacts,
321 #[cfg(test)]
322 control_dispatch_delay: Option<Duration>,
323 #[cfg(test)]
324 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
325}
326
327impl fmt::Debug for ControlHandler {
328 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
329 f.debug_struct("ControlHandler")
330 .field("registry", &self.registry)
331 .field("forwarding", &self.forwarding)
332 .field("process_liveness", &self.process_liveness.is_some())
333 .field("supervisor", &self.supervisor)
334 .field("subc_capabilities", &self.subc_capabilities)
335 .finish()
336 }
337}
338
339struct RouteOpenRequest {
340 target: RouteTarget,
341 identity: BindIdentity,
342 consumer_identity: Option<ConsumerIdentity>,
343 consumer_capabilities: Option<Vec<String>>,
344 role_versions: Option<BTreeMap<String, String>>,
345 admission_facts: Option<serde_json::Value>,
346 scope: Option<ScopeSelector>,
347}
348
349struct RouteBindReservationGuard {
350 forwarding: Arc<ForwardingTable>,
351 endpoint: ModuleEndpointId,
352 relay_corr: u64,
353 armed: bool,
354}
355
356struct ModuleControlRpcGuard {
357 forwarding: Arc<ForwardingTable>,
358 endpoint: ModuleEndpointId,
359 corr: u64,
360 armed: bool,
361}
362
363impl ModuleControlRpcGuard {
364 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
365 Self {
366 forwarding,
367 endpoint,
368 corr,
369 armed: true,
370 }
371 }
372
373 fn disarm(&mut self) {
374 self.armed = false;
375 }
376}
377
378impl Drop for ModuleControlRpcGuard {
379 fn drop(&mut self) {
380 if self.armed {
381 let _ = self
382 .forwarding
383 .cancel_module_control_rpc(self.endpoint, self.corr);
384 }
385 }
386}
387
388impl RouteBindReservationGuard {
389 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
390 Self {
391 forwarding,
392 endpoint,
393 relay_corr,
394 armed: true,
395 }
396 }
397
398 fn release_and_disarm(&mut self) {
399 if !self.armed {
400 return;
401 }
402 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
403 self.endpoint,
404 self.relay_corr,
405 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
406 ) {
407 send_goodbye_target_best_effort(
408 &self.forwarding.counters(),
409 &target,
410 "canceled route.bind",
411 );
412 }
413 self.armed = false;
414 }
415
416 fn disarm(&mut self) {
417 self.armed = false;
418 }
419}
420
421impl Drop for RouteBindReservationGuard {
422 fn drop(&mut self) {
423 self.release_and_disarm();
424 }
425}
426
427#[derive(Debug, Clone, Default)]
457pub(crate) struct RouteBindBreakers {
458 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
459}
460
461#[derive(Debug, Clone, Default)]
462pub(crate) struct RouteBindConcurrency {
463 modules: Arc<Mutex<HashMap<String, usize>>>,
464}
465
466struct RouteBindConcurrencyGuard {
467 concurrency: RouteBindConcurrency,
468 module_id: String,
469}
470
471impl RouteBindConcurrency {
472 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
475 let mut modules = self
476 .modules
477 .lock()
478 .expect("route.bind concurrency mutex poisoned");
479 let in_flight = modules.entry(module_id.to_string()).or_default();
480 if *in_flight >= limit {
481 return Err(*in_flight);
482 }
483 *in_flight += 1;
484 Ok(RouteBindConcurrencyGuard {
485 concurrency: self.clone(),
486 module_id: module_id.to_string(),
487 })
488 }
489}
490
491impl Drop for RouteBindConcurrencyGuard {
492 fn drop(&mut self) {
493 let mut modules = self
494 .concurrency
495 .modules
496 .lock()
497 .expect("route.bind concurrency mutex poisoned");
498 let remove = {
499 let in_flight = modules
500 .get_mut(&self.module_id)
501 .expect("admitted route.bind has a concurrency entry");
502 *in_flight -= 1;
503 *in_flight == 0
504 };
505 if remove {
506 modules.remove(&self.module_id);
507 }
508 }
509}
510
511#[derive(Debug, Default)]
512struct ModuleBreakerState {
513 consecutive_timeouts: u32,
515 cooldown_until: Option<Instant>,
518 probe_in_flight: Option<Arc<()>>,
523}
524
525enum RouteBindAdmission<'a> {
527 Admitted {
528 guard: RouteBindBreakerGuard<'a>,
529 probe: bool,
532 },
533 Refused {
534 consecutive_timeouts: u32,
535 retry_in: Duration,
539 probe_in_flight: bool,
540 },
541}
542
543struct RouteBindBreakerGuard<'a> {
550 breakers: RouteBindBreakers,
551 module_id: &'a str,
552 probe_token: Option<Arc<()>>,
553 settled: bool,
554}
555
556impl RouteBindBreakerGuard<'_> {
557 fn record_accepted(&mut self) -> bool {
561 self.settled = true;
562 self.breakers.record_accepted(self.module_id)
563 }
564
565 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
568 self.settled = true;
569 self.breakers.record_timeout(
570 self.module_id,
571 self.probe_token.as_ref(),
572 threshold,
573 cooldown,
574 )
575 }
576
577 fn record_inconclusive(&mut self) {
586 self.settled = true;
587 self.breakers
588 .record_inconclusive(self.module_id, self.probe_token.as_ref());
589 }
590}
591
592impl Drop for RouteBindBreakerGuard<'_> {
593 fn drop(&mut self) {
594 if !self.settled {
595 self.breakers
596 .record_inconclusive(self.module_id, self.probe_token.as_ref());
597 }
598 }
599}
600
601struct BreakerOpened {
605 consecutive_timeouts: u32,
606 reopened_after_probe: bool,
609}
610
611impl RouteBindBreakers {
612 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
613 self.modules
614 .lock()
615 .expect("route.bind breaker mutex poisoned")
616 }
617
618 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
621 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
622 probe: probe_token.is_some(),
623 guard: RouteBindBreakerGuard {
624 breakers: self.clone(),
625 module_id,
626 probe_token,
627 settled: false,
628 },
629 };
630
631 let mut modules = self.lock();
632 let Some(state) = modules.get_mut(module_id) else {
633 return admitted(None);
634 };
635 let Some(cooldown_until) = state.cooldown_until else {
636 return admitted(None);
637 };
638 if state.probe_in_flight.is_some() {
639 return RouteBindAdmission::Refused {
640 consecutive_timeouts: state.consecutive_timeouts,
641 retry_in: Duration::ZERO,
642 probe_in_flight: true,
643 };
644 }
645 let now = Instant::now();
646 if now < cooldown_until {
647 return RouteBindAdmission::Refused {
648 consecutive_timeouts: state.consecutive_timeouts,
649 retry_in: cooldown_until - now,
650 probe_in_flight: false,
651 };
652 }
653 let token = Arc::new(());
654 state.probe_in_flight = Some(Arc::clone(&token));
655 admitted(Some(token))
656 }
657
658 fn record_accepted(&self, module_id: &str) -> bool {
659 self.lock()
660 .remove(module_id)
661 .is_some_and(|state| state.cooldown_until.is_some())
662 }
663
664 fn record_timeout(
665 &self,
666 module_id: &str,
667 probe_token: Option<&Arc<()>>,
668 threshold: u32,
669 cooldown: Duration,
670 ) -> Option<BreakerOpened> {
671 let mut modules = self.lock();
672 let state = modules.entry(module_id.to_string()).or_default();
673 let was_open = state.cooldown_until.is_some();
674 let was_probe = Self::owns_probe(state, probe_token);
675 if was_probe {
676 state.probe_in_flight = None;
677 }
678 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
679 if state.consecutive_timeouts < threshold {
680 return None;
681 }
682 state.cooldown_until = Some(Instant::now() + cooldown);
683 Some(BreakerOpened {
684 consecutive_timeouts: state.consecutive_timeouts,
685 reopened_after_probe: was_open && was_probe,
686 })
687 }
688
689 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
690 state
694 .probe_in_flight
695 .as_ref()
696 .zip(token)
697 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
698 }
699
700 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
701 if let Some(state) = self.lock().get_mut(module_id) {
702 if Self::owns_probe(state, probe_token) {
703 state.probe_in_flight = None;
704 }
705 }
706 }
707
708 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
723 self.lock()
724 .remove(module_id)
725 .map(|state| state.consecutive_timeouts)
726 .filter(|discarded| *discarded > 0)
727 }
728
729 fn open_snapshot(&self) -> Option<serde_json::Value> {
736 let now = Instant::now();
737 let modules = self.lock();
738 let open = modules
739 .iter()
740 .filter_map(|(module_id, state)| {
741 let cooldown_until = state.cooldown_until?;
742 Some((
743 module_id.clone(),
744 serde_json::json!({
745 "consecutive_timeouts": state.consecutive_timeouts,
746 "cooldown_remaining_ms":
747 cooldown_until.saturating_duration_since(now).as_millis() as u64,
748 "probe_in_flight": state.probe_in_flight.is_some(),
749 }),
750 ))
751 })
752 .collect::<serde_json::Map<String, serde_json::Value>>();
753 (!open.is_empty()).then_some(serde_json::Value::Object(open))
754 }
755}
756
757impl ControlHandler {
758 pub fn new(registry: Arc<Registry>) -> Self {
759 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
760 }
761
762 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
763 let counters = forwarding.counters();
764 let route_bind_breakers = forwarding.route_bind_breakers();
768 let route_bind_concurrency = forwarding.route_bind_concurrency();
769 let route_outages = forwarding.route_outages();
770 Self {
771 registry,
772 forwarding,
773 process_liveness: None,
774 supervisor: SupervisorHandle::new(),
775 subc_capabilities: Arc::from([
776 CAP_MANIFEST_REGISTRATION.to_string(),
777 CAP_CHANNEL_LIFECYCLE.to_string(),
778 CAP_PING_PONG.to_string(),
779 CAP_SESSION_ATTACH.to_string(),
780 CAP_ADMISSION_FACTS_RELAY.to_string(),
781 CAP_SCOPES_V1.to_string(),
782 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
783 ]),
784 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
785 route_bind_relay_timeouts: BTreeMap::new(),
786 route_bind_breakers,
787 route_bind_concurrency,
788 route_outages,
789 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
790 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
791 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
792 storage_config: None,
793 machine_id: None,
794 admission_facts_carrier_module_id: None,
795 admission_facts_targets: None,
796 scopes: Arc::new(RwLock::new(ScopeTable::new(
797 crate::daemon_config::default_scope_authority_owners(),
798 ))),
799 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
800 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
801 rescan: None,
802 connected_clients: ConnectedClients::new(),
803 counters,
804 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
805 daemon_provenance: DaemonProvenanceFacts::default(),
806 #[cfg(test)]
807 control_dispatch_delay: None,
808 #[cfg(test)]
809 provenance_probe_override: None,
810 }
811 }
812
813 pub fn with_storage_config(
816 mut self,
817 storage_config: Option<crate::daemon_config::StorageConfig>,
818 ) -> Self {
819 self.storage_config = storage_config;
820 self
821 }
822
823 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
826 self.machine_id = machine_id;
827 self
828 }
829
830 pub fn with_admission_facts_config(
834 mut self,
835 carrier_module_id: Option<String>,
836 targets: Option<Vec<String>>,
837 ) -> Self {
838 self.admission_facts_carrier_module_id = carrier_module_id;
839 self.admission_facts_targets = targets;
840 self
841 }
842
843 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
847 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
848 self.scope_authority_owners = owners;
849 self
850 }
851
852 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
855 self.route_bind_relay_timeout = timeout;
856 self
857 }
858
859 pub fn with_route_bind_relay_timeouts(
865 mut self,
866 timeouts: impl IntoIterator<Item = (String, Duration)>,
867 ) -> Self {
868 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
869 self
870 }
871
872 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
878 self.route_bind_relay_timeouts
879 .get(module_id)
880 .copied()
881 .unwrap_or(self.route_bind_relay_timeout)
882 }
883
884 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
892 self.route_bind_breaker_threshold = threshold.max(1);
893 self.route_bind_breaker_cooldown = cooldown;
894 self
895 }
896
897 #[cfg(test)]
898 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
899 self.health_probe_timeout = timeout;
900 self
901 }
902
903 #[cfg(test)]
904 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
905 self.control_dispatch_delay = Some(delay);
906 self
907 }
908
909 pub fn with_process_liveness(
910 mut self,
911 process_liveness: Arc<dyn ModuleProcessLiveness>,
912 ) -> Self {
913 self.process_liveness = Some(process_liveness);
914 self
915 }
916
917 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
918 self.supervisor = supervisor;
919 self
920 }
921
922 pub fn with_daemon_provenance(
923 mut self,
924 pid: u32,
925 started_at_ms: u64,
926 executable_path: Option<PathBuf>,
927 build_git_sha: Option<String>,
928 build_lock_digest: Option<String>,
929 ) -> Self {
930 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
931 let process_start_time = process_start_time(pid);
932 self.daemon_provenance = DaemonProvenanceFacts {
933 build: DaemonBuildProvenance {
934 build_git_sha,
935 build_lock_digest,
936 },
937 pid: Some(pid),
938 started_at_ms: Some(started_at_ms),
939 start_clock: None,
940 executable_path,
941 executable_identity,
942 process_start_time,
943 probe: ExecutableIdentityProbe::default(),
944 };
945 self
946 }
947
948 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
949 self.daemon_provenance.start_clock = Some(clock);
950 self
951 }
952
953 #[cfg(test)]
954 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
955 self.provenance_probe_override = Some(result);
956 self
957 }
958
959 pub fn with_capability_config(
963 self,
964 modules: impl IntoIterator<Item = (String, bool)>,
965 reserved_capabilities: BTreeMap<String, String>,
966 ) -> Self {
967 self.capability_evaluator
968 .configure(modules, reserved_capabilities);
969 self
970 }
971
972 pub fn with_supervisor_rescan(
973 mut self,
974 supervisor: Supervisor,
975 config_path: impl Into<PathBuf>,
976 configured_port: Option<u16>,
977 ) -> Self {
978 self.rescan = Some(SupervisorRescanContext {
979 supervisor,
980 config_path: config_path.into(),
981 configured_port,
982 storage_config: self.storage_config.clone(),
983 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
984 admission_facts_targets: self.admission_facts_targets.clone(),
985 scope_authority_owners: self.scope_authority_owners.clone(),
986 });
987 self
988 }
989
990 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
991 self.connected_clients = connected_clients;
992 self
993 }
994
995 pub fn forwarding(&self) -> Arc<ForwardingTable> {
996 Arc::clone(&self.forwarding)
997 }
998
999 pub(crate) fn counters(&self) -> DaemonCounters {
1000 self.counters.clone()
1001 }
1002
1003 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1006 tokio::spawn(async move {
1007 loop {
1008 self.capability_evaluator
1009 .wait_for_change_or_deadline()
1010 .await;
1011 self.refresh_capability_requirements();
1012 }
1013 });
1014 }
1015
1016 fn runtime_capability_snapshot(
1017 &self,
1018 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1019 let runtime = self
1020 .supervisor
1021 .list()
1022 .into_iter()
1023 .map(|module| {
1024 let status = module.status().map_err(|err| {
1025 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1026 })?;
1027 Ok(RuntimeModule {
1028 module_id: status.module_id,
1029 state: status.state,
1030 enabled: status.enabled,
1031 })
1032 })
1033 .collect::<Result<Vec<_>, RouterError>>()?;
1034 let (_, registrations) = self.registry.list_modules().map_err(|err| {
1035 RouterError::backend(
1036 0,
1037 0,
1038 format!("failed to list capability registrations: {err}"),
1039 )
1040 })?;
1041 let registrations = registrations
1042 .into_iter()
1043 .map(|registration| RegisteredModule {
1044 module_id: registration.manifest.module_id,
1045 module_version: registration.manifest.module_version,
1046 capabilities: registration.manifest.capabilities,
1047 })
1048 .collect();
1049 Ok((runtime, registrations))
1050 }
1051
1052 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1059 let cached_registration = RegisteredModule {
1060 module_id: registration.manifest.module_id.clone(),
1061 module_version: registration.manifest.module_version.clone(),
1062 capabilities: registration.manifest.capabilities.clone(),
1063 };
1064 if self.capability_evaluator.record_hello(&cached_registration) {
1065 warn!(
1066 module_id = %cached_registration.module_id,
1067 "capability claims drifted from the cached manifest"
1068 );
1069 }
1070 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1071 self.enforce_capability_denies();
1072 }
1073 self.refresh_capability_requirements();
1074 }
1075
1076 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1080 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1081 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1082 self.supervisor.set_swap_promotion_observer(observer);
1083 }
1084
1085 pub fn refresh_capability_requirements(&self) {
1086 match self.runtime_capability_snapshot() {
1087 Ok((runtime, registrations)) => {
1088 log_requirement_events(
1089 self.capability_evaluator
1090 .evaluate_now(&runtime, ®istrations),
1091 );
1092 }
1093 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1094 }
1095 }
1096
1097 fn enforce_capability_denies(&self) {
1101 let (_, registrations) = match self.registry.list_modules() {
1102 Ok(snapshot) => snapshot,
1103 Err(err) => {
1104 warn!(error = %err, "failed to read registrations for capability deny census");
1105 return;
1106 }
1107 };
1108 let manifests = registrations
1109 .into_iter()
1110 .map(|registration| {
1111 (
1112 registration.manifest.module_id.clone(),
1113 registration.manifest,
1114 )
1115 })
1116 .collect::<BTreeMap<_, _>>();
1117 let census = match self.forwarding.route_census(None) {
1118 Ok(census) => census,
1119 Err(err) => {
1120 warn!(error = %err, "failed to read route census for capability deny enforcement");
1121 return;
1122 }
1123 };
1124
1125 for (target_module_id, routes) in census {
1126 let Some(target_manifest) = manifests.get(&target_module_id) else {
1127 continue;
1128 };
1129 let mut closed_routes = Vec::new();
1130 let mut module_goodbyes = Vec::new();
1131 for route in routes {
1132 let Principal::Reserved {
1133 module_id: opening_module_id,
1134 } = &route.principal
1135 else {
1136 continue;
1137 };
1138 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1139 continue;
1140 };
1141 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1142 continue;
1143 };
1144
1145 match self.forwarding.release_client_route(
1146 route.goodbye_target.connection_id,
1147 route.goodbye_target.channel,
1148 route.goodbye_target.epoch,
1149 ) {
1150 Ok(RouteRelease::Removed(module_goodbye)) => {
1151 warn!(
1152 opening_module_id,
1153 target_module_id,
1154 capability,
1155 "force-closing route because an attested capability deny edge now matches"
1156 );
1157 closed_routes.push(route);
1158 module_goodbyes.push(module_goodbye);
1159 }
1160 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1161 Err(err) => warn!(
1162 opening_module_id,
1163 target_module_id,
1164 capability,
1165 error = %err,
1166 "failed to force-close capability-denied route"
1167 ),
1168 }
1169 }
1170
1171 if closed_routes.is_empty() {
1172 continue;
1173 }
1174 send_route_control_pushes(
1175 &self.forwarding,
1176 closed_routes,
1177 ClientControlPush::RouteClosed {
1178 module_id: target_module_id,
1179 channels: Vec::new(),
1180 reason: RouteCloseReason::CapabilityDenied,
1181 drained: false,
1182 abandoned: 0,
1183 excluded_subscriptions: 0,
1184 terminal: Some(false),
1185 },
1186 );
1187 self.emit_route_goodbyes(module_goodbyes);
1188 }
1189 }
1190
1191 fn not_ready_reason(
1197 &self,
1198 registration: &crate::registry::ModuleRegistration,
1199 ) -> Option<NotReadyReason> {
1200 if !registration.ready {
1201 return Some(NotReadyReason {
1202 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1203 capability: None,
1204 });
1205 }
1206 self.first_unprovided_required_capability(registration)
1207 .map(|capability| NotReadyReason {
1208 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1209 capability: Some(capability),
1210 })
1211 }
1212
1213 fn first_unprovided_required_capability(
1231 &self,
1232 registration: &crate::registry::ModuleRegistration,
1233 ) -> Option<String> {
1234 let required = registration
1235 .manifest
1236 .capabilities
1237 .iter()
1238 .flat_map(|declarations| declarations.requires.iter())
1239 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1240 .map(|requirement| requirement.capability.as_str())
1241 .collect::<BTreeSet<_>>();
1242 if required.is_empty() {
1243 return None;
1244 }
1245 let module_id = registration.manifest.module_id.as_str();
1246 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1247 if required
1248 .iter()
1249 .any(|capability| verdict(capability).is_none())
1250 {
1251 self.refresh_capability_requirements();
1252 }
1253 required
1254 .into_iter()
1255 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1256 .map(str::to_string)
1257 }
1258
1259 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1260 self.capability_evaluator
1261 .statuses()
1262 .into_iter()
1263 .map(capability_requirement_status)
1264 .collect()
1265 }
1266
1267 fn deregister_connection(
1274 &self,
1275 connection_id: ConnectionId,
1276 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1277 self.registry.deregister_connection(connection_id)
1278 }
1279
1280 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1281 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1282 return None;
1283 }
1284 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1285 parse_client_control_request(&frame.body)
1286 else {
1287 return None;
1288 };
1289 Some(target_module_id(&target).to_string())
1290 }
1291
1292 pub(crate) fn route_open_capacity_refusal(
1293 &self,
1294 ctx: &RouteCtx,
1295 frame: &Frame,
1296 target_module_id: &str,
1297 in_flight: usize,
1298 limit: usize,
1299 ) -> Result<Frame, RouterError> {
1300 self.route_open_admission_refusal_frame(
1301 ctx,
1302 frame,
1303 target_module_id,
1304 "open_admission_full",
1305 (in_flight, limit),
1306 format!(
1307 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1308 ),
1309 )
1310 }
1311
1312 fn route_open_target_capacity_refusal(
1313 &self,
1314 ctx: &RouteCtx,
1315 frame: &Frame,
1316 target_module_id: &str,
1317 in_flight: usize,
1318 ) -> Result<Frame, RouterError> {
1319 self.route_open_admission_refusal_frame(
1320 ctx,
1321 frame,
1322 target_module_id,
1323 "target_binds_full",
1324 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1325 format!(
1326 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1327 ),
1328 )
1329 }
1330
1331 fn route_open_admission_refusal_frame(
1339 &self,
1340 ctx: &RouteCtx,
1341 frame: &Frame,
1342 target_module_id: &str,
1343 reason: &'static str,
1344 (in_flight, limit): (usize, usize),
1345 message: impl Into<String>,
1346 ) -> Result<Frame, RouterError> {
1347 let code = error_codes::TARGET_UNAVAILABLE;
1348 self.counters.increment_route_open_refused(code);
1349 info!(
1350 target: "control",
1351 code,
1352 reason,
1353 module_id = ?target_module_id,
1354 connection_id = ctx.connection_id.get(),
1355 in_flight,
1356 limit,
1357 "route.open refused"
1358 );
1359 control_error_frame(frame, code, message.into())
1360 }
1361
1362 #[cfg(test)]
1368 pub fn handle_control(
1369 &self,
1370 connection_id: ConnectionId,
1371 frame: Frame,
1372 ) -> Result<Vec<Frame>, RouterError> {
1373 match frame.header.ty {
1374 FrameType::Ping => Ok(vec![pong(&frame)?]),
1375 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1376 FrameType::Goodbye => self.handle_goodbye(connection_id),
1377 ty => Ok(vec![control_error_frame(
1378 &frame,
1379 "unsupported_control_frame",
1380 format!("unsupported channel-0 frame {ty:?}"),
1381 )?]),
1382 }
1383 }
1384
1385 pub async fn handle_control_frame(
1386 &self,
1387 ctx: &RouteCtx,
1388 frame: Frame,
1389 ) -> Result<Vec<Frame>, RouterError> {
1390 self.handle_control_frame_timed(ctx, frame, None).await
1391 }
1392
1393 pub(crate) async fn handle_control_frame_timed(
1394 &self,
1395 ctx: &RouteCtx,
1396 frame: Frame,
1397 dispatch_started_at: Option<StdInstant>,
1398 ) -> Result<Vec<Frame>, RouterError> {
1399 match frame.header.ty {
1400 FrameType::Ping => Ok(vec![pong(&frame)?]),
1401 FrameType::Hello => {
1402 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1403 }
1404 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1405 FrameType::Cancel => {
1406 if self
1407 .supervisor
1408 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1409 {
1410 Ok(Vec::new())
1411 } else {
1412 Ok(vec![control_error_frame(
1413 &frame,
1414 "unknown_subscription",
1415 "no supervisor spawn subscription has this correlation id",
1416 )?])
1417 }
1418 }
1419 FrameType::Request => {
1420 if self
1421 .forwarding
1422 .module_endpoint_for_connection(ctx.connection_id)
1423 .map_err(RouterError::Forwarding)?
1424 .is_some()
1425 {
1426 if !is_known_module_request_op(&frame.body) {
1427 return Ok(vec![control_error_frame(
1428 &frame,
1429 "unsupported_control_frame",
1430 "module-originated channel-0 REQUEST is not supported",
1431 )?]);
1432 }
1433 let request = match parse_module_control_request_from_module(&frame.body) {
1434 Ok(request) => request,
1435 Err((err, ControlRequestBodyError::UnknownOp)) => {
1436 return Ok(vec![control_error_frame(
1437 &frame,
1438 "unsupported_control_frame",
1439 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1440 )?])
1441 }
1442 Err((err, ControlRequestBodyError::InvalidBody)) => {
1443 return Ok(vec![control_error_frame(
1444 &frame,
1445 "invalid_control_body",
1446 format!("malformed module control body: {err}"),
1447 )?])
1448 }
1449 };
1450 let op = module_control_request_op(&request);
1451 let corr = frame.header.corr;
1452 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1453 let result =
1454 self.handle_module_control_request(ctx.connection_id, frame, request);
1455 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1456 return result;
1457 }
1458
1459 if is_known_module_request_op(&frame.body) {
1460 return Ok(vec![control_error_frame(
1461 &frame,
1462 "not_registered",
1463 "catalog.update requires an active module registration owned by this connection",
1464 )?]);
1465 }
1466
1467 let request = match parse_client_control_request(&frame.body) {
1468 Ok(request) => request,
1469 Err((err, ControlRequestBodyError::UnknownOp)) => {
1470 return Ok(vec![control_error_frame(
1471 &frame,
1472 "unknown_control_op",
1473 format!("unknown client control op: {err}"),
1474 )?])
1475 }
1476 Err((err, ControlRequestBodyError::InvalidBody)) => {
1477 return Ok(vec![control_error_frame(
1478 &frame,
1479 "invalid_control_body",
1480 format!("malformed client control body: {err}"),
1481 )?])
1482 }
1483 };
1484 let op = client_control_request_op(&request);
1485 let corr = frame.header.corr;
1486 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1487 #[cfg(test)]
1488 if let Some(delay) = self.control_dispatch_delay {
1489 tokio::time::sleep(delay).await;
1490 }
1491 let result = self
1492 .handle_client_control_request(ctx, frame, request)
1493 .await;
1494 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1495 result
1496 }
1497 FrameType::Push => {
1498 let Some(endpoint) = self
1499 .forwarding
1500 .module_endpoint_for_connection(ctx.connection_id)
1501 .map_err(RouterError::Forwarding)?
1502 else {
1503 return Ok(vec![control_error_frame(
1504 &frame,
1505 "unsupported_control_frame",
1506 "client-originated channel-0 PUSH is not supported",
1507 )?]);
1508 };
1509 self.handle_status_update(endpoint, frame)
1510 }
1511 FrameType::Response | FrameType::Error
1512 if self
1513 .forwarding
1514 .module_endpoint_for_connection(ctx.connection_id)
1515 .map_err(RouterError::Forwarding)?
1516 .is_some() =>
1517 {
1518 self.handle_module_relay_response(ctx.connection_id, frame)
1519 }
1520 ty => Ok(vec![control_error_frame(
1521 &frame,
1522 "unsupported_control_frame",
1523 format!("unsupported channel-0 frame {ty:?}"),
1524 )?]),
1525 }
1526 }
1527
1528 pub fn cleanup_connection(
1529 &self,
1530 connection_id: ConnectionId,
1531 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1532 let crash_closed = self
1533 .registry
1534 .get_module_by_connection(connection_id)?
1535 .and_then(|registration| {
1536 self.forwarding
1537 .module_endpoint_for_connection(connection_id)
1538 .ok()
1539 .flatten()
1540 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1541 .map(|routes| (registration.manifest.module_id, routes))
1542 });
1543 let crash_closed = crash_closed.map(|(module_id, routes)| {
1544 let terminal = match self.supervisor.get(&module_id) {
1545 None => false,
1546 Some(module) => match module.will_recover_after_connection_loss() {
1547 Ok(will_recover) => !will_recover,
1548 Err(err) => {
1549 warn!(
1550 %module_id,
1551 error = %err,
1552 "failed to read crash recovery verdict; reporting non-terminal conservatively"
1553 );
1554 false
1555 }
1556 },
1557 };
1558 let reason = match self.forwarding.is_daemon_draining() {
1562 Ok(true) => RouteCloseReason::Restart,
1563 Ok(false) => RouteCloseReason::Crash,
1564 Err(err) => {
1565 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
1566 RouteCloseReason::Crash
1567 }
1568 };
1569 (module_id, routes, reason, terminal)
1570 });
1571 let registrations = self.deregister_connection(connection_id);
1572 let cleanup = if crash_closed.is_some() {
1573 self.forwarding.cleanup_connection_counted(connection_id)
1574 } else {
1575 self.forwarding
1581 .cleanup_connection(connection_id)
1582 .map(|released| crate::forwarding::ConnectionCleanup {
1583 released,
1584 abandoned_relays: 0,
1585 })
1586 };
1587 if let Some((module_id, routes, reason, terminal)) = crash_closed {
1592 let abandoned = cleanup
1593 .as_ref()
1594 .map_or(0, |cleanup| cleanup.abandoned_relays);
1595 send_route_control_pushes(
1596 &self.forwarding,
1597 routes,
1598 ClientControlPush::RouteClosed {
1599 module_id,
1600 channels: Vec::new(),
1601 reason,
1602 drained: false,
1603 abandoned,
1604 excluded_subscriptions: 0,
1605 terminal: Some(terminal),
1606 },
1607 );
1608 }
1609 if let Ok(cleanup) = cleanup {
1610 self.emit_route_goodbyes(cleanup.released);
1611 }
1612 if matches!(®istrations, Ok(r) if !r.is_empty()) {
1616 crate::supervise::notify_registration_release();
1617 self.capability_evaluator.wake_deadline_loop();
1618 self.refresh_capability_requirements();
1619 }
1620 self.supervisor.remove_spawn_subscribers(connection_id);
1621 self.hello_launch_nonces
1624 .lock()
1625 .unwrap_or_else(|poisoned| poisoned.into_inner())
1626 .forget(connection_id);
1627 self.scopes
1628 .write()
1629 .unwrap_or_else(|poisoned| poisoned.into_inner())
1630 .release_connection(connection_id);
1631 registrations
1632 }
1633
1634 pub(crate) fn handle_route_goodbye(
1635 &self,
1636 connection_id: ConnectionId,
1637 route_channel: u16,
1638 route_epoch: u32,
1639 ) -> Result<bool, RouterError> {
1640 debug!(
1641 connection_id = connection_id.get(),
1642 route_channel, route_epoch, "handling route GOODBYE"
1643 );
1644 let RouteRelease::Removed(released_route) = self
1645 .forwarding
1646 .release_client_route(connection_id, route_channel, route_epoch)
1647 .map_err(RouterError::Forwarding)?
1648 else {
1649 return Ok(false);
1650 };
1651 self.emit_route_goodbyes(vec![released_route]);
1652 Ok(true)
1653 }
1654
1655 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
1656 for released in released_routes {
1657 let frame = match Frame::build_with_version(
1658 released.negotiated_ver,
1659 FrameType::Goodbye,
1660 control_flags(),
1661 released.channel,
1662 released.epoch,
1663 0,
1664 Vec::new(),
1665 ) {
1666 Ok(frame) => frame,
1667 Err(err) => {
1668 warn!(
1669 route_channel = released.channel,
1670 error = %err,
1671 "failed to build route GOODBYE frame"
1672 );
1673 continue;
1674 }
1675 };
1676 if !released.close_on_delivery_failure() {
1677 crate::forwarding::send_module_route_goodbye(
1678 &self.counters,
1679 &released.sink,
1680 frame,
1681 released.module_id.as_deref(),
1682 "client route released",
1683 );
1684 continue;
1685 }
1686 if let Err(err) = released.sink.try_send(frame) {
1687 warn!(
1688 target_connection_id = released.connection_id.get(),
1689 route_channel = released.channel,
1690 error = %err,
1691 "route GOODBYE was not delivered to client; closing target connection"
1692 );
1693 if self
1694 .forwarding
1695 .escalate_client_delivery_failure(
1696 released.connection_id,
1697 released.channel,
1698 released.epoch,
1699 CloseReason::new(
1700 "route_goodbye_delivery_failed",
1701 format!(
1702 "failed to enqueue route GOODBYE for channel {}: {err}",
1703 released.channel
1704 ),
1705 ),
1706 crate::forwarding::UndeliveredFrame {
1707 module_id: released.module_id.as_deref(),
1708 sink: &released.sink,
1709 },
1710 )
1711 .unwrap_or(false)
1712 {
1713 self.counters.increment_goodbye_relay_client_failed();
1714 }
1715 }
1716 }
1717 }
1718
1719 fn send_abandoned_route_bind_goodbye(
1731 &self,
1732 module_sink: &crate::FrameSink,
1733 negotiated_ver: u8,
1734 module_channel: u16,
1735 module_epoch: u32,
1736 ) {
1737 let frame = match Frame::build_with_version(
1738 negotiated_ver,
1739 FrameType::Goodbye,
1740 control_flags(),
1741 module_channel,
1742 module_epoch,
1743 0,
1744 Vec::new(),
1745 ) {
1746 Ok(frame) => frame,
1747 Err(err) => {
1748 warn!(
1749 route_channel = module_channel,
1750 error = %err,
1751 "failed to build GOODBYE for abandoned route.bind"
1752 );
1753 return;
1754 }
1755 };
1756 crate::forwarding::send_module_route_goodbye(
1757 &self.counters,
1758 module_sink,
1759 frame,
1760 None,
1761 "abandoned route.bind",
1762 );
1763 }
1764
1765 fn handle_hello(
1766 &self,
1767 connection_id: ConnectionId,
1768 sink: Option<crate::FrameSink>,
1769 frame: Frame,
1770 ) -> Result<Vec<Frame>, RouterError> {
1771 debug!(
1772 connection_id = connection_id.get(),
1773 corr = frame.header.corr,
1774 "handling HELLO"
1775 );
1776 if self
1780 .registry
1781 .get_module_by_connection(connection_id)
1782 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
1783 .is_some()
1784 {
1785 return Ok(vec![control_error_frame(
1786 &frame,
1787 "invalid_hello",
1788 "connection is already registered as a module",
1789 )?]);
1790 }
1791 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
1792 Ok(value) => value,
1793 Err(err) => {
1794 return Ok(vec![control_error_frame(
1795 &frame,
1796 "invalid_hello",
1797 format!("malformed HELLO body: {err}"),
1798 )?])
1799 }
1800 };
1801 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
1802 return Ok(vec![control_error_frame(
1803 &frame,
1804 "invalid_capability_grammar",
1805 err.to_string(),
1806 )?]);
1807 }
1808 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
1809 return Ok(vec![control_error_frame(
1810 &frame,
1811 "invalid_manifest",
1812 err.to_string(),
1813 )?]);
1814 }
1815 if let Some(provenance) = hello_value
1816 .get("manifest")
1817 .and_then(|manifest| manifest.get("provenance"))
1818 {
1819 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
1820 return Ok(vec![control_error_frame(
1821 &frame,
1822 "invalid_manifest",
1823 format!("malformed manifest provenance: {err}"),
1824 )?]);
1825 }
1826 }
1827 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
1828 Ok(hello) => hello,
1829 Err(err) => {
1830 return Ok(vec![control_error_frame(
1831 &frame,
1832 "invalid_hello",
1833 format!("malformed HELLO body: {err}"),
1834 )?])
1835 }
1836 };
1837
1838 if hello.protocol_ver != hello.manifest.protocol_ver {
1839 return Ok(vec![control_error_frame(
1840 &frame,
1841 "invalid_manifest",
1842 format!(
1843 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
1844 hello.protocol_ver, hello.manifest.protocol_ver
1845 ),
1846 )?]);
1847 }
1848
1849 if hello.manifest.module_id.trim().is_empty() {
1850 return Ok(vec![control_error_frame(
1851 &frame,
1852 "invalid_manifest",
1853 "manifest module_id must not be empty",
1854 )?]);
1855 }
1856
1857 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
1858 Ok(negotiated_ver) => negotiated_ver,
1859 Err(message) => {
1860 return Ok(vec![control_error_frame(
1861 &frame,
1862 "version_unsupported",
1863 message,
1864 )?])
1865 }
1866 };
1867
1868 let swap_admission = self
1876 .supervisor
1877 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
1878 if swap_admission == SwapHelloAdmission::Refused {
1879 warn!(
1880 module_id = %hello.manifest.module_id,
1881 connection_id = connection_id.get(),
1882 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
1883 );
1884 return Ok(vec![control_error_frame(
1885 &frame,
1886 "swap_token_invalid",
1887 format!(
1888 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
1889 hello.manifest.module_id
1890 ),
1891 )?]);
1892 }
1893 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
1894
1895 if let Some(rejection) = (!swap_candidate)
1903 .then(|| {
1904 self.supervisor.reserved_hello_rejection(
1905 &hello.manifest.module_id,
1906 hello.launch_nonce.as_deref(),
1907 )
1908 })
1909 .flatten()
1910 {
1911 let message = match rejection {
1912 ReservedHelloRejection::Exact { module_id } => format!(
1913 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
1914 ),
1915 ReservedHelloRejection::Prefix {
1916 prefix,
1917 owner_module_id,
1918 } => format!(
1919 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
1920 hello.manifest.module_id
1921 ),
1922 };
1923 return Ok(vec![control_error_frame(
1924 &frame,
1925 "reserved_module",
1926 message,
1927 )?]);
1928 }
1929
1930 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
1931 &hello.manifest.module_id,
1932 hello.manifest.capabilities.as_ref(),
1933 );
1934 if let Some(refusal) = reserved_capability_refusals.first() {
1935 let capability = refusal.capability.clone();
1936 let bound_module = refusal.claimants[0].clone();
1937 log_duplicate_claim_events(reserved_capability_refusals);
1938 return Ok(vec![control_error_frame(
1939 &frame,
1940 "reserved_capability",
1941 format!(
1942 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
1943 capability, bound_module, hello.manifest.module_id
1944 ),
1945 )?]);
1946 }
1947
1948 if self
1951 .forwarding
1952 .connection_has_client_routes(connection_id)
1953 .map_err(RouterError::Forwarding)?
1954 {
1955 return Ok(vec![control_error_frame(
1956 &frame,
1957 "invalid_hello",
1958 "connection has open client routes and cannot also register as a module",
1959 )?]);
1960 }
1961
1962 self.hello_launch_nonces
1967 .lock()
1968 .unwrap_or_else(|poisoned| poisoned.into_inner())
1969 .record(connection_id, hello.launch_nonce.as_deref());
1970 let control_ops = effective_module_control_ops(hello.control_ops);
1971 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
1974 if swap_candidate {
1975 return self.register_swap_candidate(
1976 connection_id,
1977 sink,
1978 &frame,
1979 hello.manifest,
1980 negotiated_ver,
1981 control_ops,
1982 hello_ack,
1983 );
1984 }
1985 let registration = match self.registry.register_with_control_ops(
1986 hello.manifest,
1987 negotiated_ver,
1988 connection_id,
1989 control_ops,
1990 ) {
1991 Ok(registration) => registration,
1992 Err(RegistryError::DuplicateModuleId { module_id }) => {
1993 return Ok(vec![control_error_frame(
1994 &frame,
1995 "duplicate_module_id",
1996 format!(
1997 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
1998 ),
1999 )?])
2000 }
2001 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2002 return Ok(vec![control_error_frame(
2003 &frame,
2004 "invalid_module_id",
2005 err.to_string(),
2006 )?])
2007 }
2008 Err(err) => {
2009 return Ok(vec![control_error_frame(
2010 &frame,
2011 "registry_error",
2012 err.to_string(),
2013 )?])
2014 }
2015 };
2016
2017 let reply = if let Some(sink) = sink {
2018 let concurrency = manifest_concurrency(®istration.manifest);
2034 if let Err(err) = self.forwarding.register_module_connection_acked(
2035 connection_id,
2036 registration.manifest.module_id.clone(),
2037 negotiated_ver,
2038 concurrency,
2039 sink,
2040 hello_ack,
2041 ) {
2042 if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
2046 crate::supervise::notify_registration_release();
2047 }
2048 return Ok(vec![control_error_frame(
2049 &frame,
2050 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2051 "invalid_hello"
2052 } else {
2053 forwarding_error_code(&err)
2054 },
2055 err.to_string(),
2056 )?]);
2057 }
2058 Vec::new()
2059 } else {
2060 vec![hello_ack]
2063 };
2064
2065 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2074 info!(
2075 module_id = %registration.manifest.module_id,
2076 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2077 );
2078 }
2079
2080 self.apply_registration_capabilities(®istration);
2081
2082 info!(
2083 module_id = %registration.manifest.module_id,
2084 module_version = %registration.manifest.module_version,
2085 negotiated_ver,
2086 routable_provider = manifest_provides_routable_role(®istration.manifest),
2087 connection_id = connection_id.get(),
2088 "module registered"
2089 );
2090
2091 Ok(reply)
2092 }
2093
2094 #[allow(clippy::too_many_arguments)]
2104 fn register_swap_candidate(
2105 &self,
2106 connection_id: ConnectionId,
2107 sink: Option<crate::FrameSink>,
2108 frame: &Frame,
2109 manifest: ModuleManifest,
2110 negotiated_ver: u8,
2111 control_ops: Vec<String>,
2112 hello_ack: Frame,
2113 ) -> Result<Vec<Frame>, RouterError> {
2114 let module_id = manifest.module_id.clone();
2115 let registration = match self.registry.register_candidate_with_control_ops(
2116 manifest,
2117 negotiated_ver,
2118 connection_id,
2119 control_ops,
2120 ) {
2121 Ok(registration) => registration,
2122 Err(RegistryError::DuplicateModuleId { module_id }) => {
2123 return Ok(vec![control_error_frame(
2124 frame,
2125 "duplicate_module_id",
2126 format!(
2127 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2128 ),
2129 )?])
2130 }
2131 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2132 return Ok(vec![control_error_frame(
2133 frame,
2134 "invalid_module_id",
2135 err.to_string(),
2136 )?])
2137 }
2138 Err(err) => {
2139 return Ok(vec![control_error_frame(
2140 frame,
2141 "registry_error",
2142 err.to_string(),
2143 )?])
2144 }
2145 };
2146 let reply = if let Some(sink) = sink {
2147 let concurrency = manifest_concurrency(®istration.manifest);
2151 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2152 connection_id,
2153 module_id.clone(),
2154 negotiated_ver,
2155 concurrency,
2156 sink,
2157 hello_ack,
2158 ) {
2159 if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
2160 crate::supervise::notify_registration_release();
2161 }
2162 return Ok(vec![control_error_frame(
2163 frame,
2164 forwarding_error_code(&err),
2165 err.to_string(),
2166 )?]);
2167 }
2168 Vec::new()
2169 } else {
2170 vec![hello_ack]
2171 };
2172 self.supervisor.mark_swap_candidate_admitted(&module_id);
2173 info!(
2174 module_id = %module_id,
2175 module_version = %registration.manifest.module_version,
2176 negotiated_ver,
2177 ready = registration.ready,
2178 connection_id = connection_id.get(),
2179 "swap candidate registered; not routable until cutover"
2180 );
2181 Ok(reply)
2182 }
2183
2184 fn build_hello_ack(
2185 &self,
2186 frame: &Frame,
2187 negotiated_ver: u8,
2188 module_id: &str,
2189 ) -> Result<Frame, RouterError> {
2190 let ack = ModuleHelloAckBody {
2191 negotiated_ver,
2192 subc_ops: module_subc_ops(),
2193 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2194 storage: self
2195 .storage_config
2196 .as_ref()
2197 .map(|cfg| cfg.descriptor_for(module_id)),
2198 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2199 };
2200 let body = serde_json::to_vec(&ack).map_err(|err| {
2201 RouterError::backend(
2202 0,
2203 frame.header.corr,
2204 format!("failed to encode HELLO_ACK: {err}"),
2205 )
2206 })?;
2207
2208 Frame::build_with_version(
2209 negotiated_ver,
2210 FrameType::HelloAck,
2211 control_flags(),
2212 0,
2213 0,
2214 frame.header.corr,
2215 body,
2216 )
2217 .map_err(RouterError::FrameBuild)
2218 }
2219
2220 async fn handle_client_control_request(
2221 &self,
2222 ctx: &RouteCtx,
2223 frame: Frame,
2224 request: ClientControlRequest,
2225 ) -> Result<Vec<Frame>, RouterError> {
2226 match request {
2227 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2228 ClientControlRequest::CatalogList { module_id } => {
2229 self.handle_catalog_list(frame, module_id)
2230 }
2231 ClientControlRequest::RouteOpen {
2232 target,
2233 identity,
2234 consumer_identity,
2235 consumer_capabilities,
2236 role_versions,
2237 admission_facts,
2238 scope,
2239 } => {
2240 self.handle_route_open(
2241 ctx,
2242 frame,
2243 RouteOpenRequest {
2244 target,
2245 identity,
2246 consumer_identity,
2247 consumer_capabilities,
2248 role_versions,
2249 admission_facts,
2250 scope,
2251 },
2252 )
2253 .await
2254 }
2255 ClientControlRequest::RoutePoll {
2256 route_channel,
2257 route_epoch,
2258 kind,
2259 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2260 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2261 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2262 self.handle_supervisor_spawn_snapshot(frame)
2263 }
2264 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2265 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2266 }
2267 ClientControlRequest::SupervisorRestart {
2268 module_id,
2269 drain_timeout_ms,
2270 } => {
2271 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms)
2272 .await
2273 }
2274 ClientControlRequest::SupervisorSwap {
2275 module_id,
2276 ready_timeout_ms,
2277 } => {
2278 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms)
2279 .await
2280 }
2281 ClientControlRequest::SupervisorReload { module_id } => {
2282 self.handle_supervisor_reload(frame, module_id).await
2283 }
2284 ClientControlRequest::SupervisorRescan { preview } => {
2285 self.handle_supervisor_rescan(frame, preview).await
2286 }
2287 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2288 self.handle_supervisor_release_reserved(frame, module_id)
2289 .await
2290 }
2291 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2292 self.handle_supervisor_set_enabled(frame, module_id, enabled)
2293 .await
2294 }
2295 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2296 self.handle_supervisor_health_probe(frame, module_id).await
2297 }
2298 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2299 ClientControlRequest::SupervisorRoutes { module_id } => {
2300 self.handle_supervisor_routes(frame, module_id)
2301 }
2302 ClientControlRequest::SupervisorProvenance { module_id } => {
2303 self.handle_supervisor_provenance(frame, module_id).await
2304 }
2305 ClientControlRequest::SupervisorStderrTail {
2306 module_id,
2307 max_lines,
2308 max_bytes,
2309 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2310 ClientControlRequest::SupervisorTerminals { module_id } => {
2311 self.handle_supervisor_terminals(frame, module_id).await
2312 }
2313 }
2314 }
2315
2316 fn handle_module_control_request(
2317 &self,
2318 connection_id: ConnectionId,
2319 frame: Frame,
2320 request: ModuleControlRequestFromModule,
2321 ) -> Result<Vec<Frame>, RouterError> {
2322 match request {
2323 ModuleControlRequestFromModule::CatalogUpdate {
2324 provides,
2325 capabilities,
2326 ready,
2327 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2328 ModuleControlRequestFromModule::LiveRoots {} => {
2329 let registered = self
2330 .registry
2331 .get_module_by_connection(connection_id)
2332 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2333 let Some(registration) = registered else {
2334 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2335 };
2336 let response = self
2337 .forwarding
2338 .live_roots(®istration.manifest.module_id)
2339 .map_err(RouterError::Forwarding)?;
2340 Ok(vec![control_response_body_frame(
2341 &frame,
2342 &response,
2343 "ModuleControlResponseToModule::LiveRoots",
2344 )?])
2345 }
2346 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
2347 self.handle_scope_sync(connection_id, frame, generation, scopes)
2348 }
2349 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
2350 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
2351 }
2352 }
2353 }
2354
2355 fn handle_scope_sync(
2359 &self,
2360 connection_id: ConnectionId,
2361 frame: Frame,
2362 generation: u64,
2363 scopes: Vec<ScopeRecord>,
2364 ) -> Result<Vec<Frame>, RouterError> {
2365 let Some(registration) = self
2366 .registry
2367 .get_module_by_connection(connection_id)
2368 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2369 else {
2370 return Ok(vec![control_error_frame(
2371 &frame,
2372 "not_registered",
2373 "scope.sync requires an active module registration owned by this connection",
2374 )?]);
2375 };
2376 let owner = registration.manifest.module_id;
2377 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
2378 let is_current_launch = |connection: ConnectionId| {
2379 self.hello_launch_nonces
2380 .lock()
2381 .unwrap_or_else(|poisoned| poisoned.into_inner())
2382 .presented(connection, current_nonce.as_deref())
2383 };
2384 let mut table = self
2389 .scopes
2390 .write()
2391 .unwrap_or_else(|poisoned| poisoned.into_inner());
2392 let outcome = table.sync(&owner, connection_id, is_current_launch, generation, scopes);
2393 let drained = match &outcome {
2394 Ok(applied) => self
2395 .forwarding
2396 .publish_scope_changes(&applied.tag_changes)
2397 .map_err(RouterError::Forwarding)?,
2398 Err(_) => Vec::new(),
2399 };
2400 drop(table);
2401 match outcome {
2402 Ok(applied) => {
2403 let counts = ScopeOutcomeCounts::of(&applied.results);
2404 info!(
2405 owner = %owner,
2406 generation,
2407 records = applied.results.len(),
2408 created = counts.created,
2409 replaced = counts.replaced,
2410 updated = counts.updated,
2411 unchanged = counts.unchanged,
2412 refused = counts.refused,
2413 ended = applied.ended.len(),
2414 tag_changes = applied.tag_changes.len(),
2415 routes_closed = drained.len(),
2416 "scope sync accepted"
2417 );
2418 for refused in applied
2424 .results
2425 .iter()
2426 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
2427 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
2428 {
2429 warn!(
2430 owner = %owner,
2431 generation,
2432 scope_ref = %refused.scope_ref,
2433 scope_epoch = refused.scope_epoch,
2434 code = refused.code.as_deref().unwrap_or(""),
2435 "scope record refused"
2436 );
2437 }
2438 self.close_scope_drained_routes(drained);
2439 let response = ModuleControlResponseToModule::ScopeSync {
2440 generation,
2441 results: applied.results,
2442 ended: applied.ended,
2443 };
2444 Ok(vec![control_response_body_frame(
2445 &frame,
2446 &response,
2447 "ModuleControlResponseToModule::ScopeSync",
2448 )?])
2449 }
2450 Err(refusal) => {
2451 info!(
2452 owner = %owner,
2453 generation,
2454 code = refusal.code,
2455 "scope sync refused"
2456 );
2457 Ok(vec![control_error_frame(
2458 &frame,
2459 refusal.code,
2460 refusal.message,
2461 )?])
2462 }
2463 }
2464 }
2465
2466 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
2472 if drained.is_empty() {
2473 return;
2474 }
2475 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
2476 BTreeMap::new();
2477 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
2478 for route in drained {
2479 warn!(
2480 module_id = %route.module_id,
2481 reason = ?route.reason,
2482 client_connection_id = route.client.connection_id.get(),
2483 route_channel = route.client.channel,
2484 "closing route because its scope changed"
2485 );
2486 pushes
2487 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
2488 .or_insert_with(|| (route.reason, Vec::new()))
2489 .1
2490 .push(EndpointRoute {
2491 goodbye_target: route.client.clone(),
2492 principal: Principal::Unverified,
2493 bound_at: Instant::now(),
2494 draining: false,
2495 drain_reason: None,
2496 });
2497 goodbyes.push(route.module);
2498 goodbyes.push(route.client);
2499 }
2500 for ((module_id, _), (reason, routes)) in pushes {
2501 send_route_control_pushes(
2502 &self.forwarding,
2503 routes,
2504 ClientControlPush::RouteClosed {
2505 module_id,
2506 channels: Vec::new(),
2507 reason,
2508 drained: false,
2509 abandoned: 0,
2510 excluded_subscriptions: 0,
2511 terminal: Some(false),
2512 },
2513 );
2514 }
2515 self.emit_route_goodbyes(goodbyes);
2516 }
2517
2518 fn handle_scope_describe(
2521 &self,
2522 connection_id: ConnectionId,
2523 frame: Frame,
2524 owner: Principal,
2525 scope_ref: String,
2526 ) -> Result<Vec<Frame>, RouterError> {
2527 let registered = self
2528 .registry
2529 .get_module_by_connection(connection_id)
2530 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2531 if registered.is_none() {
2532 return Ok(vec![control_error_frame(
2533 &frame,
2534 "not_registered",
2535 "scope.describe requires an active module registration owned by this connection",
2536 )?]);
2537 }
2538 let description = self
2539 .scopes
2540 .read()
2541 .unwrap_or_else(|poisoned| poisoned.into_inner())
2542 .describe(&owner, &scope_ref);
2543 let owner_configured = match &owner {
2544 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
2549 _ => false,
2550 };
2551 let response = ModuleControlResponseToModule::ScopeDescribe {
2552 status: description.status,
2553 scope_epoch: description.scope_epoch,
2554 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
2555 owner_synced: description.owner_synced,
2556 owner_configured,
2557 scope: description.stamp,
2558 };
2559 Ok(vec![control_response_body_frame(
2560 &frame,
2561 &response,
2562 "ModuleControlResponseToModule::ScopeDescribe",
2563 )?])
2564 }
2565
2566 fn handle_catalog_update(
2567 &self,
2568 connection_id: ConnectionId,
2569 frame: Frame,
2570 provides: Vec<ProviderRole>,
2571 capabilities: Option<CapabilityDeclarations>,
2572 ready: Option<bool>,
2573 ) -> Result<Vec<Frame>, RouterError> {
2574 self.refresh_capability_requirements();
2575 let Some(registration) = self
2576 .registry
2577 .get_module_by_connection(connection_id)
2578 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2579 else {
2580 return Ok(vec![control_error_frame(
2581 &frame,
2582 "not_registered",
2583 "catalog.update requires an active module registration owned by this connection",
2584 )?]);
2585 };
2586
2587 if let Some(message) =
2588 catalog_update_frozen_field_message(®istration.manifest, &provides)
2589 {
2590 return Ok(vec![control_error_frame(
2591 &frame,
2592 "catalog_update_frozen_field",
2593 message,
2594 )?]);
2595 }
2596
2597 let mut candidate = registration.manifest.clone();
2598 candidate.provides = provides.clone();
2599 candidate.capabilities = capabilities
2600 .clone()
2601 .or_else(|| registration.manifest.capabilities.clone());
2602 if let Err(err) = candidate.validate_capability_grammar() {
2603 return Ok(vec![control_error_frame(
2604 &frame,
2605 "invalid_capability_grammar",
2606 err.to_string(),
2607 )?]);
2608 }
2609
2610 let mut conflicts = self
2613 .capability_evaluator
2614 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
2615 if let Some(conflict) = conflicts.first() {
2616 let message = format!(
2617 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2618 conflict.capability, conflict.claimants[0], candidate.module_id
2619 );
2620 for conflict in &mut conflicts {
2621 conflict.source = DuplicateClaimSource::CatalogUpdate;
2622 }
2623 log_duplicate_claim_events(conflicts);
2624 return Ok(vec![control_error_frame(
2625 &frame,
2626 "reserved_capability",
2627 message,
2628 )?]);
2629 }
2630
2631 let updated = self
2632 .registry
2633 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
2634 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2635 if updated.is_none() {
2636 return Ok(vec![control_error_frame(
2637 &frame,
2638 "not_registered",
2639 "catalog.update requires an active module registration owned by this connection",
2640 )?]);
2641 }
2642 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
2643 log_duplicate_claim_events(
2644 self.capability_evaluator
2645 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
2646 );
2647 }
2648 if capability_census_trigger(
2649 registration.manifest.capabilities.as_ref(),
2650 updated
2651 .as_ref()
2652 .and_then(|entry| entry.manifest.capabilities.as_ref()),
2653 ) {
2654 self.enforce_capability_denies();
2655 }
2656 self.refresh_capability_requirements();
2657
2658 let response = ModuleControlResponseToModule::CatalogUpdate {};
2659 control_response_body_frame(
2660 &frame,
2661 &response,
2662 "ModuleControlResponseToModule::CatalogUpdate",
2663 )
2664 .map(|frame| vec![frame])
2665 }
2666
2667 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
2668 self.refresh_capability_requirements();
2669 let mut counters = self.counters.snapshot();
2675 if let (Ok((connections_with_routes, max)), Some(obj)) = (
2676 self.forwarding.client_route_concentration(),
2677 counters.as_object_mut(),
2678 ) {
2679 obj.insert(
2680 "client_connections_with_routes".into(),
2681 connections_with_routes.into(),
2682 );
2683 obj.insert("max_routes_on_one_connection".into(), max.into());
2684 }
2685 if let (Some(open_breakers), Some(obj)) = (
2691 self.route_bind_breakers.open_snapshot(),
2692 counters.as_object_mut(),
2693 ) {
2694 obj.insert("route_bind_breakers_open".into(), open_breakers);
2695 }
2696 let response = ClientControlResponse::ServerDescribe {
2697 protocol_ver: PROTOCOL_VERSION,
2698 subc_ops: subc_ops(),
2699 capabilities: self.subc_capabilities.as_ref().to_vec(),
2700 connected_clients: self.connected_clients.count(),
2701 counters: Some(counters),
2702 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
2703 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
2704 capability_requirements: self.capability_requirement_statuses(),
2705 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2706 };
2707 Ok(vec![control_response_body_frame(
2708 &frame,
2709 &response,
2710 "ClientControlResponse::ServerDescribe",
2711 )?])
2712 }
2713
2714 fn handle_catalog_list(
2715 &self,
2716 frame: Frame,
2717 module_id: Option<String>,
2718 ) -> Result<Vec<Frame>, RouterError> {
2719 let (generation, modules) = self.registry.list_modules().map_err(|err| {
2720 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
2721 })?;
2722 let entries = modules
2723 .into_iter()
2724 .filter(|registration| {
2725 module_id
2726 .as_deref()
2727 .map(|wanted| registration.manifest.module_id == wanted)
2728 .unwrap_or(true)
2729 })
2730 .map(|registration| {
2731 let not_ready = self.not_ready_reason(®istration);
2732 let roles = registration.manifest.provides;
2733 CatalogEntry {
2734 module_id: registration.manifest.module_id,
2735 ready: not_ready.is_none(),
2736 not_ready,
2737 module_version: Some(registration.manifest.module_version),
2738 roles,
2739 control_ops: registration.control_ops,
2740 capabilities: registration.manifest.capabilities,
2741 self_signals: registration.manifest.self_signals,
2742 }
2743 })
2744 .collect();
2745 let response = ClientControlResponse::CatalogList {
2746 generation,
2747 modules: entries,
2748 subc_ops: subc_ops(),
2749 };
2750 Ok(vec![control_response_body_frame(
2751 &frame,
2752 &response,
2753 "ClientControlResponse::CatalogList",
2754 )?])
2755 }
2756
2757 fn route_open_principal(
2758 &self,
2759 frame: &Frame,
2760 consumer_identity: Option<ConsumerIdentity>,
2761 ) -> Result<Result<Principal, Frame>, RouterError> {
2762 let Some(consumer_identity) = consumer_identity else {
2763 return Ok(Ok(Principal::Direct));
2764 };
2765
2766 if self.supervisor.spawned_consumer_authorized(
2767 &consumer_identity.module_id,
2768 &consumer_identity.launch_nonce,
2769 ) {
2770 return Ok(Ok(Principal::Reserved {
2771 module_id: consumer_identity.module_id,
2772 }));
2773 }
2774
2775 Ok(Err(control_error_frame(
2776 frame,
2777 "bad_consumer_identity",
2778 format!(
2779 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
2780 consumer_identity.module_id
2781 ),
2782 )?))
2783 }
2784
2785 fn route_open_refusal_frame(
2792 &self,
2793 ctx: &RouteCtx,
2794 frame: &Frame,
2795 module_id: &str,
2796 reason: &'static str,
2797 code: &'static str,
2798 message: impl Into<String>,
2799 ) -> Result<Frame, RouterError> {
2800 self.observe_route_open_refusal(ctx, module_id, reason, code);
2801 control_error_frame(frame, code, message.into())
2802 }
2803
2804 fn route_open_breaker_refusal_frame(
2822 &self,
2823 ctx: &RouteCtx,
2824 frame: &Frame,
2825 module_id: &str,
2826 consecutive_timeouts: u32,
2827 retry_in: Duration,
2828 probe_in_flight: bool,
2829 ) -> Result<Frame, RouterError> {
2830 self.counters
2831 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
2832 debug!(
2833 target: "control",
2834 code = "module_timeout",
2835 module_id = ?module_id,
2836 connection_id = ctx.connection_id.get(),
2837 consecutive_timeouts,
2838 retry_in_ms = retry_in.as_millis() as u64,
2839 probe_in_flight,
2840 "route.open refused by open bind-relay breaker"
2841 );
2842 let detail = if probe_in_flight {
2849 "one test route is already being tried; retry once it settles".to_string()
2850 } else {
2851 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
2852 };
2853 control_error_frame(
2854 frame,
2855 "module_timeout",
2856 format!(
2857 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
2858 timed out in a row); {detail}; its established routes are unaffected"
2859 ),
2860 )
2861 }
2862
2863 fn observe_route_open_refusal(
2876 &self,
2877 ctx: &RouteCtx,
2878 module_id: &str,
2879 reason: &'static str,
2880 code: &'static str,
2881 ) {
2882 self.counters.increment_route_open_refused(code);
2883 info!(
2884 target: "control",
2885 code,
2886 reason,
2887 module_id = ?module_id,
2888 connection_id = ctx.connection_id.get(),
2889 "route.open refused"
2890 );
2891 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
2892 self.route_outages.record_not_serving(module_id, reason);
2893 }
2894 }
2895
2896 fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
2941 self.route_outages.record_accepted(module_id);
2942 self.counters.increment_route_open_accepted(principal);
2943 info!(
2944 target: "control",
2945 principal,
2946 module_id,
2947 connection_id = ctx.connection_id.get(),
2948 "route.open accepted"
2949 );
2950 }
2951
2952 fn supervised_absent_route_open_refusal_frame(
2953 &self,
2954 ctx: &RouteCtx,
2955 frame: &Frame,
2956 module_id: &str,
2957 code: &'static str,
2958 status: &crate::supervise::ModuleStatus,
2959 ) -> Result<Frame, RouterError> {
2960 self.counters.increment_route_open_refused(code);
2961 info!(
2962 target: "control",
2963 code,
2964 reason = "supervised_not_registered",
2965 module_id = ?module_id,
2966 connection_id = ctx.connection_id.get(),
2967 state = %status.state,
2968 enabled = status.enabled,
2969 live = status.live,
2970 "route.open refused"
2971 );
2972 self.route_outages
2976 .record_not_serving(module_id, "supervised_not_registered");
2977 control_error_frame(
2978 frame,
2979 code,
2980 format!(
2981 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
2982 status.state, status.enabled, status.live
2983 ),
2984 )
2985 }
2986
2987 async fn handle_route_open(
2988 &self,
2989 ctx: &RouteCtx,
2990 frame: Frame,
2991 request: RouteOpenRequest,
2992 ) -> Result<Vec<Frame>, RouterError> {
2993 let RouteOpenRequest {
2994 target,
2995 mut identity,
2996 consumer_identity,
2997 consumer_capabilities,
2998 role_versions,
2999 admission_facts,
3000 scope,
3001 } = request;
3002 let target_module_id = target_module_id(&target).to_string();
3003 if self
3004 .registry
3005 .get_module_by_connection(ctx.connection_id)
3006 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3007 .is_some()
3008 {
3009 return Ok(vec![control_error_frame(
3010 &frame,
3011 "invalid_request",
3012 "module connections cannot open client routes",
3013 )?]);
3014 }
3015 debug!(
3016 connection_id = ctx.connection_id.get(),
3017 corr = frame.header.corr,
3018 module_id = %target_module_id,
3019 "handling route.open"
3020 );
3021
3022 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
3028 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
3029 self.observe_route_open_refusal(
3030 ctx,
3031 &target_module_id,
3032 "invalid_role_versions",
3033 error_codes::INVALID_REQUEST,
3034 );
3035 return Ok(vec![control_error_body_frame(
3036 &frame,
3037 ErrorBody {
3038 code: error_codes::INVALID_REQUEST.to_string(),
3039 message: error.to_string(),
3040 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
3041 },
3042 )?]);
3043 }
3044
3045 let Some(registration) = self
3064 .registry
3065 .get_module(&target_module_id)
3066 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3067 else {
3068 if let Some((status, warming)) =
3069 self.supervisor_status(&target_module_id, frame.header.corr)?
3070 {
3071 if status.protocol == ModuleProtocol::None {
3078 return Ok(vec![self.route_open_refusal_frame(
3079 ctx,
3080 &frame,
3081 &target_module_id,
3082 "protocol_none",
3083 error_codes::MODULE_NO_PROTOCOL,
3084 format!(
3085 "module_id '{target_module_id}' is declared protocol: none; \
3086 it speaks no subc wire and serves no routes"
3087 ),
3088 )?]);
3089 }
3090 let code = if warming {
3091 "module_warming"
3092 } else {
3093 "target_unavailable"
3094 };
3095 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
3096 ctx,
3097 &frame,
3098 &target_module_id,
3099 code,
3100 &status,
3101 )?]);
3102 }
3103 if let Some(removed_ago_ms) =
3104 self.supervisor.removal_tombstone_age_ms(&target_module_id)
3105 {
3106 return Ok(vec![self.route_open_refusal_frame(
3107 ctx,
3108 &frame,
3109 &target_module_id,
3110 "removed",
3111 error_codes::MODULE_REMOVED,
3112 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
3113 )?]);
3114 }
3115 return Ok(vec![self.route_open_refusal_frame(
3116 ctx,
3117 &frame,
3118 &target_module_id,
3119 "not_registered",
3120 error_codes::UNKNOWN_MODULE,
3121 format!("module_id '{target_module_id}' is not registered"),
3122 )?]);
3123 };
3124
3125 if !registration.ready {
3129 self.counters
3130 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
3131 info!(
3132 target: "control",
3133 code = error_codes::MODULE_WARMING,
3134 module_id = ?target_module_id,
3135 connection_id = ctx.connection_id.get(),
3136 reason = "declared_not_ready",
3137 "route.open refused"
3138 );
3139 self.route_outages
3142 .record_not_serving(&target_module_id, "declared_not_ready");
3143 return Ok(vec![control_error_body_frame(
3144 &frame,
3145 ErrorBody {
3146 code: error_codes::MODULE_WARMING.to_string(),
3147 message: format!(
3148 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
3149 ),
3150 detail: Some(serde_json::json!({
3151 "reason": "declared_not_ready"
3152 })),
3153 },
3154 )?]);
3155 }
3156
3157 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
3176 self.counters
3177 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
3178 info!(
3179 target: "control",
3180 code = error_codes::MODULE_WARMING,
3181 module_id = ?target_module_id,
3182 connection_id = ctx.connection_id.get(),
3183 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3184 capability = %capability,
3185 "route.open refused"
3186 );
3187 return Ok(vec![control_error_body_frame(
3188 &frame,
3189 ErrorBody {
3190 code: error_codes::MODULE_WARMING.to_string(),
3191 message: format!(
3192 "module_id '{target_module_id}' requires capability '{capability}', \
3193 which no registered module provides; retry"
3194 ),
3195 detail: Some(serde_json::json!({
3196 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3197 "capability": capability,
3198 })),
3199 },
3200 )?]);
3201 }
3202
3203 if !target_has_required_role(&target, ®istration.manifest.provides) {
3204 return Ok(vec![self.route_open_refusal_frame(
3205 ctx,
3206 &frame,
3207 &target_module_id,
3208 "role_not_provided",
3209 "target_unavailable",
3210 format!("module_id '{target_module_id}' does not provide the requested target"),
3211 )?]);
3212 }
3213
3214 if registration.state != ChannelState::Active {
3215 return Ok(vec![self.route_open_refusal_frame(
3216 ctx,
3217 &frame,
3218 &target_module_id,
3219 "registration_not_active",
3220 "target_unavailable",
3221 format!("module_id '{target_module_id}' is not active"),
3222 )?]);
3223 }
3224
3225 if self
3226 .forwarding
3227 .module_is_draining(&target_module_id)
3228 .map_err(RouterError::Forwarding)?
3229 {
3230 return Ok(vec![self.route_open_refusal_frame(
3231 ctx,
3232 &frame,
3233 &target_module_id,
3234 "reloading",
3235 "module_reloading",
3236 format!("module_id '{target_module_id}' is reloading"),
3237 )?]);
3238 }
3239
3240 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
3241 process_liveness.process_live(&target_module_id) == Some(false)
3242 }) {
3243 if process_liveness.process_replacing(&target_module_id) {
3250 return Ok(vec![self.route_open_refusal_frame(
3251 ctx,
3252 &frame,
3253 &target_module_id,
3254 "reloading",
3255 "module_reloading",
3256 format!("module_id '{target_module_id}' is reloading"),
3257 )?]);
3258 }
3259 return Ok(vec![self.route_open_refusal_frame(
3260 ctx,
3261 &frame,
3262 &target_module_id,
3263 "supervisor_not_live",
3264 "target_unavailable",
3265 format!("module_id '{target_module_id}' is not live"),
3266 )?]);
3267 }
3268
3269 if !self
3270 .forwarding
3271 .has_live_module_connection(&target_module_id)
3272 .map_err(RouterError::Forwarding)?
3273 {
3274 return Ok(vec![self.route_open_refusal_frame(
3275 ctx,
3276 &frame,
3277 &target_module_id,
3278 "no_forwarding_connection",
3279 "target_unavailable",
3280 format!("module_id '{target_module_id}' has no live forwarding connection"),
3281 )?]);
3282 }
3283
3284 if let Some(error) =
3285 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
3286 {
3287 self.observe_route_open_refusal(
3288 ctx,
3289 &target_module_id,
3290 "op_not_allowed",
3291 "op_not_allowed",
3292 );
3293 return Ok(vec![error]);
3294 }
3295
3296 let principal = match self.route_open_principal(&frame, consumer_identity)? {
3297 Ok(principal) => principal,
3298 Err(error) => {
3299 self.observe_route_open_refusal(
3300 ctx,
3301 &target_module_id,
3302 "bad_consumer_identity",
3303 "bad_consumer_identity",
3304 );
3305 return Ok(vec![error]);
3306 }
3307 };
3308
3309 if let Principal::Reserved {
3313 module_id: opening_module_id,
3314 } = &principal
3315 {
3316 if let Some(opening_registration) = self
3317 .registry
3318 .get_module(opening_module_id)
3319 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3320 {
3321 if let Some(capability) =
3322 denied_capability(&opening_registration.manifest, ®istration.manifest)
3323 {
3324 warn!(
3325 opening_module_id,
3326 target_module_id,
3327 capability,
3328 "refusing route.open because an attested capability deny edge matches"
3329 );
3330 return Ok(vec![self.route_open_refusal_frame(
3331 ctx,
3332 &frame,
3333 &target_module_id,
3334 "capability_deny_edge",
3335 "capability_forbidden",
3336 format!(
3337 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
3338 ),
3339 )?]);
3340 }
3341 }
3342 }
3343
3344 if admission_facts.is_some() {
3345 let carrier_matches = matches!(
3346 &principal,
3347 Principal::Reserved { module_id }
3348 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
3349 );
3350 if !carrier_matches {
3351 return Ok(vec![self.route_open_refusal_frame(
3352 ctx,
3353 &frame,
3354 &target_module_id,
3355 "admission_facts_carrier_not_permitted",
3356 "admission_facts_not_permitted",
3357 "admission facts may only be carried by the configured reserved module",
3358 )?]);
3359 }
3360
3361 let target_allowed = self
3362 .admission_facts_targets
3363 .as_ref()
3364 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
3365 if !target_allowed {
3366 return Ok(vec![self.route_open_refusal_frame(
3367 ctx,
3368 &frame,
3369 &target_module_id,
3370 "admission_facts_target_not_listed",
3371 "admission_facts_target_not_allowed",
3372 format!(
3373 "admission facts are not permitted for target module_id '{target_module_id}'"
3374 ),
3375 )?]);
3376 }
3377
3378 }
3382
3383 let (bound_scope, scope_stamp) = match scope {
3389 None => (None, None),
3390 Some(selector) => {
3391 let owner_configured = match &selector.owner {
3392 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3398 _ => false,
3399 };
3400 let admitted = self
3401 .scopes
3402 .read()
3403 .unwrap_or_else(|poisoned| poisoned.into_inner())
3404 .admit(&principal, &target_module_id, &selector, owner_configured)
3405 .and_then(|admission| {
3406 crate::scopes::check_target_flow_support(
3407 &admission.stamp,
3408 &target_module_id,
3409 registration.manifest.capabilities.as_ref(),
3410 )?;
3411 Ok(admission)
3412 });
3413 match admitted {
3414 Ok(admission) => (
3415 Some(BoundScope {
3416 owner: admission.owner,
3417 scope_ref: admission.stamp.scope_ref.clone(),
3418 tag: admission.tag,
3419 }),
3420 Some(admission.stamp),
3421 ),
3422 Err(refusal) => {
3423 return Ok(vec![self.route_open_refusal_frame(
3424 ctx,
3425 &frame,
3426 &target_module_id,
3427 refusal.code,
3428 refusal.code,
3429 refusal.message,
3430 )?]);
3431 }
3432 }
3433 }
3434 };
3435
3436 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
3458 Ok(project_root) => project_root,
3459 Err(err) => {
3460 return Ok(vec![control_error_frame(
3461 &frame,
3462 "invalid_project_root",
3463 err.to_string(),
3464 )?])
3465 }
3466 };
3467 identity.project_root = project_root.as_path().to_path_buf();
3468
3469 let _concurrency_guard = match self
3480 .route_bind_concurrency
3481 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
3482 {
3483 Ok(guard) => guard,
3484 Err(in_flight) => {
3485 return Ok(vec![self.route_open_target_capacity_refusal(
3486 ctx,
3487 &frame,
3488 &target_module_id,
3489 in_flight,
3490 )?]);
3491 }
3492 };
3493
3494 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
3497 RouteBindAdmission::Admitted { guard, probe } => {
3498 if probe {
3499 info!(
3500 module_id = %target_module_id,
3501 connection_id = ctx.connection_id.get(),
3502 "route.bind breaker half-open: admitting one probe"
3503 );
3504 }
3505 guard
3506 }
3507 RouteBindAdmission::Refused {
3508 consecutive_timeouts,
3509 retry_in,
3510 probe_in_flight,
3511 } => {
3512 return Ok(vec![self.route_open_breaker_refusal_frame(
3513 ctx,
3514 &frame,
3515 &target_module_id,
3516 consecutive_timeouts,
3517 retry_in,
3518 probe_in_flight,
3519 )?]);
3520 }
3521 };
3522
3523 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
3528 let relay_deadline = Instant::now() + route_bind_relay_timeout;
3529 let pending = match self
3530 .forwarding
3531 .begin_route_bind_relay_for(
3532 ctx.connection_id,
3533 ctx.egress.clone(),
3534 response_version(&frame),
3535 frame.header.corr,
3536 &target_module_id,
3537 principal.clone(),
3538 bound_scope,
3539 Some(project_root),
3540 relay_deadline,
3541 )
3542 .await
3543 {
3544 Ok(pending) => pending,
3545 Err(err) => {
3546 return Ok(vec![self.route_open_refusal_frame(
3547 ctx,
3548 &frame,
3549 &target_module_id,
3550 "relay_reservation_failed",
3551 forwarding_error_code(&err),
3552 err.to_string(),
3553 )?])
3554 }
3555 };
3556 let crate::forwarding::PendingRouteBindRelay {
3557 endpoint,
3558 module_sink,
3559 negotiated_ver,
3560 client_channel,
3561 client_epoch,
3562 module_channel,
3563 module_epoch,
3564 corr: relay_corr,
3565 receiver,
3566 } = pending;
3567 let mut reservation =
3568 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
3569
3570 if let Some(stamp) = scope_stamp
3575 .as_ref()
3576 .filter(|stamp| stamp.attributes.flow_id.is_some())
3577 {
3578 let relay_registration = self
3579 .registry
3580 .get_module_by_connection(endpoint.connection_id)
3581 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3582 if let Err(refusal) = crate::scopes::check_target_flow_support(
3583 stamp,
3584 &target_module_id,
3585 relay_registration
3586 .as_ref()
3587 .and_then(|registration| registration.manifest.capabilities.as_ref()),
3588 ) {
3589 reservation.release_and_disarm();
3590 return Ok(vec![self.route_open_refusal_frame(
3591 ctx,
3592 &frame,
3593 &target_module_id,
3594 refusal.code,
3595 refusal.code,
3596 refusal.message,
3597 )?]);
3598 }
3599 }
3600
3601 debug!(
3602 connection_id = ctx.connection_id.get(),
3603 client_channel,
3604 client_epoch,
3605 module_channel,
3606 module_epoch,
3607 "reserved route handle pair"
3608 );
3609 let principal_label = match &principal {
3612 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
3613 Principal::Direct => "direct".to_string(),
3614 other => format!("{other:?}"),
3615 };
3616 let relay = ModuleControlRequest::RouteBind {
3617 route_channel: module_channel,
3618 epoch: module_epoch,
3619 target,
3620 identity,
3621 principal: Some(principal),
3622 consumer_capabilities,
3623 role_versions,
3624 admission_facts,
3625 scope: scope_stamp,
3626 };
3627 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
3628 RouterError::backend(
3629 0,
3630 frame.header.corr,
3631 format!("failed to encode route.bind request: {err}"),
3632 )
3633 })?;
3634 let relay_frame = Frame::build_with_version(
3635 negotiated_ver,
3636 FrameType::Request,
3637 control_flags(),
3638 0,
3639 0,
3640 relay_corr,
3641 relay_body,
3642 )
3643 .map_err(RouterError::FrameBuild)?;
3644
3645 if let Err(err) = module_sink.send(relay_frame).await {
3646 reservation.release_and_disarm();
3647 return Ok(vec![self.route_open_refusal_frame(
3648 ctx,
3649 &frame,
3650 &target_module_id,
3651 "relay_send_failed",
3652 "target_unavailable",
3653 err.to_string(),
3654 )?]);
3655 }
3656
3657 if !self
3658 .forwarding
3659 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
3660 .map_err(RouterError::Forwarding)?
3661 {
3662 self.send_abandoned_route_bind_goodbye(
3663 &module_sink,
3664 negotiated_ver,
3665 module_channel,
3666 module_epoch,
3667 );
3668 }
3669
3670 match timeout_at(relay_deadline, receiver).await {
3671 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
3672 reservation.disarm();
3673 if breaker.record_accepted() {
3674 info!(
3675 module_id = %target_module_id,
3676 "route.bind breaker closed: the probe was accepted"
3677 );
3678 }
3679 self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
3680 Ok(Vec::new())
3681 }
3682 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
3683 reservation.release_and_disarm();
3684 breaker.record_inconclusive();
3688 let scope_code = match body.code.as_str() {
3692 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
3693 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
3694 _ => None,
3695 };
3696 if let Some(code) = scope_code {
3697 self.observe_route_open_refusal(
3698 ctx,
3699 &target_module_id,
3700 "scope_changed_before_commit",
3701 code,
3702 );
3703 return Ok(vec![control_error_body_frame(&frame, body)?]);
3704 }
3705 self.counters
3706 .increment_route_open_refused("module_rejected");
3707 info!(
3708 target: "control",
3709 code = "module_rejected",
3710 module_code = ?body.code,
3711 module_id = ?target_module_id,
3712 connection_id = ctx.connection_id.get(),
3713 "route.open refused"
3714 );
3715 Ok(vec![control_error_body_frame(&frame, body)?])
3716 }
3717 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
3718 reservation.release_and_disarm();
3719 breaker.record_inconclusive();
3720 tracing::warn!(
3727 module_id = %target_module_id,
3728 "route.bind relay abandoned: {message}"
3729 );
3730 Ok(vec![self.route_open_refusal_frame(
3731 ctx,
3732 &frame,
3733 &target_module_id,
3734 "relay_abandoned",
3735 "target_unavailable",
3736 message,
3737 )?])
3738 }
3739 Ok(Err(_)) => {
3740 reservation.release_and_disarm();
3741 breaker.record_inconclusive();
3742 Ok(vec![self.route_open_refusal_frame(
3743 ctx,
3744 &frame,
3745 &target_module_id,
3746 "relay_waiter_canceled",
3747 "target_unavailable",
3748 "route.bind relay waiter was canceled before the module responded",
3749 )?])
3750 }
3751 Err(_) => {
3752 reservation.release_and_disarm();
3753 if let Some(opened) = breaker.record_timeout(
3757 self.route_bind_breaker_threshold,
3758 self.route_bind_breaker_cooldown,
3759 ) {
3760 warn!(
3761 module_id = %target_module_id,
3762 consecutive_timeouts = opened.consecutive_timeouts,
3763 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
3764 reopened_after_probe = opened.reopened_after_probe,
3765 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
3766 );
3767 }
3768 tracing::warn!(
3776 module_id = %target_module_id,
3777 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
3778 "route.bind relay timed out: module did not ack within budget"
3779 );
3780 Ok(vec![self.route_open_refusal_frame(
3781 ctx,
3782 &frame,
3783 &target_module_id,
3784 "relay_timed_out",
3785 "module_timeout",
3786 format!(
3787 "module_id '{target_module_id}' did not answer route.bind within {:?}",
3788 route_bind_relay_timeout
3789 ),
3790 )?])
3791 }
3792 }
3793 }
3794
3795 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3796 let response = ClientControlResponse::SupervisorSpawnSnapshot {
3797 snapshot: self.supervisor.spawn_snapshot(),
3798 };
3799 Ok(vec![control_response_body_frame(
3800 &frame,
3801 &response,
3802 "ClientControlResponse::SupervisorSpawnSnapshot",
3803 )?])
3804 }
3805
3806 fn handle_supervisor_spawn_subscribe(
3807 &self,
3808 ctx: &RouteCtx,
3809 frame: Frame,
3810 since: Option<SpawnCursor>,
3811 ) -> Result<Vec<Frame>, RouterError> {
3812 match self.supervisor.subscribe_spawns(
3813 ctx.connection_id,
3814 frame.header.corr,
3815 response_version(&frame),
3816 since,
3817 ctx.egress.clone(),
3818 ) {
3819 Ok(()) => Ok(Vec::new()),
3820 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
3821 Ok(vec![control_error_body_frame(
3822 &frame,
3823 ErrorBody {
3824 code: "spawn_cursor_incarnation_mismatch".to_string(),
3825 message: "spawn cursor belongs to a different daemon incarnation"
3826 .to_string(),
3827 detail: Some(serde_json::json!({
3828 "current_daemon_incarnation": current
3829 })),
3830 },
3831 )?])
3832 }
3833 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
3834 &frame,
3835 ErrorBody {
3836 code: "spawn_cursor_too_old".to_string(),
3837 message: "spawn cursor predates the retained event ring".to_string(),
3838 detail: Some(serde_json::json!({
3839 "oldest_retained_cursor": oldest
3840 })),
3841 },
3842 )?]),
3843 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
3844 0,
3845 frame.header.corr,
3846 format!("failed to open supervisor spawn subscription: {error}"),
3847 )),
3848 }
3849 }
3850
3851 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3852 let generation = self
3853 .registry
3854 .generation()
3855 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3856 let mut modules = Vec::new();
3857 for module in self.supervisor.list() {
3858 let status = module.status_for_control("list").map_err(|err| {
3859 RouterError::backend(
3860 0,
3861 frame.header.corr,
3862 format!("failed to read supervisor status: {err}"),
3863 )
3864 })?;
3865 let (configured, _) = module.configuration().map_err(|err| {
3866 RouterError::backend(
3867 0,
3868 frame.header.corr,
3869 format!("failed to read module configuration: {err}"),
3870 )
3871 })?;
3872 let image = module.running_image_agreement().await;
3874 let resources = Some(module.child_resource_usage());
3877 let pending_reload = Some(reload_verdict(
3878 &configured.program,
3879 status.spawned_from.as_deref(),
3880 image,
3881 ));
3882 modules.push(SupervisorEntry {
3883 launch_nonce_env: Some(!cfg!(unix)),
3886 module_id: status.module_id,
3887 state: status.state.to_string(),
3888 enabled: status.enabled,
3889 live: status.live,
3890 protocol: status.protocol,
3891 health: status.health.status,
3892 pending_reload,
3893 last_probe_ms: status.health.last_probe_ms,
3894 last_exit_code: status.last_exit.as_ref().and_then(|e| e.code),
3895 last_exit_signal: status.last_exit.as_ref().and_then(|e| e.signal),
3896 last_exit_ms: status.last_exit.as_ref().map(|e| e.at_ms),
3897 last_exit_kind: status.last_exit.as_ref().map(|e| e.kind.into()),
3898 restart_count: Some(status.restart_count),
3899 max_restarts: Some(status.max_restarts),
3900 lifetime_restarts: Some(status.lifetime_restarts),
3901 spawn_generation: Some(status.spawn_generation),
3902 restart_window_secs: Some(status.restart_window.as_secs()),
3903 drain_timeout_ms: Some(status.drain_timeout.as_millis() as u64),
3904 restart_backoff_ms: Some(status.restart_backoff.as_millis() as u64),
3905 restart_max_backoff_ms: Some(status.restart_max_backoff.as_millis() as u64),
3906 resources,
3907 });
3908 }
3909 let response = ClientControlResponse::SupervisorList {
3910 generation,
3911 modules,
3912 };
3913 Ok(vec![control_response_body_frame(
3914 &frame,
3915 &response,
3916 "ClientControlResponse::SupervisorList",
3917 )?])
3918 }
3919
3920 fn handle_supervisor_stderr_tail(
3921 &self,
3922 frame: Frame,
3923 module_id: String,
3924 max_lines: Option<u32>,
3925 max_bytes: Option<u32>,
3926 ) -> Result<Vec<Frame>, RouterError> {
3927 let Some(module) = self.supervisor.get(&module_id) else {
3928 return Ok(vec![control_error_frame(
3929 &frame,
3930 "unknown_module",
3931 format!("module_id '{module_id}' is not supervised"),
3932 )?]);
3933 };
3934
3935 let snapshot = module.stderr_tail(
3936 max_lines.map(|value| value as usize),
3937 max_bytes.map(|value| value as usize),
3938 );
3939
3940 let response = ClientControlResponse::SupervisorStderrTail {
3941 module_id,
3942 tail: StderrTail {
3943 capture: match snapshot.capture {
3944 CaptureState::Captured => StderrCaptureState::Captured,
3945 CaptureState::Incomplete { reason } => {
3946 StderrCaptureState::Incomplete { reason }
3947 }
3948 CaptureState::NotCaptured { reason } => {
3949 StderrCaptureState::NotCaptured { reason }
3950 }
3951 },
3952 entries: snapshot
3953 .entries
3954 .into_iter()
3955 .map(|entry| match entry {
3956 TailEntry::Line {
3957 text,
3958 truncated,
3959 at_ms,
3960 } => StderrTailEntry::Line {
3961 text,
3962 truncated,
3963 at_ms,
3964 },
3965 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
3966 })
3967 .collect(),
3968 dropped_lines: snapshot.dropped_lines,
3969 },
3970 };
3971 Ok(vec![control_response_body_frame(
3972 &frame,
3973 &response,
3974 "ClientControlResponse::SupervisorStderrTail",
3975 )?])
3976 }
3977
3978 async fn handle_supervisor_terminals(
3979 &self,
3980 frame: Frame,
3981 module_id: String,
3982 ) -> Result<Vec<Frame>, RouterError> {
3983 let Some(module) = self.supervisor.get(&module_id) else {
3984 return Ok(vec![control_error_frame(
3985 &frame,
3986 "unknown_module",
3987 format!("module_id '{module_id}' is not supervised"),
3988 )?]);
3989 };
3990
3991 let terminals = module
3994 .read_durable_terminal_history()
3995 .await
3996 .map_err(|error| {
3997 RouterError::backend(
3998 0,
3999 frame.header.corr,
4000 format!("failed to read terminal history: {error}"),
4001 )
4002 })?;
4003 let response = ClientControlResponse::SupervisorTerminals {
4004 module_id,
4005 terminals,
4006 };
4007 Ok(vec![control_response_body_frame(
4008 &frame,
4009 &response,
4010 "ClientControlResponse::SupervisorTerminals",
4011 )?])
4012 }
4013
4014 fn handle_supervisor_routes(
4015 &self,
4016 frame: Frame,
4017 module_id: Option<String>,
4018 ) -> Result<Vec<Frame>, RouterError> {
4019 let modules = self
4020 .forwarding
4021 .route_census(module_id.as_deref())
4022 .map_err(RouterError::Forwarding)?
4023 .into_iter()
4024 .map(|(module_id, routes)| SupervisorRouteModule {
4025 module_id,
4026 routes: routes
4027 .into_iter()
4028 .map(|route| SupervisorRoute {
4029 consumer: match route.principal {
4030 Principal::Reserved { module_id } => {
4031 SupervisorRouteConsumer::Reserved { module_id }
4032 }
4033 Principal::Direct | Principal::Unverified => {
4034 SupervisorRouteConsumer::Direct {
4035 connection_id: route.goodbye_target.connection_id.get(),
4036 }
4037 }
4038 },
4039 age_ms: Instant::now()
4040 .saturating_duration_since(route.bound_at)
4041 .as_millis()
4042 .try_into()
4043 .unwrap_or(u64::MAX),
4044 draining: route.draining,
4045 drain_reason: route.drain_reason,
4046 })
4047 .collect(),
4048 })
4049 .collect();
4050 let response = ClientControlResponse::SupervisorRoutes { modules };
4051 Ok(vec![control_response_body_frame(
4052 &frame,
4053 &response,
4054 "ClientControlResponse::SupervisorRoutes",
4055 )?])
4056 }
4057
4058 async fn handle_supervisor_provenance(
4059 &self,
4060 frame: Frame,
4061 module_id: Option<String>,
4062 ) -> Result<Vec<Frame>, RouterError> {
4063 let mut selected = if let Some(module_id) = module_id {
4064 let Some(module) = self.supervisor.get(&module_id) else {
4065 return Ok(vec![control_error_frame(
4066 &frame,
4067 "unknown_module",
4068 format!("module_id '{module_id}' is not supervised"),
4069 )?]);
4070 };
4071 vec![module]
4072 } else {
4073 self.supervisor.list()
4074 };
4075
4076 let mut modules = Vec::with_capacity(selected.len());
4077 for module in selected.drain(..) {
4078 let (status, observed_image) = module
4079 .status_and_running_image_agreement()
4080 .await
4081 .map_err(|err| {
4082 RouterError::backend(
4083 0,
4084 frame.header.corr,
4085 format!("failed to read supervisor status: {err}"),
4086 )
4087 })?;
4088 let module_declared = self
4089 .registry
4090 .get_module(&status.module_id)
4091 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4092 .and_then(|registration| registration.manifest.provenance)
4093 .map(|build| ModuleDeclaredProvenance::Reported { build })
4094 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
4095 #[cfg(test)]
4096 let running_image = match &self.provenance_probe_override {
4097 Some(result) => result.clone(),
4098 None => observed_image,
4099 };
4100 #[cfg(not(test))]
4101 let running_image = observed_image;
4102 modules.push(SupervisorModuleProvenance {
4103 module_id: status.module_id,
4104 module_declared,
4105 daemon_observed: SupervisorObservedProcess {
4106 pid: status.pid,
4107 spawned_at_ms: status.spawned_at_ms,
4108 spawned_from: status.spawned_from,
4109 running_image,
4110 },
4111 });
4112 }
4113 let daemon = SupervisorDaemonProvenance {
4114 daemon_build: self.daemon_provenance.build.clone(),
4115 daemon_observed: DaemonObservedProcess {
4116 pid: self.daemon_provenance.pid,
4117 started_at_ms: self
4118 .daemon_provenance
4119 .start_clock
4120 .map(|clock| clock.started_at_ms())
4121 .or(self.daemon_provenance.started_at_ms),
4122 running_image: self
4123 .daemon_provenance
4124 .probe
4125 .observe(
4126 self.daemon_provenance.pid,
4127 self.daemon_provenance.executable_path.as_deref(),
4128 self.daemon_provenance.executable_identity,
4129 self.daemon_provenance.process_start_time,
4130 )
4131 .await,
4132 },
4133 };
4134 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
4135 Ok(vec![control_response_body_frame(
4136 &frame,
4137 &response,
4138 "ClientControlResponse::SupervisorProvenance",
4139 )?])
4140 }
4141
4142 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4143 self.refresh_capability_requirements();
4144 let generation = self
4145 .registry
4146 .generation()
4147 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4148 let modules = self
4149 .supervisor
4150 .list()
4151 .into_iter()
4152 .map(|module| {
4153 let status = module.status_for_control("health").map_err(|err| {
4154 RouterError::backend(
4155 0,
4156 frame.header.corr,
4157 format!("failed to read supervisor health: {err}"),
4158 )
4159 })?;
4160 let module_id = status.module_id;
4161 let capability_detail = self
4162 .capability_evaluator
4163 .required_problem_detail(&module_id);
4164 Ok(SupervisorHealthEntry {
4165 module_id,
4166 status: status.health.status,
4167 detail: append_capability_problem_detail(
4168 status.health.detail,
4169 capability_detail,
4170 ),
4171 metrics: status.health.metrics,
4172 consecutive_failures: status.health.consecutive_failures,
4173 late_answer_count: status.health.late_answer_count,
4174 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
4175 last_action: status.health.last_action,
4176 last_action_ms: status.health.last_action_ms,
4177 last_probe_ms: status.health.last_probe_ms,
4178 })
4179 })
4180 .collect::<Result<Vec<_>, RouterError>>()?;
4181 let response = ClientControlResponse::SupervisorHealth {
4182 generation,
4183 modules,
4184 };
4185 Ok(vec![control_response_body_frame(
4186 &frame,
4187 &response,
4188 "ClientControlResponse::SupervisorHealth",
4189 )?])
4190 }
4191
4192 async fn handle_supervisor_restart(
4193 &self,
4194 frame: Frame,
4195 module_id: String,
4196 drain_timeout_ms: Option<u64>,
4197 ) -> Result<Vec<Frame>, RouterError> {
4198 let operation_lock = self.supervisor.operation_lock();
4199 let _operation_guard = operation_lock.lock().await;
4200 let Some(module) = self.supervisor.get(&module_id) else {
4201 return Ok(vec![control_error_frame(
4202 &frame,
4203 "unknown_module",
4204 format!("module_id '{module_id}' is not supervised"),
4205 )?]);
4206 };
4207
4208 self.route_outages.mark_operator_action(&module_id);
4209 if let Err(err) = module.restart(drain_timeout_ms).await {
4210 self.route_outages
4211 .operator_action_ended_unrefused(&module_id);
4212 let (code, message) = match err {
4213 crate::supervise::SuperviseError::Disabled { .. } => {
4214 ("module_disabled", err.to_string())
4215 }
4216 crate::supervise::SuperviseError::SwapInProgress { .. } => {
4217 ("swap_in_progress", err.to_string())
4218 }
4219 _ => (
4220 "target_unavailable",
4221 format!("failed to restart module_id '{module_id}': {err}"),
4222 ),
4223 };
4224 return Ok(vec![control_error_frame(&frame, code, message)?]);
4225 }
4226
4227 let response = ClientControlResponse::SupervisorAck {
4228 module_id,
4229 applied: true,
4230 };
4231 Ok(vec![control_response_body_frame(
4232 &frame,
4233 &response,
4234 "ClientControlResponse::SupervisorAck",
4235 )?])
4236 }
4237
4238 async fn handle_supervisor_swap(
4242 &self,
4243 frame: Frame,
4244 module_id: String,
4245 ready_timeout_ms: Option<u64>,
4246 ) -> Result<Vec<Frame>, RouterError> {
4247 let module = {
4254 let operation_lock = self.supervisor.operation_lock();
4255 let _operation_guard = operation_lock.lock().await;
4256 self.supervisor.get(&module_id)
4257 };
4258 let Some(module) = module else {
4259 return Ok(vec![control_error_frame(
4260 &frame,
4261 "unknown_module",
4262 format!("module_id '{module_id}' is not supervised"),
4263 )?]);
4264 };
4265
4266 self.route_outages.mark_operator_action(&module_id);
4267 if let Err(err) = module
4268 .swap(ready_timeout_ms.map(Duration::from_millis))
4269 .await
4270 {
4271 self.route_outages
4272 .operator_action_ended_unrefused(&module_id);
4273 use crate::supervise::SuperviseError;
4274 let message = err.to_string();
4275 let error = match err {
4276 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
4277 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
4278 code: "swap_refused".to_string(),
4279 message,
4280 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
4281 },
4282 SuperviseError::SwapFailed {
4283 arm,
4284 candidate_exit,
4285 ..
4286 } => ErrorBody {
4287 code: "swap_failed".to_string(),
4288 message,
4289 detail: Some(serde_json::json!({
4290 "arm": arm.as_str(),
4291 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
4292 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
4293 })),
4294 },
4295 _ => ErrorBody::new(
4296 "target_unavailable",
4297 format!("failed to swap module_id '{module_id}': {message}"),
4298 ),
4299 };
4300 return Ok(vec![control_error_body_frame(&frame, error)?]);
4301 }
4302 self.route_outages
4306 .operator_action_ended_unrefused(&module_id);
4307
4308 let response = ClientControlResponse::SupervisorAck {
4309 module_id,
4310 applied: true,
4311 };
4312 Ok(vec![control_response_body_frame(
4313 &frame,
4314 &response,
4315 "ClientControlResponse::SupervisorAck",
4316 )?])
4317 }
4318
4319 async fn handle_supervisor_reload(
4320 &self,
4321 frame: Frame,
4322 module_id: String,
4323 ) -> Result<Vec<Frame>, RouterError> {
4324 let operation_lock = self.supervisor.operation_lock();
4325 let _operation_guard = operation_lock.lock().await;
4326 let Some(module) = self.supervisor.get(&module_id) else {
4327 return Ok(vec![control_error_frame(
4328 &frame,
4329 "unknown_module",
4330 format!("module_id '{module_id}' is not supervised"),
4331 )?]);
4332 };
4333
4334 self.route_outages.mark_operator_action(&module_id);
4335 if let Err(err) = module.reload().await {
4336 self.route_outages
4337 .operator_action_ended_unrefused(&module_id);
4338 let (code, message) = match err {
4339 crate::supervise::SuperviseError::Disabled { .. } => {
4340 ("module_disabled", err.to_string())
4341 }
4342 crate::supervise::SuperviseError::SwapInProgress { .. } => {
4343 ("swap_in_progress", err.to_string())
4344 }
4345 _ => (
4346 "reload_failed",
4347 format!("failed to reload module_id '{module_id}': {err}"),
4348 ),
4349 };
4350 return Ok(vec![control_error_frame(&frame, code, message)?]);
4351 }
4352
4353 let response = ClientControlResponse::SupervisorAck {
4354 module_id,
4355 applied: true,
4356 };
4357 Ok(vec![control_response_body_frame(
4358 &frame,
4359 &response,
4360 "ClientControlResponse::SupervisorAck",
4361 )?])
4362 }
4363
4364 async fn handle_supervisor_rescan(
4365 &self,
4366 frame: Frame,
4367 preview: bool,
4368 ) -> Result<Vec<Frame>, RouterError> {
4369 let Some(context) = self.rescan.clone() else {
4370 return Ok(vec![control_error_frame(
4371 &frame,
4372 "rescan_unavailable",
4373 "the daemon was not started with a reloadable config path".to_string(),
4374 )?]);
4375 };
4376
4377 let operation_lock = self.supervisor.operation_lock();
4378 let _operation_guard = operation_lock.lock().await;
4379 let loaded = match crate::daemon_config::load(&context.config_path) {
4380 Ok(config) => config,
4381 Err(err) => {
4382 return Ok(vec![control_error_frame(
4383 &frame,
4384 "invalid_daemon_config",
4385 format!("supervisor rescan rejected daemon config: {err}"),
4386 )?])
4387 }
4388 };
4389 let Some(config) = loaded else {
4397 return Ok(vec![control_error_frame(
4398 &frame,
4399 "invalid_daemon_config",
4400 format!(
4401 "daemon config not found at {}; refusing to rescan (an absent config would \
4402 retire every supervised module)",
4403 context.config_path.display()
4404 ),
4405 )?]);
4406 };
4407 let (
4408 configured_port,
4409 storage_config,
4410 admission_facts_carrier_module_id,
4411 admission_facts_targets,
4412 scope_authority_owners,
4413 modules,
4414 reserved_capabilities,
4415 ) = (
4416 config.port,
4417 config.storage,
4418 config.admission_facts_carrier_module_id,
4419 config.admission_facts_targets,
4420 config.scope_authority_owners,
4421 config.modules,
4422 config.reserved_capabilities,
4423 );
4424
4425 let mut restart_required = Vec::new();
4434 for section in RestartRequiredSection::ALL {
4435 let changed = match section {
4436 RestartRequiredSection::Port => configured_port != context.configured_port,
4437 RestartRequiredSection::Storage => storage_config != context.storage_config,
4438 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
4439 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
4440 }
4441 RestartRequiredSection::AdmissionFactsTargets => {
4442 admission_facts_targets != context.admission_facts_targets
4443 }
4444 RestartRequiredSection::ScopeAuthorityOwners => {
4445 scope_authority_owners != context.scope_authority_owners
4446 }
4447 };
4448 if changed {
4449 restart_required.push(section.label().to_string());
4450 }
4451 }
4452 if !restart_required.is_empty() {
4453 warn!(
4454 config_path = %context.config_path.display(),
4455 sections = %restart_required.join(", "),
4456 "daemon config changed outside the modules section; restart the daemon to apply those changes"
4457 );
4458 }
4459
4460 for configured in &modules {
4461 if let Err(err) = validate_spec(&configured.module_spec()) {
4462 return Ok(vec![control_error_frame(
4463 &frame,
4464 "invalid_daemon_config",
4465 format!("supervisor rescan rejected daemon config: {err}"),
4466 )?]);
4467 }
4468 }
4469
4470 let configured_capabilities = modules
4471 .iter()
4472 .map(|module| (module.module_id.clone(), module.enabled))
4473 .collect::<Vec<_>>();
4474 let preview_capability_warnings = if preview {
4475 let (_, registrations) = self.runtime_capability_snapshot()?;
4476 let current_modules = self
4477 .supervisor
4478 .list()
4479 .into_iter()
4480 .map(|module| module.module_id().to_string())
4481 .collect::<BTreeSet<_>>();
4482 let resulting_modules = configured_capabilities.clone();
4483 let removed = current_modules
4484 .into_iter()
4485 .filter(|module_id| {
4486 !resulting_modules
4487 .iter()
4488 .any(|(configured_id, _)| configured_id == module_id)
4489 })
4490 .collect::<Vec<_>>();
4491 self.capability_evaluator.preview_removal_warnings(
4492 resulting_modules,
4493 &removed,
4494 ®istrations,
4495 )
4496 } else {
4497 Vec::new()
4498 };
4499 let result = match self
4500 .reconcile_supervised_modules(&context.supervisor, modules, preview)
4501 .await
4502 {
4503 Ok(result) => result,
4504 Err(message) => {
4505 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
4506 }
4507 };
4508 if !preview {
4509 self.capability_evaluator
4510 .configure(configured_capabilities, reserved_capabilities);
4511 self.capability_evaluator.wake_deadline_loop();
4512 self.refresh_capability_requirements();
4513 }
4514 let mut result = result;
4515 result.restart_required = restart_required;
4516 result.capability_warnings = preview_capability_warnings;
4517 let response = ClientControlResponse::SupervisorRescan { result };
4518 Ok(vec![control_response_body_frame(
4519 &frame,
4520 &response,
4521 "ClientControlResponse::SupervisorRescan",
4522 )?])
4523 }
4524
4525 async fn handle_supervisor_release_reserved(
4526 &self,
4527 frame: Frame,
4528 module_id: String,
4529 ) -> Result<Vec<Frame>, RouterError> {
4530 let Some(context) = self.rescan.clone() else {
4531 return Ok(vec![control_error_frame(
4532 &frame,
4533 "release_unavailable",
4534 "reserved-id release requires a daemon started with a reloadable config path",
4535 )?]);
4536 };
4537 let operation_lock = self.supervisor.operation_lock();
4538 let _operation_guard = operation_lock.lock().await;
4539 let loaded = match crate::daemon_config::load(&context.config_path) {
4540 Ok(Some(config)) => config,
4541 Ok(None) => {
4542 return Ok(vec![control_error_frame(
4543 &frame,
4544 "invalid_daemon_config",
4545 format!(
4546 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
4547 context.config_path.display()
4548 ),
4549 )?])
4550 }
4551 Err(err) => {
4552 return Ok(vec![control_error_frame(
4553 &frame,
4554 "invalid_daemon_config",
4555 format!("unable to verify reserved-id release against daemon config: {err}"),
4556 )?])
4557 }
4558 };
4559 if loaded
4560 .modules
4561 .iter()
4562 .any(|configured| configured.module_id == module_id)
4563 {
4564 return Ok(vec![control_error_frame(
4565 &frame,
4566 "reserved_module_configured",
4567 format!(
4568 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
4569 ),
4570 )?]);
4571 }
4572 if !self.supervisor.release_retained_reserved_gate(&module_id) {
4573 return Ok(vec![control_error_frame(
4574 &frame,
4575 "reserved_gate_not_retained",
4576 format!(
4577 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
4578 ),
4579 )?]);
4580 }
4581
4582 let response = ClientControlResponse::SupervisorAck {
4583 module_id,
4584 applied: true,
4585 };
4586 Ok(vec![control_response_body_frame(
4587 &frame,
4588 &response,
4589 "ClientControlResponse::SupervisorAck",
4590 )?])
4591 }
4592
4593 async fn reconcile_supervised_modules(
4602 &self,
4603 supervisor: &Supervisor,
4604 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
4605 preview: bool,
4606 ) -> Result<SupervisorRescanResult, String> {
4607 let mut current = BTreeMap::new();
4608 for module in self.supervisor.list() {
4609 let (spec, health) = module.configuration().map_err(|err| {
4610 format!(
4611 "failed to read configuration for module_id '{}': {err}",
4612 module.module_id()
4613 )
4614 })?;
4615 let enabled = module
4616 .status()
4617 .map_err(|err| {
4618 format!(
4619 "failed to read status for module_id '{}': {err}",
4620 module.module_id()
4621 )
4622 })?
4623 .enabled;
4624 current.insert(
4625 module.module_id().to_string(),
4626 (module, spec, health, enabled),
4627 );
4628 }
4629 let configured = configured_modules
4630 .into_iter()
4631 .map(|module| (module.module_id.clone(), module))
4632 .collect::<BTreeMap<_, _>>();
4633
4634 let added = configured
4635 .keys()
4636 .filter(|module_id| !current.contains_key(*module_id))
4637 .cloned()
4638 .collect::<Vec<_>>();
4639 let removed = current
4640 .keys()
4641 .filter(|module_id| !configured.contains_key(*module_id))
4642 .cloned()
4643 .collect::<Vec<_>>();
4644 let mut changed_pending_reload = Vec::new();
4645 let mut configuration_changes = BTreeSet::new();
4646 let mut enabled_changes = BTreeSet::new();
4647 let mut unchanged = 0_u32;
4648
4649 for (module_id, configured_module) in &configured {
4650 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
4651 else {
4652 continue;
4653 };
4654 let launch_changed = *current_spec != configured_module.module_spec();
4658 let configuration_changed =
4659 launch_changed || *current_health != configured_module.health;
4660 let enabled_changed = *current_enabled != configured_module.enabled;
4661 if configuration_changed {
4662 configuration_changes.insert(module_id.clone());
4663 }
4664 if launch_changed {
4665 changed_pending_reload.push(module_id.clone());
4666 }
4667 if enabled_changed {
4668 enabled_changes.insert(module_id.clone());
4669 }
4670 if !configuration_changed && !enabled_changed {
4671 unchanged = unchanged.saturating_add(1);
4672 }
4673 }
4674
4675 if preview {
4681 return Ok(SupervisorRescanResult {
4682 added,
4683 removed,
4684 changed_pending_reload,
4685 enabled_changes: enabled_changes.iter().cloned().collect(),
4686 unchanged,
4687 preview: true,
4688 restart_required: Vec::new(),
4692 capability_warnings: Vec::new(),
4693 });
4694 }
4695
4696 for module_id in &removed {
4697 let module = ¤t
4698 .get(module_id)
4699 .expect("removed module came from current supervisor state")
4700 .0;
4701 module.retire().await.map_err(|err| {
4702 format!("failed to retire module_id '{module_id}' during rescan: {err}")
4703 })?;
4704 self.supervisor.record_rescan_removal(module_id);
4738 self.supervisor.retire(module_id);
4739 self.route_outages.forget(module_id);
4740 }
4741
4742 for module_id in configured.keys() {
4743 let Some((module, _, _, _)) = current.get(module_id) else {
4744 continue;
4745 };
4746 let configured_module = configured
4747 .get(module_id)
4748 .expect("configured module id came from configured map");
4749 if configuration_changes.contains(module_id) {
4750 module
4751 .update_configuration(
4752 configured_module.module_spec(),
4753 configured_module.health.clone(),
4754 configured_module.drain_timeout_ms,
4755 )
4756 .await
4757 .map_err(|err| {
4758 format!(
4759 "failed to update module_id '{module_id}' configuration during rescan: {err}"
4760 )
4761 })?;
4762 }
4763 if enabled_changes.contains(module_id) {
4764 self.route_outages.mark_operator_action(module_id);
4767 module
4768 .set_enabled(configured_module.enabled)
4769 .await
4770 .map_err(|err| {
4771 self.route_outages.operator_action_ended_unrefused(module_id);
4772 format!(
4773 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
4774 configured_module.enabled
4775 )
4776 })?;
4777 }
4778 }
4779
4780 for module_id in &added {
4781 let configured_module = configured
4782 .get(module_id)
4783 .expect("added module id came from configured map");
4784 supervisor
4785 .supervise_configured_with_health(
4786 configured_module.module_spec(),
4787 configured_module.enabled,
4788 configured_module.health.clone(),
4789 configured_module.drain_timeout_ms,
4790 configured_module.restart,
4791 )
4792 .map_err(|err| {
4793 format!("failed to add module_id '{module_id}' during rescan: {err}")
4794 })?;
4795 }
4796
4797 Ok(SupervisorRescanResult {
4798 added,
4799 removed,
4800 changed_pending_reload,
4801 enabled_changes: enabled_changes.iter().cloned().collect(),
4802 unchanged,
4803 preview: false,
4804 restart_required: Vec::new(),
4807 capability_warnings: Vec::new(),
4808 })
4809 }
4810
4811 async fn handle_supervisor_set_enabled(
4812 &self,
4813 frame: Frame,
4814 module_id: String,
4815 enabled: bool,
4816 ) -> Result<Vec<Frame>, RouterError> {
4817 let operation_lock = self.supervisor.operation_lock();
4818 let _operation_guard = operation_lock.lock().await;
4819 let Some(module) = self.supervisor.get(&module_id) else {
4820 return Ok(vec![control_error_frame(
4821 &frame,
4822 "unknown_module",
4823 format!("module_id '{module_id}' is not supervised"),
4824 )?]);
4825 };
4826
4827 self.route_outages.mark_operator_action(&module_id);
4830 let applied = match module.set_enabled(enabled).await {
4831 Ok(applied) => applied,
4832 Err(err) => {
4833 self.route_outages
4834 .operator_action_ended_unrefused(&module_id);
4835 return Ok(vec![control_error_frame(
4836 &frame,
4837 "target_unavailable",
4838 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
4839 )?]);
4840 }
4841 };
4842 if !applied {
4843 self.route_outages
4846 .operator_action_ended_unrefused(&module_id);
4847 }
4848
4849 self.capability_evaluator.wake_deadline_loop();
4850 self.refresh_capability_requirements();
4851 let response = ClientControlResponse::SupervisorAck { module_id, applied };
4852 Ok(vec![control_response_body_frame(
4853 &frame,
4854 &response,
4855 "ClientControlResponse::SupervisorAck",
4856 )?])
4857 }
4858
4859 async fn handle_supervisor_health_probe(
4860 &self,
4861 frame: Frame,
4862 module_id: String,
4863 ) -> Result<Vec<Frame>, RouterError> {
4864 self.refresh_capability_requirements();
4865 let Some(registration) = self
4866 .registry
4867 .get_module(&module_id)
4868 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4869 else {
4870 return Ok(vec![control_error_frame(
4871 &frame,
4872 "unknown_module",
4873 format!("module_id '{module_id}' is not registered"),
4874 )?]);
4875 };
4876
4877 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
4890 {
4891 return Ok(vec![control_error_frame(
4892 &frame,
4893 "health_not_advertised",
4894 format!("module_id '{module_id}' did not advertise health.check"),
4895 )?]);
4896 }
4897
4898 let deadline = Instant::now() + self.health_probe_timeout;
4899 let pending = match self.forwarding.begin_module_control_rpc_for(
4900 &module_id,
4901 MODULE_CONTROL_OP_HEALTH_CHECK,
4902 deadline,
4903 ) {
4904 Ok(pending) => pending,
4905 Err(err) => {
4906 return Ok(vec![control_error_frame(
4907 &frame,
4908 forwarding_error_code(&err),
4909 err.to_string(),
4910 )?])
4911 }
4912 };
4913
4914 let PendingModuleControlRpc {
4915 endpoint,
4916 module_sink,
4917 negotiated_ver,
4918 corr: probe_corr,
4919 receiver,
4920 } = pending;
4921 let mut guard =
4922 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
4923 let probe_body =
4924 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
4925 RouterError::backend(
4926 0,
4927 frame.header.corr,
4928 format!("failed to encode health.check request: {err}"),
4929 )
4930 })?;
4931 let probe_frame = Frame::build_with_version(
4932 negotiated_ver,
4933 FrameType::Request,
4934 control_flags(),
4935 0,
4936 0,
4937 probe_corr,
4938 probe_body,
4939 )
4940 .map_err(RouterError::FrameBuild)?;
4941
4942 if let Err(err) = module_sink.send(probe_frame).await {
4943 return Ok(vec![control_error_frame(
4944 &frame,
4945 "target_unavailable",
4946 err.to_string(),
4947 )?]);
4948 }
4949
4950 match timeout_at(deadline, receiver).await {
4951 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
4952 guard.disarm();
4953 let Some(report) = response.health_report() else {
4954 return Ok(vec![control_error_frame(
4955 &frame,
4956 "invalid_control_body",
4957 "health.check RPC returned a non-health response",
4958 )?]);
4959 };
4960 let HealthReport {
4965 status,
4966 detail,
4967 metrics,
4968 } = report;
4969 let capability_detail = self
4970 .capability_evaluator
4971 .required_problem_detail(&module_id);
4972 let response = ClientControlResponse::SupervisorHealthProbe {
4973 module_id,
4974 status,
4975 detail: append_capability_problem_detail(detail, capability_detail),
4976 metrics,
4977 };
4978 Ok(vec![control_response_body_frame(
4979 &frame,
4980 &response,
4981 "ClientControlResponse::SupervisorHealthProbe",
4982 )?])
4983 }
4984 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
4985 guard.disarm();
4986 Ok(vec![control_error_body_frame(&frame, body)?])
4987 }
4988 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
4989 guard.disarm();
4990 Ok(vec![control_error_frame(
4991 &frame,
4992 "target_unavailable",
4993 message,
4994 )?])
4995 }
4996 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
4997 guard.disarm();
4998 Ok(vec![control_error_frame(
4999 &frame,
5000 "invalid_control_body",
5001 message,
5002 )?])
5003 }
5004 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5005 guard.disarm();
5006 Ok(vec![control_error_frame(
5007 &frame,
5008 "invalid_control_body",
5009 format!("expected module-control op '{expected}', got '{actual}'"),
5010 )?])
5011 }
5012 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
5013 guard.disarm();
5014 Ok(vec![control_error_frame(
5015 &frame,
5016 "module_timeout",
5017 format!(
5018 "module_id '{module_id}' answered health.check after {:?}",
5019 self.health_probe_timeout
5020 ),
5021 )?])
5022 }
5023 Ok(Err(_)) => Ok(vec![control_error_frame(
5024 &frame,
5025 "target_unavailable",
5026 "health.check waiter was canceled before the module responded",
5027 )?]),
5028 Err(_) => Ok(vec![control_error_frame(
5029 &frame,
5030 "module_timeout",
5031 format!(
5032 "module_id '{module_id}' did not answer health.check within {:?}",
5033 self.health_probe_timeout
5034 ),
5035 )?]),
5036 }
5037 }
5038
5039 fn supervisor_status(
5040 &self,
5041 module_id: &str,
5042 corr: u64,
5043 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
5044 self.supervisor
5045 .get(module_id)
5046 .map(|module| {
5047 let warming = module.is_warming_for_control("status").map_err(|err| {
5048 RouterError::backend(
5049 0,
5050 corr,
5051 format!(
5052 "failed to read supervisor warming state for module_id '{module_id}': {err}"
5053 ),
5054 )
5055 })?;
5056 module.status_for_control("status").map_err(|err| {
5057 RouterError::backend(
5058 0,
5059 corr,
5060 format!(
5061 "failed to read supervisor status for module_id '{module_id}': {err}"
5062 ),
5063 )
5064 }).map(|status| (status, warming))
5065 })
5066 .transpose()
5067 }
5068
5069 fn guard_module_control_op(
5070 &self,
5071 frame: &Frame,
5072 module_id: &str,
5073 op: &str,
5074 ) -> Result<Option<Frame>, RouterError> {
5075 if self.module_grants_op(module_id, op, frame.header.corr)? {
5076 return Ok(None);
5077 }
5078
5079 Ok(Some(control_error_frame(
5080 frame,
5081 "op_not_allowed",
5082 format!("module_id '{module_id}' did not grant control op '{op}'"),
5083 )?))
5084 }
5085
5086 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
5087 let Some(registration) = self
5088 .registry
5089 .get_module(module_id)
5090 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
5091 else {
5092 return Ok(false);
5093 };
5094 Ok(module_registration_grants_op(®istration.control_ops, op))
5095 }
5096
5097 fn handle_status_update(
5098 &self,
5099 endpoint: ModuleEndpointId,
5100 frame: Frame,
5101 ) -> Result<Vec<Frame>, RouterError> {
5102 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
5103 Ok(update) => update,
5104 Err(err) => {
5105 if is_known_module_push_op(&frame.body) {
5110 return Ok(vec![control_error_frame(
5111 &frame,
5112 "invalid_control_body",
5113 format!("malformed module control push body: {err}"),
5114 )?]);
5115 }
5116 return Ok(Vec::new());
5117 }
5118 };
5119
5120 match update {
5121 ModuleControlPush::RouteStatus {
5122 route_channel,
5123 route_epoch,
5124 status,
5125 } => {
5126 self.forwarding
5127 .cache_status(endpoint, route_channel, route_epoch, status)
5128 .map_err(RouterError::Forwarding)?;
5129 }
5130 }
5131 Ok(Vec::new())
5132 }
5133
5134 fn handle_route_poll(
5135 &self,
5136 ctx: &RouteCtx,
5137 frame: Frame,
5138 route_channel: u16,
5139 route_epoch: u32,
5140 kind: PollKind,
5141 ) -> Result<Vec<Frame>, RouterError> {
5142 let snapshot = self
5143 .forwarding
5144 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
5145 .map_err(RouterError::Forwarding)?;
5146 let response = match (kind, snapshot) {
5147 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
5148 ClientControlResponse::RoutePoll {
5149 route_channel,
5150 route_epoch,
5151 status,
5152 live: None,
5153 }
5154 }
5155 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
5156 route_channel,
5157 route_epoch,
5158 status: None,
5159 live: None,
5160 },
5161 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
5162 let live = self
5188 .process_liveness
5189 .as_ref()
5190 .and_then(|source| source.process_live(&module_id))
5191 .unwrap_or(true);
5192 ClientControlResponse::RoutePoll {
5193 route_channel,
5194 route_epoch,
5195 status: None,
5196 live: Some(live),
5197 }
5198 }
5199 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
5200 route_channel,
5201 route_epoch,
5202 status: None,
5203 live: Some(false),
5204 },
5205 };
5206
5207 Ok(vec![control_response_body_frame(
5208 &frame,
5209 &response,
5210 "ClientControlResponse::RoutePoll",
5211 )?])
5212 }
5213
5214 pub(crate) fn observe_module_control_completion(
5215 &self,
5216 completion: ModuleControlRpcCompletion,
5217 ) -> bool {
5218 match completion {
5219 ModuleControlRpcCompletion::Unknown => false,
5220 ModuleControlRpcCompletion::Settled => true,
5221 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
5222 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
5223 info!(
5224 module_id = %module_id,
5225 latency_ms,
5226 "late health.check answer proves the module is alive"
5227 );
5228 match self
5229 .supervisor
5230 .record_late_health_answer(&module_id, latency_ms)
5231 {
5232 Ok(true) => {}
5233 Ok(false) => debug!(
5234 module_id = %module_id,
5235 latency_ms,
5236 "late health.check answer has no active supervisor snapshot"
5237 ),
5238 Err(err) => warn!(
5239 module_id = %module_id,
5240 latency_ms,
5241 error = %err,
5242 "failed to record late health.check answer"
5243 ),
5244 }
5245 true
5246 }
5247 }
5248 }
5249
5250 fn refuse_to_end_module_connection_for_a_client(
5271 &self,
5272 module_connection_id: ConnectionId,
5273 corr: u64,
5274 err: ForwardingError,
5275 ) -> Result<(), RouterError> {
5276 if let ForwardingError::ConnectionClosing { connection_id } = err {
5277 if connection_id != module_connection_id {
5278 warn!(
5279 module_connection_id = module_connection_id.get(),
5280 client_connection_id = connection_id.get(),
5281 corr,
5282 "dropping a route.bind response for a closing client; the module connection keeps serving"
5283 );
5284 return Ok(());
5285 }
5286 }
5287 Err(RouterError::Forwarding(err))
5288 }
5289
5290 fn handle_module_relay_response(
5291 &self,
5292 connection_id: ConnectionId,
5293 frame: Frame,
5294 ) -> Result<Vec<Frame>, RouterError> {
5295 let mut secondary_error = None;
5296 let outcome = match frame.header.ty {
5297 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
5298 Ok(probe) if probe.op == "route.bind" => {
5299 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
5300 Ok(ModuleControlResponse::RouteBindAck {}) => {
5301 RouteBindRelayOutcome::Accepted
5302 }
5303 Ok(other) => {
5304 let message =
5305 format!("route.bind response carried unexpected body: {other:?}");
5306 secondary_error = Some(control_error_frame(
5307 &frame,
5308 "invalid_control_body",
5309 message.clone(),
5310 )?);
5311 RouteBindRelayOutcome::ModuleGone(message)
5312 }
5313 Err(err) => {
5314 let message = format!("malformed route.bind response body: {err}");
5315 secondary_error = Some(control_error_frame(
5316 &frame,
5317 "invalid_control_body",
5318 message.clone(),
5319 )?);
5320 RouteBindRelayOutcome::ModuleGone(message)
5321 }
5322 }
5323 }
5324 Ok(probe) => {
5325 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
5326 {
5327 Ok(response) => ModuleControlRpcOutcome::Response(response),
5328 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
5329 "malformed {} response body: {err}",
5330 probe.op
5331 )),
5332 };
5333 let completion = self
5334 .forwarding
5335 .complete_module_control_rpc(
5336 connection_id,
5337 frame.header.corr,
5338 Some(&probe.op),
5339 outcome,
5340 )
5341 .map_err(RouterError::Forwarding)?;
5342 if !self.observe_module_control_completion(completion) {
5343 debug!(
5344 connection_id = connection_id.get(),
5345 corr = frame.header.corr,
5346 op = %probe.op,
5347 "dropping late or unknown module-control RPC response"
5348 );
5349 }
5350 return Ok(Vec::new());
5351 }
5352 Err(err) => {
5353 if let Some(expected_op) = self
5354 .forwarding
5355 .pending_module_control_op(connection_id, frame.header.corr)
5356 .map_err(RouterError::Forwarding)?
5357 {
5358 let completion = self
5359 .forwarding
5360 .complete_module_control_rpc(
5361 connection_id,
5362 frame.header.corr,
5363 None,
5364 ModuleControlRpcOutcome::MalformedResponse(format!(
5365 "malformed {expected_op} response body: {err}"
5366 )),
5367 )
5368 .map_err(RouterError::Forwarding)?;
5369 if !self.observe_module_control_completion(completion) {
5370 debug!(
5371 connection_id = connection_id.get(),
5372 corr = frame.header.corr,
5373 "dropping late malformed module-control RPC response"
5374 );
5375 }
5376 return Ok(Vec::new());
5377 }
5378 let message = format!("malformed route.bind response body: {err}");
5379 secondary_error = Some(control_error_frame(
5380 &frame,
5381 "invalid_control_body",
5382 message.clone(),
5383 )?);
5384 RouteBindRelayOutcome::ModuleGone(message)
5385 }
5386 },
5387 FrameType::Error => {
5388 if self
5389 .forwarding
5390 .pending_module_control_op(connection_id, frame.header.corr)
5391 .map_err(RouterError::Forwarding)?
5392 .is_some()
5393 {
5394 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
5395 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
5396 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
5397 "malformed module-control ERROR body: {err}"
5398 )),
5399 };
5400 let completion = self
5401 .forwarding
5402 .complete_module_control_rpc(
5403 connection_id,
5404 frame.header.corr,
5405 None,
5406 outcome,
5407 )
5408 .map_err(RouterError::Forwarding)?;
5409 if !self.observe_module_control_completion(completion) {
5410 debug!(
5411 connection_id = connection_id.get(),
5412 corr = frame.header.corr,
5413 "dropping late or unknown module-control RPC error"
5414 );
5415 }
5416 return Ok(Vec::new());
5417 }
5418 match serde_json::from_slice::<ErrorBody>(&frame.body) {
5419 Ok(body) => RouteBindRelayOutcome::Rejected(body),
5420 Err(err) => {
5421 let message = format!("malformed route.bind ERROR body: {err}");
5422 secondary_error = Some(control_error_frame(
5423 &frame,
5424 "invalid_control_body",
5425 message.clone(),
5426 )?);
5427 RouteBindRelayOutcome::ModuleGone(message)
5428 }
5429 }
5430 }
5431 ty => {
5432 return Ok(vec![control_error_frame(
5433 &frame,
5434 "unsupported_control_frame",
5435 format!("unsupported module channel-0 frame {ty:?}"),
5436 )?])
5437 }
5438 };
5439
5440 let settled =
5441 self.forwarding
5442 .complete_pending_relay(connection_id, frame.header.corr, outcome);
5443 let completion = match settled {
5444 Ok(completion) => completion,
5445 Err(err) => {
5446 self.refuse_to_end_module_connection_for_a_client(
5447 connection_id,
5448 frame.header.corr,
5449 err,
5450 )?;
5451 return Ok(secondary_error.into_iter().collect());
5452 }
5453 };
5454 if let Some(target) = completion.abandoned.as_ref() {
5455 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
5456 }
5457 if !completion.settled {
5458 debug!(
5459 connection_id = connection_id.get(),
5460 corr = frame.header.corr,
5461 frame_type = ?frame.header.ty,
5462 "dropping late or unknown route.bind relay response"
5463 );
5464 }
5465 Ok(secondary_error.into_iter().collect())
5466 }
5467
5468 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
5469 debug!(connection_id = connection_id.get(), "handling GOODBYE");
5470 self.cleanup_connection(connection_id)
5474 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
5475 Ok(Vec::new())
5476 }
5477}
5478
5479impl Default for ControlHandler {
5480 fn default() -> Self {
5481 Self::new(Arc::new(Registry::default()))
5482 }
5483}
5484
5485impl crate::supervise::SwapPromotionObserver for ControlHandler {
5486 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
5487 self.apply_registration_capabilities(registration);
5488 }
5489}
5490
5491fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
5492 CapabilityRequirementStatus {
5493 consumer: status.consumer,
5494 capability: status.capability,
5495 need: match status.need {
5496 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
5497 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
5498 },
5499 verdict: status.verdict.as_str().to_string(),
5500 episode_seq: status.episode_seq,
5501 config_satisfiable: status.config_satisfiable,
5502 runtime_available: status.runtime_available,
5503 detail: status.detail,
5504 }
5505}
5506
5507fn append_capability_problem_detail(
5508 detail: Option<String>,
5509 capability_detail: Option<String>,
5510) -> Option<String> {
5511 match (detail, capability_detail) {
5512 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
5513 (Some(detail), None) => Some(detail),
5514 (None, Some(capability_detail)) => Some(capability_detail),
5515 (None, None) => None,
5516 }
5517}
5518
5519fn subc_ops() -> Vec<String> {
5520 SUBC_CONTROL_OPS
5521 .iter()
5522 .map(|op| (*op).to_string())
5523 .collect()
5524}
5525
5526fn module_subc_ops() -> Vec<String> {
5527 SUBC_CONTROL_OPS
5528 .iter()
5529 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
5530 .map(|op| (*op).to_string())
5531 .collect()
5532}
5533
5534#[cfg(test)]
5535fn module_baseline_control_ops() -> Vec<String> {
5536 MODULE_BASELINE_CONTROL_OPS
5537 .iter()
5538 .map(|op| (*op).to_string())
5539 .collect()
5540}
5541
5542fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
5543 let mut seen = HashSet::new();
5544 let mut effective = Vec::new();
5545 for op in MODULE_BASELINE_CONTROL_OPS {
5546 if seen.insert((*op).to_string()) {
5547 effective.push((*op).to_string());
5548 }
5549 }
5550 for op in declared.unwrap_or_default() {
5551 if seen.insert(op.clone()) {
5552 effective.push(op);
5553 }
5554 }
5555 effective
5556}
5557
5558fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
5559 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
5560}
5561
5562fn target_module_id(target: &RouteTarget) -> &str {
5563 match target {
5564 RouteTarget::ToolProvider { module_id }
5565 | RouteTarget::ManagementSurface { module_id }
5566 | RouteTarget::InternalService { module_id, .. } => module_id,
5567 }
5568}
5569
5570fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
5571 roles.iter().any(|role| match (target, role) {
5572 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
5573 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
5574 (
5575 RouteTarget::InternalService { service_id, .. },
5576 ProviderRole::InternalService {
5577 service_id: provided,
5578 ..
5579 },
5580 ) => service_id == provided,
5581 _ => false,
5582 })
5583}
5584
5585fn is_routable_role(role: &ProviderRole) -> bool {
5586 matches!(
5587 role,
5588 ProviderRole::ToolProvider { .. }
5589 | ProviderRole::ManagementSurface { .. }
5590 | ProviderRole::InternalService { .. }
5591 )
5592}
5593
5594#[derive(Debug, Clone, Copy, PartialEq, Eq)]
5595enum ControlRequestBodyError {
5596 UnknownOp,
5597 InvalidBody,
5598}
5599
5600#[derive(Debug, Deserialize)]
5601struct ControlOpProbe {
5602 op: String,
5603}
5604
5605const MODULE_PUSH_OPS: &[&str] = &["route.status"];
5608
5609fn is_known_module_push_op(body: &[u8]) -> bool {
5610 serde_json::from_slice::<ControlOpProbe>(body)
5611 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
5612 .unwrap_or(false)
5613}
5614
5615fn is_known_module_request_op(body: &[u8]) -> bool {
5616 serde_json::from_slice::<ControlOpProbe>(body)
5617 .map(|probe| is_module_to_subc_op(&probe.op))
5618 .unwrap_or(false)
5619}
5620
5621fn is_module_to_subc_op(op: &str) -> bool {
5622 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
5623}
5624
5625fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
5626 debug!(
5627 op = %op,
5628 connection_id = connection_id.get(),
5629 corr,
5630 "control dispatch"
5631 );
5632}
5633
5634fn log_slow_control_dispatch(
5635 dispatch_started_at: Option<StdInstant>,
5636 op: &'static str,
5637 connection_id: ConnectionId,
5638 corr: u64,
5639) {
5640 let Some(dispatch_started_at) = dispatch_started_at else {
5641 return;
5642 };
5643 let elapsed = dispatch_started_at.elapsed();
5644 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
5645 warn!(
5646 op = %op,
5647 connection_id = connection_id.get(),
5648 corr,
5649 elapsed_ms = elapsed.as_millis() as u64,
5650 "slow control dispatch"
5651 );
5652 }
5653}
5654
5655fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
5656 match request {
5657 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
5658 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
5659 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
5660 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
5661 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
5662 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
5663 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
5664 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
5665 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
5666 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
5667 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
5668 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
5669 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
5670 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
5671 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
5672 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
5673 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
5674 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
5675 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
5676 }
5677}
5678
5679fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
5680 match request {
5681 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
5682 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
5683 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
5684 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
5685 }
5686}
5687
5688fn parse_client_control_request(
5689 body: &[u8],
5690) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
5691 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
5692 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5693 Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
5694 ControlRequestBodyError::InvalidBody
5695 }
5696 Ok(_) => ControlRequestBodyError::UnknownOp,
5697 Err(_) => ControlRequestBodyError::InvalidBody,
5698 };
5699 (err, classification)
5700 })
5701}
5702
5703fn parse_module_control_request_from_module(
5704 body: &[u8],
5705) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
5706 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
5707 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5708 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
5709 Ok(_) => ControlRequestBodyError::UnknownOp,
5710 Err(_) => ControlRequestBodyError::InvalidBody,
5711 };
5712 (err, classification)
5713 })
5714}
5715
5716#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
5717enum ProviderRoleKind {
5718 ToolProvider,
5719 PipelineStage,
5720 ManagementSurface,
5721 InternalService,
5722}
5723
5724fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
5725 match role {
5726 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
5727 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
5728 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
5729 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
5730 }
5731}
5732
5733fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
5734 roles.iter().map(provider_role_kind).collect()
5735}
5736
5737const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
5740
5741#[derive(Debug, Default, PartialEq, Eq)]
5743struct ScopeOutcomeCounts {
5744 created: usize,
5745 replaced: usize,
5746 updated: usize,
5747 unchanged: usize,
5748 refused: usize,
5749}
5750
5751impl ScopeOutcomeCounts {
5752 fn of(results: &[ScopeRecordResult]) -> Self {
5753 let mut counts = Self::default();
5754 for result in results {
5755 let slot = match result.outcome {
5756 ScopeRecordOutcome::Created => &mut counts.created,
5757 ScopeRecordOutcome::Replaced => &mut counts.replaced,
5758 ScopeRecordOutcome::Updated => &mut counts.updated,
5759 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
5760 ScopeRecordOutcome::Refused => &mut counts.refused,
5761 };
5762 *slot += 1;
5763 }
5764 counts
5765 }
5766}
5767
5768#[cfg(test)]
5769mod scope_outcome_count_tests {
5770 use super::*;
5771
5772 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
5773 ScopeRecordResult {
5774 scope_ref: "r".to_string(),
5775 scope_epoch: 1,
5776 outcome,
5777 code: None,
5778 message: None,
5779 version: None,
5780 parent_state: None,
5781 }
5782 }
5783
5784 #[test]
5787 fn every_outcome_is_counted_in_its_own_field() {
5788 let results = [
5789 result(ScopeRecordOutcome::Created),
5790 result(ScopeRecordOutcome::Created),
5791 result(ScopeRecordOutcome::Replaced),
5792 result(ScopeRecordOutcome::Updated),
5793 result(ScopeRecordOutcome::Unchanged),
5794 result(ScopeRecordOutcome::Refused),
5795 result(ScopeRecordOutcome::Refused),
5796 result(ScopeRecordOutcome::Refused),
5797 ];
5798 assert_eq!(
5799 ScopeOutcomeCounts::of(&results),
5800 ScopeOutcomeCounts {
5801 created: 2,
5802 replaced: 1,
5803 updated: 1,
5804 unchanged: 1,
5805 refused: 3,
5806 }
5807 );
5808 }
5809}
5810
5811fn capability_census_trigger(
5815 old: Option<&CapabilityDeclarations>,
5816 new: Option<&CapabilityDeclarations>,
5817) -> bool {
5818 let old_provides = old
5819 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
5820 .unwrap_or_default();
5821 let old_denies = old
5822 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
5823 .unwrap_or_default();
5824 let new = new.cloned().unwrap_or(CapabilityDeclarations {
5825 provides: Vec::new(),
5826 requires: Vec::new(),
5827 must_never_reach: Vec::new(),
5828 });
5829
5830 new.provides
5831 .iter()
5832 .any(|capability| !old_provides.contains(capability))
5833 || new
5834 .must_never_reach
5835 .iter()
5836 .any(|capability| !old_denies.contains(capability))
5837}
5838
5839fn denied_capability<'a>(
5842 opening_manifest: &'a ModuleManifest,
5843 target_manifest: &ModuleManifest,
5844) -> Option<&'a str> {
5845 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
5846 let target_capabilities = target_manifest.capabilities.as_ref()?;
5847 opening_capabilities
5848 .must_never_reach
5849 .iter()
5850 .find(|denied| {
5851 target_capabilities
5852 .provides
5853 .iter()
5854 .any(|provided| provided == *denied)
5855 })
5856 .map(String::as_str)
5857}
5858
5859fn catalog_update_frozen_field_message(
5860 registered: &ModuleManifest,
5861 provides: &[ProviderRole],
5862) -> Option<String> {
5863 let old_has_provides = !registered.provides.is_empty();
5864 let new_has_provides = !provides.is_empty();
5865 if old_has_provides != new_has_provides {
5866 return Some(format!(
5867 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
5868 registered.module_id
5869 ));
5870 }
5871
5872 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
5873 return Some(format!(
5874 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
5875 registered.module_id
5876 ));
5877 }
5878
5879 let registered_concurrency = manifest_concurrency(registered);
5880 let mut candidate = registered.clone();
5881 candidate.provides = provides.to_vec();
5882 let candidate_concurrency = manifest_concurrency(&candidate);
5883 if candidate_concurrency != registered_concurrency {
5884 return Some(format!(
5885 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
5886 registered.module_id, registered_concurrency, candidate_concurrency
5887 ));
5888 }
5889
5890 None
5893}
5894
5895fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
5896 manifest.provides.iter().any(is_routable_role)
5897}
5898
5899fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
5905 manifest
5906 .provides
5907 .iter()
5908 .find_map(|provider| match provider {
5909 ProviderRole::ToolProvider { concurrency, .. }
5910 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
5911 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
5912 })
5913 .unwrap_or(Concurrency::ModuleManaged)
5914}
5915
5916fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
5923 let has_management_surface = manifest
5924 .provides
5925 .iter()
5926 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
5927 if !has_management_surface {
5928 return false;
5929 }
5930 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
5931 return false;
5932 };
5933 let Some(provides) = raw
5934 .get("manifest")
5935 .and_then(|manifest| manifest.get("provides"))
5936 .and_then(serde_json::Value::as_array)
5937 else {
5938 return false;
5939 };
5940 provides.iter().any(|role| {
5945 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
5946 && role.get("concurrency").is_none()
5947 })
5948}
5949
5950fn negotiate_version(peer_version: u8) -> Result<u8, String> {
5951 if peer_version != PROTOCOL_VERSION {
5952 return Err(format!(
5953 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
5954 ));
5955 }
5956 Ok(PROTOCOL_VERSION)
5957}
5958
5959fn pong(frame: &Frame) -> Result<Frame, RouterError> {
5960 Frame::build_with_version(
5961 response_version(frame),
5962 FrameType::Pong,
5963 frame.header.flags,
5964 0,
5965 0,
5966 frame.header.corr,
5967 Vec::new(),
5968 )
5969 .map_err(RouterError::FrameBuild)
5970}
5971
5972fn control_error_frame(
5973 frame: &Frame,
5974 code: &'static str,
5975 message: impl Into<String>,
5976) -> Result<Frame, RouterError> {
5977 control_error_body_frame(
5978 frame,
5979 ErrorBody {
5980 code: code.to_string(),
5981 message: message.into(),
5982 detail: None,
5983 },
5984 )
5985}
5986
5987fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
5988 let body = serde_json::to_vec(&error).map_err(|err| {
5989 RouterError::backend(
5990 0,
5991 frame.header.corr,
5992 format!("failed to encode control ERROR: {err}"),
5993 )
5994 })?;
5995
5996 Frame::build_with_version(
5997 response_version(frame),
5998 FrameType::Error,
5999 control_flags(),
6000 0,
6001 0,
6002 frame.header.corr,
6003 body,
6004 )
6005 .map_err(RouterError::FrameBuild)
6006}
6007
6008fn control_response_body_frame<T: Serialize>(
6009 frame: &Frame,
6010 reply: &T,
6011 label: &'static str,
6012) -> Result<Frame, RouterError> {
6013 let body = serde_json::to_vec(reply).map_err(|err| {
6014 RouterError::backend(
6015 0,
6016 frame.header.corr,
6017 format!("failed to encode {label}: {err}"),
6018 )
6019 })?;
6020
6021 Frame::build_with_version(
6022 response_version(frame),
6023 FrameType::Response,
6024 control_flags(),
6025 0,
6026 0,
6027 frame.header.corr,
6028 body,
6029 )
6030 .map_err(RouterError::FrameBuild)
6031}
6032
6033fn forwarding_error_code(err: &ForwardingError) -> &'static str {
6063 match err {
6064 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
6065 ForwardingError::NoModuleConnection => "target_unavailable",
6066 ForwardingError::ModuleReloading { .. } => "module_reloading",
6067 ForwardingError::ClientRouteChannelExhausted { .. }
6068 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
6069 ForwardingError::StaleModuleEndpoint
6070 | ForwardingError::UnknownReservation { .. }
6071 | ForwardingError::ConnectionClosing { .. }
6072 | ForwardingError::ClientEgressClosed { .. }
6073 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
6074 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
6077 ForwardingError::RelayCorrelationExhausted
6078 | ForwardingError::RouteOpenBuild(_)
6079 | ForwardingError::Poisoned => "forwarding_error",
6080 }
6081}
6082
6083fn response_version(frame: &Frame) -> u8 {
6084 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
6085 frame.header.ver
6086 } else {
6087 PROTOCOL_VERSION
6088 }
6089}
6090
6091fn control_flags() -> Flags {
6092 Flags::new(false, Priority::Passive, false)
6093}
6094
6095fn send_goodbye_target_best_effort(
6100 counters: &DaemonCounters,
6101 target: &GoodbyeTarget,
6102 context: &'static str,
6103) {
6104 let Ok(frame) = Frame::build_with_version(
6105 target.negotiated_ver,
6106 FrameType::Goodbye,
6107 control_flags(),
6108 target.channel,
6109 target.epoch,
6110 0,
6111 Vec::new(),
6112 ) else {
6113 return;
6114 };
6115 crate::forwarding::send_module_route_goodbye(
6116 counters,
6117 &target.sink,
6118 frame,
6119 target.module_id.as_deref(),
6120 context,
6121 );
6122}
6123
6124pub(crate) fn send_route_control_pushes(
6125 forwarding: &ForwardingTable,
6126 routes: Vec<EndpointRoute>,
6127 push: ClientControlPush,
6128) {
6129 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
6130 for route in routes {
6131 let target = route.goodbye_target;
6132 if let Some((existing, channels)) = targets
6133 .iter_mut()
6134 .find(|(existing, _)| existing.connection_id == target.connection_id)
6135 {
6136 debug_assert_eq!(
6137 existing.negotiated_ver, target.negotiated_ver,
6138 "one connection cannot negotiate multiple frame versions"
6139 );
6140 if !channels.contains(&target.channel) {
6141 channels.push(target.channel);
6142 }
6143 continue;
6144 }
6145 let channel = target.channel;
6146 targets.push((target, vec![channel]));
6147 }
6148 for (target, mut channels) in targets {
6149 channels.sort_unstable();
6150 let mut push = push.clone();
6151 match &mut push {
6152 ClientControlPush::RouteClosing {
6153 channels: covered, ..
6154 }
6155 | ClientControlPush::RouteClosed {
6156 channels: covered, ..
6157 } => *covered = channels,
6158 }
6159 let body = match serde_json::to_vec(&push) {
6160 Ok(body) => body,
6161 Err(err) => {
6162 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
6163 continue;
6164 }
6165 };
6166 let frame = match Frame::build_with_version(
6167 target.negotiated_ver,
6168 FrameType::Push,
6169 control_flags(),
6170 0,
6171 0,
6172 0,
6173 body.clone(),
6174 ) {
6175 Ok(frame) => frame,
6176 Err(err) => {
6177 warn!(
6178 route_channel = target.channel,
6179 error = %err,
6180 "failed to build route lifecycle control PUSH frame"
6181 );
6182 continue;
6183 }
6184 };
6185 if let Err(err) = target.sink.try_send(frame) {
6186 if target.close_on_delivery_failure() {
6187 warn!(
6188 target_connection_id = target.connection_id.get(),
6189 route_channel = target.channel,
6190 error = %err,
6191 "route lifecycle control PUSH was not delivered to client; closing target connection"
6192 );
6193 let _ = forwarding.escalate_client_delivery_failure(
6194 target.connection_id,
6195 target.channel,
6196 target.epoch,
6197 CloseReason::new(
6198 "route_lifecycle_push_delivery_failed",
6199 format!(
6200 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
6201 target.channel
6202 ),
6203 ),
6204 crate::forwarding::UndeliveredFrame {
6205 module_id: target.module_id.as_deref(),
6206 sink: &target.sink,
6207 },
6208 );
6209 }
6210 }
6211 }
6212}
6213
6214#[cfg(test)]
6215mod tests {
6216 #[cfg(unix)]
6217 #[tokio::test]
6218 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
6219 let dir = subc_test_support::TestTempDir::new("rescan-live-health");
6220 let path = dir.join("subc.jsonc");
6221 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
6222 "program":"/bin/sleep","args":["60"],"protocol":"none",
6223 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
6224 }}}).to_string()).unwrap();
6225 let mut configured = crate::daemon_config::load(&path)
6226 .unwrap()
6227 .unwrap()
6228 .modules
6229 .pop()
6230 .unwrap();
6231 let registry = std::sync::Arc::new(crate::Registry::default());
6232 let handle = crate::SupervisorHandle::new();
6233 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
6234 .with_handle(handle.clone());
6235 let module = supervisor
6236 .supervise_configured_with_health(
6237 configured.module_spec(),
6238 true,
6239 configured.health.clone(),
6240 None,
6241 configured.restart,
6242 )
6243 .unwrap();
6244 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
6245 let before = module.status().unwrap().pid;
6246 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
6247 configured.health.cadence = std::time::Duration::from_secs(3600);
6248 let health_only = handler
6249 .reconcile_supervised_modules(&supervisor, vec![configured.clone()], false)
6250 .await
6251 .unwrap();
6252 assert!(
6253 health_only.changed_pending_reload.is_empty(),
6254 "health policy is already applied live"
6255 );
6256 assert_eq!(module.status().unwrap().pid, before);
6257 assert_eq!(
6258 module.configuration().unwrap().1.http,
6259 configured.health.http
6260 );
6261 configured.args = vec!["61".into()];
6262 let launch = handler
6263 .reconcile_supervised_modules(&supervisor, vec![configured], false)
6264 .await
6265 .unwrap();
6266 assert_eq!(launch.changed_pending_reload, ["stock"]);
6267 assert_eq!(
6268 module.status().unwrap().pid,
6269 before,
6270 "a launch edit is stored until reload"
6271 );
6272 module.drain().await.unwrap();
6273 }
6274 use std::{
6275 collections::BTreeMap,
6276 fmt,
6277 path::PathBuf,
6278 sync::{Arc, Mutex},
6279 time::Duration,
6280 };
6281 use subc_test_support::TestTempDir;
6282
6283 use serde_json::{json, Value};
6284 use subc_protocol::{
6285 manifest::{
6286 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
6287 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
6288 },
6289 session::HealthStatus,
6290 FrameType,
6291 };
6292
6293 use super::*;
6294 use crate::{
6295 forwarding::{DataRoute, DataRouteState},
6296 registry::ChannelState,
6297 router::FrameSink,
6298 stderr_tail::DEFAULT_MAX_LINE_BYTES,
6299 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
6300 RouteCtx, Router,
6301 };
6302 use tokio::{
6303 sync::mpsc,
6304 time::{sleep, Instant},
6305 };
6306 use tracing::{
6307 field::{Field, Visit},
6308 Event, Subscriber,
6309 };
6310 use tracing_subscriber::{layer::Context, prelude::*, Layer};
6311
6312 fn fake_aft_stub_path() -> PathBuf {
6331 let mut path = std::env::current_exe().expect("current_exe available in tests");
6332 path.pop(); path.pop(); path.push(if cfg!(windows) {
6335 "fake-aft-stub.exe"
6336 } else {
6337 "fake-aft-stub"
6338 });
6339 assert!(
6340 path.exists(),
6341 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
6342 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
6343 path.display()
6344 );
6345 path
6346 }
6347
6348 fn client_retries(code: &str) -> bool {
6353 subc_protocol::error_codes::is_retryable_route_open(code)
6354 }
6355
6356 #[test]
6367 fn retryability_of_forwarding_codes_matches_the_failure() {
6368 let transient = [
6371 ForwardingError::NoModuleConnection,
6372 ForwardingError::ModuleReloading {
6373 module_id: "m".into(),
6374 },
6375 ForwardingError::StaleModuleEndpoint,
6376 ForwardingError::UnknownReservation {
6377 client_channel: 1,
6378 module_channel: 1,
6379 },
6380 ForwardingError::ConnectionClosing {
6381 connection_id: ConnectionId::new(1),
6382 },
6383 ForwardingError::ClientEgressClosed {
6384 connection_id: ConnectionId::new(1),
6385 },
6386 ForwardingError::ModuleEgressUnavailable {
6387 connection_id: ConnectionId::new(1),
6388 },
6389 ];
6390 for err in transient {
6391 let code = forwarding_error_code(&err);
6392 assert!(
6393 client_retries(code),
6394 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
6395 );
6396 }
6397
6398 let permanent = [
6403 ForwardingError::ConnectionRoleConflict {
6404 connection_id: ConnectionId::new(1),
6405 },
6406 ForwardingError::ClientRouteChannelExhausted {
6407 connection_id: ConnectionId::new(1),
6408 },
6409 ForwardingError::ModuleRouteChannelExhausted {
6410 endpoint: ModuleEndpointId {
6411 connection_id: ConnectionId::new(1),
6412 generation: 1,
6413 },
6414 },
6415 ForwardingError::RelayCorrelationExhausted,
6416 ForwardingError::RouteOpenBuild("x".into()),
6417 ForwardingError::Poisoned,
6418 ];
6419 for err in permanent {
6420 let code = forwarding_error_code(&err);
6421 assert!(
6422 !client_retries(code),
6423 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
6424 );
6425 }
6426 }
6427
6428 #[tokio::test]
6456 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
6457 let handler = ControlHandler::default();
6458 let frame =
6459 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
6460
6461 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
6465 assert_eq!(
6466 stamped,
6467 Principal::Direct,
6468 "a caller that proved nothing must not be stamped as a supervised module"
6469 );
6470
6471 let forged = handler
6476 .route_open_principal(
6477 &frame,
6478 Some(ConsumerIdentity {
6479 module_id: "aft".to_string(),
6480 launch_nonce: "not-a-real-nonce".to_string(),
6481 }),
6482 )
6483 .unwrap();
6484 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
6485 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
6486 }
6487
6488 #[test]
6505 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
6506 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
6507 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
6508 let ClientControlRequest::RouteOpen {
6509 consumer_identity, ..
6510 } = parsed
6511 else {
6512 panic!("route.open body must parse as RouteOpen");
6513 };
6514 assert_eq!(
6515 consumer_identity,
6516 Some(ConsumerIdentity {
6517 module_id: "aft".to_string(),
6518 launch_nonce: "n".to_string(),
6519 }),
6520 "the wire field name must reach the value route_open_principal reads"
6521 );
6522 }
6523
6524 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
6525 ModuleManifest::builder(module_id, "0.1.0")
6526 .protocol_ver(protocol_ver)
6527 .provides(vec![ProviderRole::ToolProvider {
6528 tools: vec![Tool {
6529 name: "read".to_string(),
6530 description: None,
6531 execution_mode: ExecutionMode::Pure,
6532 schema: json!({"type": "object"}),
6533 }],
6534 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
6535 concurrency: Concurrency::ModuleManaged,
6536 emits_push: true,
6537 sub_supervises: true,
6538 }])
6539 .build()
6540 }
6541
6542 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
6543 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
6544 }
6545
6546 fn hello_frame_with_control_ops(
6547 module_id: &str,
6548 protocol_ver: u8,
6549 corr: u64,
6550 control_ops: Option<Vec<String>>,
6551 ) -> Frame {
6552 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
6553 }
6554
6555 fn hello_frame_with_nonce(
6556 module_id: &str,
6557 protocol_ver: u8,
6558 corr: u64,
6559 launch_nonce: Option<&str>,
6560 ) -> Frame {
6561 hello_frame_full(
6562 module_id,
6563 protocol_ver,
6564 corr,
6565 None,
6566 launch_nonce.map(ToOwned::to_owned),
6567 )
6568 }
6569
6570 fn hello_frame_full(
6571 module_id: &str,
6572 protocol_ver: u8,
6573 corr: u64,
6574 control_ops: Option<Vec<String>>,
6575 launch_nonce: Option<String>,
6576 ) -> Frame {
6577 let body = serde_json::to_vec(&ModuleHelloBody {
6578 manifest: manifest(module_id, protocol_ver),
6579 protocol_ver,
6580 control_ops,
6581 launch_nonce,
6582 })
6583 .unwrap();
6584 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
6585 }
6586
6587 fn non_routable_hello_frame_with_control_ops(
6588 module_id: &str,
6589 corr: u64,
6590 control_ops: Option<Vec<String>>,
6591 ) -> Frame {
6592 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
6593 manifest.provides.clear();
6594 let body = serde_json::to_vec(&ModuleHelloBody {
6595 manifest,
6596 protocol_ver: PROTOCOL_VERSION,
6597 control_ops,
6598 launch_nonce: None,
6599 })
6600 .unwrap();
6601 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
6602 }
6603
6604 fn capability_grammar_hello_frame(
6605 capabilities: Value,
6606 runtime_computed: Option<Value>,
6607 corr: u64,
6608 ) -> Frame {
6609 let mut body = serde_json::to_value(ModuleHelloBody {
6610 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
6611 protocol_ver: PROTOCOL_VERSION,
6612 control_ops: None,
6613 launch_nonce: None,
6614 })
6615 .expect("HELLO body serializes");
6616 body["manifest"]["capabilities"] = capabilities;
6617 if let Some(runtime_computed) = runtime_computed {
6618 body["runtime_computed"] = runtime_computed;
6619 }
6620 Frame::build(
6621 FrameType::Hello,
6622 control_flags(),
6623 0,
6624 0,
6625 corr,
6626 serde_json::to_vec(&body).expect("HELLO body reserializes"),
6627 )
6628 .expect("HELLO frame builds")
6629 }
6630
6631 fn channel_request(channel: u16, corr: u64) -> Frame {
6632 Frame::build(
6633 FrameType::Request,
6634 Flags::new(true, Priority::Interactive, false),
6635 channel,
6636 0,
6637 corr,
6638 b"opaque".to_vec(),
6639 )
6640 .unwrap()
6641 }
6642
6643 fn route_ctx(
6644 connection_id: ConnectionId,
6645 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
6646 let (tx, rx) = mpsc::channel(8);
6647 (
6648 RouteCtx {
6649 connection_id,
6650 egress: FrameSink::new(tx),
6651 },
6652 rx,
6653 )
6654 }
6655
6656 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
6657 serde_json::from_slice(&frame.body).unwrap()
6658 }
6659
6660 async fn hello_via_sink(
6665 handler: &ControlHandler,
6666 ctx: &RouteCtx,
6667 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
6668 hello: Frame,
6669 ) -> Frame {
6670 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
6671 assert!(
6672 replies.is_empty(),
6673 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
6674 );
6675 let ack = rx
6676 .try_recv()
6677 .expect("HELLO_ACK is queued on the module sink")
6678 .frame;
6679 assert_eq!(ack.header.ty, FrameType::HelloAck);
6680 ack
6681 }
6682
6683 fn parse_error(frame: &Frame) -> Value {
6684 serde_json::from_slice(&frame.body).unwrap()
6685 }
6686
6687 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
6688 serde_json::from_slice(&frame.body).unwrap()
6689 }
6690
6691 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
6692 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
6693 route_channel,
6694 route_epoch: 0,
6695 kind,
6696 })
6697 .unwrap();
6698 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6699 }
6700
6701 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
6702 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
6703 module_id: module_id.to_string(),
6704 })
6705 .unwrap();
6706 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6707 }
6708
6709 fn route_open_frame(corr: u64, module_id: &str, project_root: TestTempDir) -> Frame {
6710 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
6711 }
6712
6713 fn route_open_frame_with_consumer_capabilities(
6714 corr: u64,
6715 module_id: &str,
6716 project_root: TestTempDir,
6717 consumer_capabilities: Option<Vec<String>>,
6718 ) -> Frame {
6719 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6720 target: RouteTarget::ToolProvider {
6721 module_id: module_id.to_string(),
6722 },
6723 identity: BindIdentity::new(
6724 project_root.path().to_path_buf(),
6725 "unit".to_string(),
6726 "session".to_string(),
6727 ),
6728 consumer_identity: None,
6729 consumer_capabilities,
6730 role_versions: None,
6731 admission_facts: None,
6732 scope: None,
6733 })
6734 .unwrap();
6735 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6736 }
6737
6738 fn route_open_frame_with_role_versions(
6739 corr: u64,
6740 module_id: &str,
6741 project_root: TestTempDir,
6742 role_versions: Option<BTreeMap<String, String>>,
6743 ) -> Frame {
6744 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6745 target: RouteTarget::ToolProvider {
6746 module_id: module_id.to_string(),
6747 },
6748 identity: BindIdentity::new(
6749 project_root.path().to_path_buf(),
6750 "unit".to_string(),
6751 format!("session-{corr}"),
6752 ),
6753 consumer_identity: None,
6754 consumer_capabilities: None,
6755 role_versions,
6756 admission_facts: None,
6757 scope: None,
6758 })
6759 .unwrap();
6760 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6761 }
6762
6763 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
6764 entries
6765 .iter()
6766 .map(|(role, version)| (role.to_string(), version.to_string()))
6767 .collect()
6768 }
6769
6770 fn route_open_frame_with_admission_facts(
6771 corr: u64,
6772 module_id: &str,
6773 project_root: TestTempDir,
6774 consumer_identity: Option<subc_control::ConsumerIdentity>,
6775 facts: Option<Value>,
6776 ) -> Frame {
6777 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6778 target: RouteTarget::ToolProvider {
6779 module_id: module_id.to_string(),
6780 },
6781 identity: BindIdentity::new(
6782 project_root.path().to_path_buf(),
6783 "unit".to_string(),
6784 format!("session-{corr}"),
6785 ),
6786 consumer_identity,
6787 consumer_capabilities: None,
6788 role_versions: None,
6789 admission_facts: facts,
6790 scope: None,
6791 })
6792 .unwrap();
6793 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6794 }
6795
6796 #[derive(Clone, Default)]
6797 struct EventCapture {
6798 events: Arc<Mutex<Vec<CapturedEvent>>>,
6799 }
6800
6801 #[derive(Clone, Debug)]
6802 struct CapturedEvent {
6803 target: String,
6804 level: tracing::Level,
6805 fields: BTreeMap<String, String>,
6806 }
6807
6808 impl EventCapture {
6809 fn events(&self) -> Vec<CapturedEvent> {
6810 self.events.lock().unwrap().clone()
6811 }
6812 }
6813
6814 impl<S> Layer<S> for EventCapture
6815 where
6816 S: Subscriber,
6817 {
6818 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
6819 let mut visitor = EventFieldVisitor::default();
6820 event.record(&mut visitor);
6821 self.events.lock().unwrap().push(CapturedEvent {
6822 target: event.metadata().target().to_string(),
6823 level: *event.metadata().level(),
6824 fields: visitor.fields,
6825 });
6826 }
6827 }
6828
6829 #[derive(Default)]
6830 struct EventFieldVisitor {
6831 fields: BTreeMap<String, String>,
6832 }
6833
6834 impl Visit for EventFieldVisitor {
6835 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
6836 self.fields
6837 .insert(field.name().to_string(), format!("{value:?}"));
6838 }
6839 }
6840
6841 fn health_response(corr: u64, status: HealthStatus) -> Frame {
6842 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
6843 status,
6844 detail: Some("warming".to_string()),
6845 metrics: Some(json!({"queue_depth": 3})),
6846 })
6847 .unwrap();
6848 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
6849 }
6850
6851 fn route_bind_ack(corr: u64) -> Frame {
6852 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
6853 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
6854 }
6855
6856 fn unique_project_root(label: &str) -> TestTempDir {
6857 TestTempDir::new(label)
6858 }
6859
6860 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
6861 match parse_route_poll(frame) {
6862 ClientControlResponse::RoutePoll {
6863 status: None,
6864 live: Some(live),
6865 ..
6866 } => assert_eq!(live, expected_live),
6867 other => panic!("unexpected route.poll response: {other:?}"),
6868 }
6869 }
6870
6871 fn bind_liveness_route(
6872 registry: &Registry,
6873 forwarding: &ForwardingTable,
6874 module_id: &str,
6875 ) -> (RouteCtx, u16, u32) {
6876 let module_connection = ConnectionId::new(101);
6877 let client_connection = ConnectionId::new(202);
6878 let registration = registry
6879 .register_with_control_ops(
6880 manifest(module_id, PROTOCOL_VERSION),
6881 PROTOCOL_VERSION,
6882 module_connection,
6883 module_baseline_control_ops(),
6884 )
6885 .unwrap();
6886 let (module_tx, _module_rx) = mpsc::channel(8);
6887 let endpoint = forwarding
6888 .register_module_connection(
6889 module_connection,
6890 module_id.to_string(),
6891 PROTOCOL_VERSION,
6892 manifest_concurrency(®istration.manifest),
6893 FrameSink::new(module_tx),
6894 )
6895 .unwrap();
6896 let (client_ctx, _client_rx) = route_ctx(client_connection);
6897 let pending = forwarding
6898 .begin_route_bind_relay_for_test(
6899 client_connection,
6900 client_ctx.egress.clone(),
6901 1,
6902 module_id,
6903 )
6904 .unwrap();
6905 assert_eq!(pending.endpoint, endpoint);
6906 let route_channel = pending.client_channel;
6907 let route_epoch = pending.client_epoch;
6908 forwarding
6909 .complete_pending_relay(
6910 module_connection,
6911 pending.corr,
6912 RouteBindRelayOutcome::Accepted,
6913 )
6914 .unwrap();
6915 (client_ctx, route_channel, route_epoch)
6916 }
6917
6918 struct FakeProcessLiveness {
6919 live: Option<bool>,
6920 }
6921
6922 impl ModuleProcessLiveness for FakeProcessLiveness {
6923 fn process_live(&self, _module_id: &str) -> Option<bool> {
6924 self.live
6925 }
6926 }
6927
6928 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6929 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
6930 {
6931 let registry = Arc::new(Registry::default());
6932 let supervisor_handle = SupervisorHandle::new();
6933 let supervisor = Supervisor::new_for_test(
6934 Arc::clone(®istry),
6935 RestartPolicy::new(1, Duration::from_millis(10)),
6936 )
6937 .with_handle(supervisor_handle.clone());
6938 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
6939 let module = supervisor
6940 .spawn(ModuleSpec {
6941 module_id: "stderr-tail-wire".to_string(),
6942 program: fake_aft_stub_path(),
6943 args: Vec::new(),
6944 env: vec![
6945 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
6946 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
6947 ],
6948 reserved: false,
6949 reserved_prefixes: Vec::new(),
6950 protocol: ModuleProtocol::Subc,
6951 overlap: Default::default(),
6952 })
6953 .unwrap();
6954
6955 let deadline = Instant::now() + Duration::from_secs(5);
6956 loop {
6957 let tail = module.stderr_tail(None, None);
6958 if tail
6959 .entries
6960 .iter()
6961 .any(|entry| matches!(entry, TailEntry::ProcessStart))
6962 && tail.entries.iter().any(|entry| {
6963 matches!(
6964 entry,
6965 TailEntry::Line {
6966 truncated: true,
6967 ..
6968 }
6969 )
6970 })
6971 {
6972 break;
6973 }
6974 assert!(
6975 Instant::now() < deadline,
6976 "module did not produce a truncated line and restart boundary: {tail:?}"
6977 );
6978 sleep(Duration::from_millis(10)).await;
6979 }
6980
6981 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
6982 let request = ClientControlRequest::SupervisorStderrTail {
6983 module_id: "stderr-tail-wire".to_string(),
6984 max_lines: None,
6985 max_bytes: None,
6986 };
6987 let frame = Frame::build(
6988 FrameType::Request,
6989 control_flags(),
6990 0,
6991 0,
6992 1,
6993 serde_json::to_vec(&request).unwrap(),
6994 )
6995 .unwrap();
6996 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6997 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
6998 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
6999 serde_json::from_slice(&responses[0].body).unwrap()
7000 else {
7001 panic!("expected supervisor.stderr_tail response");
7002 };
7003
7004 assert!(
7005 tail.entries
7006 .iter()
7007 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
7008 "the control response lost the restart boundary"
7009 );
7010 let Some(StderrTailEntry::Line {
7011 text,
7012 truncated,
7013 at_ms,
7014 }) = tail.entries.iter().find(|entry| {
7015 matches!(
7016 entry,
7017 StderrTailEntry::Line {
7018 truncated: true,
7019 ..
7020 }
7021 )
7022 })
7023 else {
7024 panic!("the control response lost the truncated line");
7025 };
7026 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
7027 assert!(*truncated);
7028 assert!(
7029 at_ms.is_some(),
7030 "the control response lost the line's capture time"
7031 );
7032 }
7033
7034 #[tokio::test(flavor = "current_thread")]
7039 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
7040 let dir = TestTempDir::new("terminals-off-worker");
7041 let journal_path = dir.join("terminals.jsonl");
7042 let registry = Arc::new(Registry::default());
7043 let supervisor_handle = SupervisorHandle::new();
7044 let supervisor =
7045 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
7046 .with_handle(supervisor_handle.clone())
7047 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
7048 let module = supervisor
7049 .spawn(ModuleSpec {
7050 module_id: "terminal-off-worker".to_string(),
7051 program: fake_aft_stub_path(),
7052 args: Vec::new(),
7053 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
7054 reserved: false,
7055 reserved_prefixes: Vec::new(),
7056 protocol: ModuleProtocol::Subc,
7057 overlap: Default::default(),
7058 })
7059 .unwrap();
7060 let deadline = Instant::now() + Duration::from_secs(5);
7061 while module.terminal_history().entries.len() != 2 {
7062 assert!(Instant::now() < deadline, "module did not record two exits");
7063 sleep(Duration::from_millis(10)).await;
7064 }
7065
7066 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
7067 let handler =
7068 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
7069 let frame = Frame::build(
7070 FrameType::Request,
7071 control_flags(),
7072 0,
7073 0,
7074 1,
7075 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
7076 module_id: "terminal-off-worker".to_string(),
7077 })
7078 .unwrap(),
7079 )
7080 .unwrap();
7081 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7082 let spawned_at = std::time::Instant::now();
7083 let read = tokio::spawn({
7084 let handler = Arc::clone(&handler);
7085 async move { handler.handle_control_frame(&ctx, frame).await }
7086 });
7087 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
7090 .await
7091 .unwrap()
7092 .expect("the history read reached its pause");
7093 let elapsed = spawned_at.elapsed();
7094 assert!(
7095 elapsed < Duration::from_secs(2) && !read.is_finished(),
7096 "this task could not run while the history read was paused \
7097 (resumed after {elapsed:?}, read finished: {})",
7098 read.is_finished()
7099 );
7100
7101 drop(release);
7102 let responses = read.await.unwrap().unwrap();
7103 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
7104 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
7105 panic!("expected supervisor.terminals response");
7106 };
7107 assert_eq!(terminals.entries.len(), 2);
7108 assert_eq!(terminals.journal_skipped_lines, 0);
7109 assert_eq!(terminals.journal_read_errors, 0);
7110 }
7111
7112 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7113 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
7114 let registry = Arc::new(Registry::default());
7115 let supervisor_handle = SupervisorHandle::new();
7116 let supervisor =
7117 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
7118 .with_handle(supervisor_handle.clone());
7119 let module = supervisor
7120 .spawn(ModuleSpec {
7121 module_id: "terminal-golden".to_string(),
7122 program: fake_aft_stub_path(),
7123 args: Vec::new(),
7124 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
7125 reserved: false,
7126 reserved_prefixes: Vec::new(),
7127 protocol: ModuleProtocol::Subc,
7128 overlap: Default::default(),
7129 })
7130 .unwrap();
7131
7132 let deadline = Instant::now() + Duration::from_secs(5);
7133 while module.terminal_history().entries.len() != 2 {
7134 assert!(
7135 Instant::now() < deadline,
7136 "module did not retain two terminal exits: {:?}",
7137 module.terminal_history()
7138 );
7139 sleep(Duration::from_millis(10)).await;
7140 }
7141
7142 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
7143 let request = ClientControlRequest::SupervisorTerminals {
7144 module_id: "terminal-golden".to_string(),
7145 };
7146 let frame = Frame::build(
7147 FrameType::Request,
7148 control_flags(),
7149 0,
7150 0,
7151 1,
7152 serde_json::to_vec(&request).unwrap(),
7153 )
7154 .unwrap();
7155 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7156 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
7157 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
7158 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
7159 panic!("expected supervisor.terminals response");
7160 };
7161 assert_eq!(terminals.entries.len(), 2);
7162 assert_eq!(terminals.dropped, 0);
7163
7164 let mut rendered = serde_json::to_value(response).unwrap();
7165 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
7168 for (index, entry) in rendered["entries"]
7169 .as_array_mut()
7170 .expect("terminal response entries array")
7171 .iter_mut()
7172 .enumerate()
7173 {
7174 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
7175 }
7176
7177 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
7178 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
7179 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
7180 if std::env::var_os("UPDATE_GOLDEN").is_some() {
7181 std::fs::write(&golden_path, &serialized).unwrap();
7182 }
7183 let expected: Value =
7184 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
7185 assert_eq!(rendered, expected);
7186 }
7187
7188 #[test]
7189 fn hello_registers_manifest_and_returns_ack() {
7190 let registry = Arc::new(Registry::default());
7191 let handler = ControlHandler::new(Arc::clone(®istry));
7192 let conn = ConnectionId::new(1);
7193
7194 let responses = handler
7195 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
7196 .unwrap();
7197
7198 assert_eq!(responses.len(), 1);
7199 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
7200 assert_eq!(responses[0].header.channel, 0);
7201 assert_eq!(responses[0].header.corr, 7);
7202 let ack = parse_ack(&responses[0]);
7203 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
7204 assert!(ack
7205 .subc_capabilities
7206 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
7207 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
7208 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
7209 assert!(ack
7210 .subc_ops
7211 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
7212 assert!(ack
7213 .subc_ops
7214 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
7215
7216 let registration = registry.get_module("aft").unwrap().unwrap();
7217 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
7218 assert_eq!(registration.state, ChannelState::Active);
7219 assert_eq!(registration.connection_id, conn);
7220 assert_eq!(registration.control_ops, module_baseline_control_ops());
7221 }
7222
7223 #[test]
7224 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
7225 let invalid_identifiers = [
7226 ("case_change", "credentials-Provider/v1"),
7227 ("leading_zero", "credentials-provider/v01"),
7228 ("trailing_hyphen", "credentials-provider-/v1"),
7229 ("consecutive_hyphens", "credentials--provider/v1"),
7230 ("uppercase", "Credentials-provider/v1"),
7231 ("missing_v", "credentials-provider/1"),
7232 ("whitespace", "credentials provider/v1"),
7233 ("zero_version", "credentials-provider/v0"),
7234 ("out_of_range_version", "credentials-provider/v4294967296"),
7235 (
7236 "overlength_name",
7237 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
7238 ),
7239 ];
7240 let mut cases = invalid_identifiers
7241 .into_iter()
7242 .map(|(name, identifier)| {
7243 (
7244 format!("identifier_{name}"),
7245 "capabilities.provides[0]".to_string(),
7246 identifier.to_string(),
7247 json!({ "provides": [identifier] }),
7248 None,
7249 )
7250 })
7251 .collect::<Vec<_>>();
7252 cases.extend([
7253 (
7254 "unknown_need".to_string(),
7255 "capabilities.requires[0].need".to_string(),
7256 "deferred".to_string(),
7257 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
7258 None,
7259 ),
7260 (
7261 "duplicate_provides".to_string(),
7262 "capabilities.provides[1]".to_string(),
7263 "credentials-provider/v1".to_string(),
7264 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
7265 None,
7266 ),
7267 (
7268 "duplicate_must_never_reach".to_string(),
7269 "capabilities.must_never_reach[1]".to_string(),
7270 "credentials-provider/v1".to_string(),
7271 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
7272 None,
7273 ),
7274 (
7275 "duplicate_requires_same_need".to_string(),
7276 "capabilities.requires[1]".to_string(),
7277 "credentials-provider/v1".to_string(),
7278 json!({ "requires": [
7279 { "capability": "credentials-provider/v1", "need": "required" },
7280 { "capability": "credentials-provider/v1", "need": "required" }
7281 ] }),
7282 None,
7283 ),
7284 (
7285 "duplicate_requires_conflicting_need".to_string(),
7286 "capabilities.requires[1]".to_string(),
7287 "credentials-provider/v1".to_string(),
7288 json!({ "requires": [
7289 { "capability": "credentials-provider/v1", "need": "required" },
7290 { "capability": "credentials-provider/v1", "need": "optional" }
7291 ] }),
7292 None,
7293 ),
7294 (
7295 "capabilities_root_pointer".to_string(),
7296 "runtime_computed[0]".to_string(),
7297 "/capabilities".to_string(),
7298 json!({}),
7299 Some(json!(["/capabilities"])),
7300 ),
7301 (
7302 "capabilities_descendant_pointer".to_string(),
7303 "runtime_computed[0]".to_string(),
7304 "/capabilities/provides".to_string(),
7305 json!({}),
7306 Some(json!(["/capabilities/provides"])),
7307 ),
7308 (
7309 "malformed_pointer_without_leading_slash".to_string(),
7310 "runtime_computed[0]".to_string(),
7311 "capabilities".to_string(),
7312 json!({}),
7313 Some(json!(["capabilities"])),
7314 ),
7315 (
7316 "malformed_pointer_escape".to_string(),
7317 "runtime_computed[0]".to_string(),
7318 "/roles/~2/tools".to_string(),
7319 json!({}),
7320 Some(json!(["/roles/~2/tools"])),
7321 ),
7322 (
7323 "unknown_capabilities_field".to_string(),
7324 "capabilities.future".to_string(),
7325 "<array>".to_string(),
7326 json!({ "future": [] }),
7327 None,
7328 ),
7329 ]);
7330
7331 for (index, (name, field, value, capabilities, runtime_computed)) in
7332 cases.into_iter().enumerate()
7333 {
7334 let registry = Arc::new(Registry::default());
7335 let handler = ControlHandler::new(Arc::clone(®istry));
7336 let response = handler
7337 .handle_control(
7338 ConnectionId::new((index + 1) as u64),
7339 capability_grammar_hello_frame(
7340 capabilities,
7341 runtime_computed,
7342 index as u64 + 1,
7343 ),
7344 )
7345 .expect("invalid HELLO returns a refusal");
7346
7347 assert_eq!(response.len(), 1, "{name} must emit one refusal");
7348 let error = parse_error(&response[0]);
7349 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
7350 let message = error["message"]
7351 .as_str()
7352 .expect("error message is a string");
7353 assert!(
7354 message.contains(&field),
7355 "{name}: field missing from {message}"
7356 );
7357 assert!(
7358 message.contains(&value),
7359 "{name}: value missing from {message}"
7360 );
7361 assert_eq!(
7362 registry
7363 .active_registration_count()
7364 .expect("registry reads"),
7365 0,
7366 "{name}: refused HELLO must not create a catalog entry"
7367 );
7368 }
7369 }
7370
7371 #[test]
7372 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
7373 let registry = Arc::new(Registry::default());
7374 let handler = ControlHandler::new(Arc::clone(®istry));
7375 let capabilities = json!({
7376 "provides": ["credentials-provider/v1"],
7377 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
7378 "must_never_reach": ["federation-transport/v1"]
7379 });
7380 let response = handler
7381 .handle_control(
7382 ConnectionId::new(99),
7383 capability_grammar_hello_frame(
7384 capabilities.clone(),
7385 Some(json!(["/roles/0/tools"])),
7386 99,
7387 ),
7388 )
7389 .expect("valid HELLO registers");
7390 assert_eq!(response[0].header.ty, FrameType::HelloAck);
7391
7392 let request = Frame::build(
7393 FrameType::Request,
7394 control_flags(),
7395 0,
7396 0,
7397 100,
7398 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7399 .expect("catalog request serializes"),
7400 )
7401 .expect("catalog request frame builds");
7402 let response = handler
7403 .handle_catalog_list(request, None)
7404 .expect("catalog list succeeds");
7405 let ClientControlResponse::CatalogList { modules, .. } =
7406 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
7407 else {
7408 panic!("catalog request must return catalog.list");
7409 };
7410 assert_eq!(modules.len(), 1);
7411 assert_eq!(
7412 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
7413 capabilities
7414 );
7415 }
7416
7417 #[test]
7418 fn catalog_list_mirrors_management_operation_description() {
7419 let registry = Arc::new(Registry::default());
7420 let handler = ControlHandler::new(Arc::clone(®istry));
7421 let description = "List managed records and return their identifiers and metadata.";
7422 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
7423 manifest.provides = vec![ProviderRole::ManagementSurface {
7424 operations: vec![ManagementOperation {
7425 name: "records.list".to_string(),
7426 kind: ManagementOperationKind::Query,
7427 description: Some(description.to_string()),
7428 }],
7429 config_schema: json!({"type": "object"}),
7430 observability: vec![ObservabilitySurface {
7431 name: "records.stats".to_string(),
7432 kind: ObservabilityKind::Snapshot,
7433 }],
7434 identity_scope: vec![IdentityScope::Project],
7435 concurrency: Concurrency::ModuleManaged,
7436 }];
7437 registry
7438 .register_with_control_ops(
7439 manifest,
7440 PROTOCOL_VERSION,
7441 ConnectionId::new(99),
7442 Vec::new(),
7443 )
7444 .expect("described management manifest registers");
7445
7446 let request = Frame::build(
7447 FrameType::Request,
7448 control_flags(),
7449 0,
7450 0,
7451 100,
7452 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7453 .expect("catalog request serializes"),
7454 )
7455 .expect("catalog request frame builds");
7456 let response = handler
7457 .handle_catalog_list(request, None)
7458 .expect("catalog list succeeds");
7459 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
7460 assert_eq!(
7461 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
7462 "catalog.list must preserve the declared operation description verbatim"
7463 );
7464 }
7465
7466 #[test]
7467 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
7468 let registry = Arc::new(Registry::default());
7469 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
7470 [("vault".to_string(), true), ("squatter".to_string(), true)],
7471 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
7472 );
7473 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
7474 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
7475 provides: vec!["credentials-provider/v1".to_string()],
7476 requires: Vec::new(),
7477 must_never_reach: Vec::new(),
7478 });
7479 let frame = Frame::build(
7480 FrameType::Hello,
7481 control_flags(),
7482 0,
7483 0,
7484 77,
7485 serde_json::to_vec(&ModuleHelloBody {
7486 manifest: squatter,
7487 protocol_ver: PROTOCOL_VERSION,
7488 control_ops: None,
7489 launch_nonce: None,
7490 })
7491 .expect("HELLO serializes"),
7492 )
7493 .expect("HELLO frame builds");
7494 let response = handler
7495 .handle_control(ConnectionId::new(77), frame)
7496 .expect("reserved claim receives a typed refusal");
7497 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
7498 assert_eq!(
7499 registry
7500 .active_registration_count()
7501 .expect("registry reads"),
7502 0,
7503 "a reserved capability refusal must not leave a catalog entry"
7504 );
7505 }
7506
7507 #[test]
7508 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
7509 for settlement in ["timeout", "inconclusive", "drop"] {
7510 let breakers = RouteBindBreakers::default();
7511 let RouteBindAdmission::Admitted {
7512 guard: mut old,
7513 probe: false,
7514 } = breakers.admit("prov")
7515 else {
7516 panic!("ordinary relay admitted")
7517 };
7518 let RouteBindAdmission::Admitted {
7519 guard: mut opener, ..
7520 } = breakers.admit("prov")
7521 else {
7522 panic!("second relay admitted")
7523 };
7524 assert!(
7525 !opener
7526 .record_timeout(1, Duration::ZERO)
7527 .unwrap()
7528 .reopened_after_probe
7529 );
7530 let RouteBindAdmission::Admitted {
7531 guard: mut probe,
7532 probe: true,
7533 } = breakers.admit("prov")
7534 else {
7535 panic!("one cooldown probe admitted")
7536 };
7537 match settlement {
7538 "timeout" => assert!(
7539 !old.record_timeout(1, Duration::ZERO)
7540 .unwrap()
7541 .reopened_after_probe
7542 ),
7543 "inconclusive" => old.record_inconclusive(),
7544 "drop" => drop(old),
7545 _ => unreachable!(),
7546 }
7547 assert!(
7548 matches!(
7549 breakers.admit("prov"),
7550 RouteBindAdmission::Refused {
7551 probe_in_flight: true,
7552 ..
7553 }
7554 ),
7555 "{settlement} of a pre-open relay cannot release the real probe"
7556 );
7557 assert!(
7558 probe
7559 .record_timeout(1, Duration::ZERO)
7560 .unwrap()
7561 .reopened_after_probe
7562 );
7563 assert!(matches!(
7564 breakers.admit("prov"),
7565 RouteBindAdmission::Admitted { probe: true, .. }
7566 ));
7567 }
7568 let breakers = RouteBindBreakers::default();
7569 let admit = || match breakers.admit("prov") {
7570 RouteBindAdmission::Admitted { guard, .. } => guard,
7571 _ => panic!("relay admitted"),
7572 };
7573 admit().record_timeout(1, Duration::ZERO);
7574 let mut old_probe = admit();
7575 breakers.reset_for_new_module_connection("prov");
7576 admit().record_timeout(1, Duration::ZERO);
7577 let _new_probe = admit();
7578 old_probe.record_inconclusive();
7579 assert!(matches!(
7580 breakers.admit("prov"),
7581 RouteBindAdmission::Refused {
7582 probe_in_flight: true,
7583 ..
7584 }
7585 ));
7586 }
7587
7588 #[tokio::test]
7589 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
7590 for candidate in [false, true] {
7591 let registry = Arc::new(Registry::default());
7592 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
7593 [("vault".to_string(), true), ("squatter".to_string(), true)],
7594 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
7595 );
7596 let conn = ConnectionId::new(77);
7597 let (ctx, mut rx) = route_ctx(conn);
7598 let initial = capability_manifest("squatter", &[], &[]);
7599 if candidate {
7600 registry
7601 .register_candidate_with_control_ops(
7602 initial.clone(),
7603 PROTOCOL_VERSION,
7604 conn,
7605 module_baseline_control_ops(),
7606 )
7607 .unwrap();
7608 handler
7609 .forwarding
7610 .register_candidate_module_connection(
7611 conn,
7612 "squatter".to_string(),
7613 PROTOCOL_VERSION,
7614 manifest_concurrency(&initial),
7615 ctx.egress.clone(),
7616 )
7617 .unwrap();
7618 } else {
7619 hello_via_sink(
7620 &handler,
7621 &ctx,
7622 &mut rx,
7623 hello_frame_with_manifest(initial.clone(), 1),
7624 )
7625 .await;
7626 }
7627 let response = handler
7628 .handle_control_frame(
7629 &ctx,
7630 catalog_update_with_capabilities_frame(
7631 2,
7632 capability_manifest("squatter", &["credentials-provider/v1"], &[])
7633 .capabilities
7634 .unwrap(),
7635 ),
7636 )
7637 .await
7638 .unwrap();
7639 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
7640 assert_eq!(
7641 registry
7642 .get_module_by_connection(conn)
7643 .unwrap()
7644 .unwrap()
7645 .manifest,
7646 initial
7647 );
7648 }
7649 }
7650
7651 #[test]
7652 fn server_describe_surfaces_required_capability_verdict_fields() {
7653 let registry = Arc::new(Registry::default());
7654 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
7655 [
7656 ("consumer".to_string(), true),
7657 ("provider".to_string(), false),
7658 ],
7659 BTreeMap::new(),
7660 );
7661 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
7662 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
7663 provides: Vec::new(),
7664 requires: vec![subc_protocol::manifest::CapabilityRequirement {
7665 capability: "credentials-provider/v1".to_string(),
7666 need: subc_protocol::manifest::CapabilityNeed::Required,
7667 }],
7668 must_never_reach: Vec::new(),
7669 });
7670 let hello = Frame::build(
7671 FrameType::Hello,
7672 control_flags(),
7673 0,
7674 0,
7675 78,
7676 serde_json::to_vec(&ModuleHelloBody {
7677 manifest: consumer,
7678 protocol_ver: PROTOCOL_VERSION,
7679 control_ops: None,
7680 launch_nonce: None,
7681 })
7682 .expect("HELLO serializes"),
7683 )
7684 .expect("HELLO frame builds");
7685 handler
7686 .handle_control(ConnectionId::new(78), hello)
7687 .expect("consumer registers");
7688 let describe = Frame::build(
7689 FrameType::Request,
7690 control_flags(),
7691 0,
7692 0,
7693 79,
7694 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
7695 .expect("request serializes"),
7696 )
7697 .expect("describe frame builds");
7698 let response = handler
7699 .handle_server_describe(describe)
7700 .expect("server.describe succeeds");
7701 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
7702 let requirement = &rendered["capability_requirements"][0];
7703 assert_eq!(requirement["consumer"], "consumer");
7704 assert_eq!(requirement["verdict"], "never_provided");
7705 assert_eq!(requirement["episode_seq"], 1);
7706 assert_eq!(requirement["config_satisfiable"], false);
7707 assert_eq!(requirement["runtime_available"], false);
7708 assert!(requirement["detail"]
7709 .as_str()
7710 .expect("detail string")
7711 .contains("credentials-provider/v1"));
7712 }
7713
7714 #[test]
7715 fn catalog_list_omits_capabilities_for_legacy_manifest() {
7716 let registry = Arc::new(Registry::default());
7717 let handler = ControlHandler::new(Arc::clone(®istry));
7718 let hello = handler
7719 .handle_control(
7720 ConnectionId::new(101),
7721 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
7722 )
7723 .expect("legacy HELLO registers");
7724 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
7725
7726 let request = Frame::build(
7727 FrameType::Request,
7728 control_flags(),
7729 0,
7730 0,
7731 102,
7732 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7733 .expect("catalog request serializes"),
7734 )
7735 .expect("catalog request frame builds");
7736 let response = handler
7737 .handle_catalog_list(request, None)
7738 .expect("catalog list succeeds");
7739 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
7740 assert!(
7741 body["modules"][0].get("capabilities").is_none(),
7742 "legacy manifest must retain an absent capabilities field on catalog.list"
7743 );
7744 }
7745
7746 #[test]
7747 fn hello_ack_omits_storage_when_no_storage_config() {
7748 let registry = Arc::new(Registry::default());
7749 let handler = ControlHandler::new(Arc::clone(®istry));
7750 let responses = handler
7751 .handle_control(
7752 ConnectionId::new(1),
7753 hello_frame("aft", PROTOCOL_VERSION, 7),
7754 )
7755 .unwrap();
7756 let ack = parse_ack(&responses[0]);
7757 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
7758 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
7759 }
7760
7761 #[tokio::test]
7762 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
7763 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
7764 let registry = Arc::new(Registry::default());
7765 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
7766 let responses = handler
7767 .handle_control(
7768 ConnectionId::new(1),
7769 hello_frame("aft", PROTOCOL_VERSION, 7),
7770 )
7771 .unwrap();
7772 let ack = parse_ack(&responses[0]);
7773 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
7774
7775 let described = handler
7776 .handle_control_frame(
7777 &route_ctx(ConnectionId::new(2)).0,
7778 Frame::build(
7779 FrameType::Request,
7780 control_flags(),
7781 0,
7782 0,
7783 9,
7784 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
7785 )
7786 .unwrap(),
7787 )
7788 .await
7789 .unwrap();
7790 let ClientControlResponse::ServerDescribe { machine_id, .. } =
7791 serde_json::from_slice(&described[0].body).unwrap()
7792 else {
7793 panic!("server.describe answered with another shape");
7794 };
7795 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
7796 }
7797
7798 #[test]
7799 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
7800 let registry = Arc::new(Registry::default());
7803 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
7804 crate::daemon_config::StorageConfig::Sqlite {
7805 data_home: std::path::PathBuf::from("/data"),
7806 },
7807 ));
7808
7809 let responses = handler
7810 .handle_control(
7811 ConnectionId::new(1),
7812 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
7813 )
7814 .unwrap();
7815 let ack = parse_ack(&responses[0]);
7816 assert_eq!(
7817 ack.storage,
7818 Some(serde_json::json!({
7819 "module_id": "alfonso-routing",
7820 "storage_namespace": "default",
7821 "isolation": { "kind": "module" },
7822 "backend": {
7823 "backend": "sqlite",
7824 "path": "/data/cortexkit/alfonso-routing/store.db"
7825 }
7826 })),
7827 "the delivered descriptor is the module's own sqlite store path"
7828 );
7829 }
7830
7831 #[test]
7832 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
7833 let registry = Arc::new(Registry::default());
7834 let handler = ControlHandler::new(Arc::clone(®istry));
7835 let conn = ConnectionId::new(1);
7836 let responses = handler
7837 .handle_control(
7838 conn,
7839 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
7840 )
7841 .unwrap();
7842 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
7843 let registration = registry.get_module("aft").unwrap().unwrap();
7844 assert_eq!(registration.control_ops, module_baseline_control_ops());
7845
7846 let frame =
7847 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
7848 assert!(handler
7849 .guard_module_control_op(&frame, "aft", "route.bind")
7850 .unwrap()
7851 .is_none());
7852 let error = handler
7853 .guard_module_control_op(&frame, "aft", "test.synthetic")
7854 .unwrap()
7855 .expect("synthetic ungranted op should be rejected");
7856 assert_eq!(error.header.ty, FrameType::Error);
7857 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
7858 }
7859
7860 #[test]
7861 fn hello_control_ops_some_adds_optional_grants() {
7862 let registry = Arc::new(Registry::default());
7863 let handler = ControlHandler::new(Arc::clone(®istry));
7864 handler
7865 .handle_control(
7866 ConnectionId::new(1),
7867 hello_frame_with_control_ops(
7868 "aft",
7869 PROTOCOL_VERSION,
7870 7,
7871 Some(vec![
7872 "future.synthetic".to_string(),
7873 "route.bind".to_string(),
7874 ]),
7875 ),
7876 )
7877 .unwrap();
7878 let registration = registry.get_module("aft").unwrap().unwrap();
7879 assert_eq!(
7880 registration.control_ops,
7881 vec![
7882 "route.bind".to_string(),
7883 "route.status".to_string(),
7884 "future.synthetic".to_string(),
7885 ]
7886 );
7887 let frame =
7888 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
7889 assert!(handler
7890 .guard_module_control_op(&frame, "aft", "future.synthetic")
7891 .unwrap()
7892 .is_none());
7893 }
7894
7895 #[tokio::test]
7896 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
7897 let registry = Arc::new(Registry::default());
7898 let forwarding = Arc::new(ForwardingTable::default());
7899 let handler =
7900 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
7901 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
7902 hello_via_sink(
7903 &handler,
7904 &module_ctx,
7905 &mut module_rx,
7906 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
7907 )
7908 .await;
7909
7910 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
7911 let responses = handler
7912 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
7913 .await
7914 .unwrap();
7915 assert_eq!(responses.len(), 1);
7916 assert_eq!(responses[0].header.ty, FrameType::Error);
7917 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
7918 assert!(module_rx.try_recv().is_err());
7919 }
7920
7921 #[tokio::test]
7922 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
7923 let registry = Arc::new(Registry::default());
7924 let forwarding = Arc::new(ForwardingTable::default());
7925 let handler =
7926 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
7927 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
7928 hello_via_sink(
7929 &handler,
7930 &module_ctx,
7931 &mut module_rx,
7932 hello_frame_with_control_ops(
7933 "aft",
7934 PROTOCOL_VERSION,
7935 7,
7936 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
7937 ),
7938 )
7939 .await;
7940
7941 let project_root = unique_project_root("demux");
7942 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
7943 let route_handler = handler.clone();
7944 let route_task = tokio::spawn(async move {
7945 route_handler
7946 .handle_control_frame(
7947 &route_client_ctx,
7948 route_open_frame(100, "aft", project_root),
7949 )
7950 .await
7951 .unwrap()
7952 });
7953 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7954 .await
7955 .unwrap()
7956 .unwrap();
7957 assert!(matches!(
7958 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
7959 ModuleControlRequest::RouteBind { .. }
7960 ));
7961
7962 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
7963 let health_handler = handler.clone();
7964 let health_task = tokio::spawn(async move {
7965 health_handler
7966 .handle_control_frame(
7967 &health_client_ctx,
7968 supervisor_health_probe_frame(101, "aft"),
7969 )
7970 .await
7971 .unwrap()
7972 });
7973 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7974 .await
7975 .unwrap()
7976 .unwrap();
7977 assert_eq!(
7978 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
7979 ModuleControlRequest::HealthCheck {}
7980 );
7981
7982 handler
7983 .handle_control_frame(
7984 &module_ctx,
7985 health_response(health_frame.header.corr, HealthStatus::Degraded),
7986 )
7987 .await
7988 .unwrap();
7989 let health_response = health_task.await.unwrap();
7990 assert_eq!(health_response.len(), 1);
7991 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
7992 ClientControlResponse::SupervisorHealthProbe {
7993 module_id,
7994 status,
7995 detail,
7996 metrics,
7997 } => {
7998 assert_eq!(module_id, "aft");
7999 assert_eq!(status, HealthStatus::Degraded);
8000 assert_eq!(detail.as_deref(), Some("warming"));
8001 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
8002 }
8003 other => panic!("unexpected health response: {other:?}"),
8004 }
8005
8006 handler
8007 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8008 .await
8009 .unwrap();
8010 let route_response = route_task.await.unwrap();
8011 assert!(route_response.is_empty());
8012 let published = route_client_rx.recv().await.unwrap();
8013 assert!(matches!(
8014 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8015 ClientControlResponse::RouteOpen { .. }
8016 ));
8017 }
8018
8019 async fn relay_route_open(
8024 handler: &ControlHandler,
8025 client_connection: ConnectionId,
8026 client_egress: &FrameSink,
8027 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
8028 corr: u64,
8029 module_id: &str,
8030 project_root_label: &str,
8031 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
8032 let ctx = RouteCtx {
8033 connection_id: client_connection,
8034 egress: client_egress.clone(),
8035 };
8036 let handler = handler.clone();
8037 let project_root = unique_project_root(project_root_label);
8038 let module_id = module_id.to_string();
8039 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
8040 let task = tokio::spawn(async move {
8041 let _guard = tracing::dispatcher::set_default(&dispatch);
8042 handler
8043 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
8044 .await
8045 .unwrap()
8046 });
8047 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
8048 .await
8049 .expect("module receives the relayed route.bind")
8050 .expect("module egress is open");
8051 (task, bind.frame)
8052 }
8053
8054 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
8055 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
8056 ModuleControlRequest::RouteBind {
8057 route_channel,
8058 epoch,
8059 ..
8060 } => (route_channel, epoch),
8061 other => panic!("expected a route.bind request, got {other:?}"),
8062 }
8063 }
8064
8065 fn published_route(frame: &Frame) -> (u16, u32) {
8066 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
8067 ClientControlResponse::RouteOpen {
8068 route_channel,
8069 route_epoch,
8070 } => (route_channel, route_epoch),
8071 other => panic!("expected a route.open response, got {other:?}"),
8072 }
8073 }
8074
8075 #[tokio::test]
8095 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
8096 let registry = Arc::new(Registry::default());
8097 let forwarding = Arc::new(ForwardingTable::default());
8098 let handler =
8099 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8100
8101 let module_connection = ConnectionId::new(30);
8102 let (module_ctx, mut module_rx) = route_ctx(module_connection);
8103 hello_via_sink(
8104 &handler,
8105 &module_ctx,
8106 &mut module_rx,
8107 hello_frame("aft", PROTOCOL_VERSION, 7),
8108 )
8109 .await;
8110
8111 let dying_client = ConnectionId::new(31);
8112 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
8113
8114 let (first_task, first_bind) = relay_route_open(
8117 &handler,
8118 dying_client,
8119 &dying_ctx.egress,
8120 &mut module_rx,
8121 100,
8122 "aft",
8123 "closing-first",
8124 )
8125 .await;
8126 handler
8127 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
8128 .await
8129 .unwrap();
8130 assert!(first_task.await.unwrap().is_empty());
8131 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
8132
8133 let (second_task, second_bind) = relay_route_open(
8135 &handler,
8136 dying_client,
8137 &dying_ctx.egress,
8138 &mut module_rx,
8139 101,
8140 "aft",
8141 "closing-second",
8142 )
8143 .await;
8144 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
8145
8146 assert!(forwarding
8149 .escalate_client_delivery_failure(
8150 dying_client,
8151 first_channel,
8152 first_epoch,
8153 CloseReason::new(
8154 "module_to_client_delivery_failed",
8155 "client egress refused a module frame",
8156 ),
8157 crate::forwarding::UndeliveredFrame {
8158 module_id: None,
8159 sink: &dying_ctx.egress,
8160 },
8161 )
8162 .unwrap());
8163 assert!(!dying_ctx.egress.is_closed());
8164
8165 let ack = handler
8167 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
8168 .await;
8169 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
8170 if module_loop_error.is_some() {
8171 handler.cleanup_connection(module_connection).unwrap();
8175 }
8176 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8180 .await
8181 .ok()
8182 .flatten();
8183
8184 assert!(
8186 registry
8187 .get_module_by_connection(module_connection)
8188 .unwrap()
8189 .is_some(),
8190 "one client's closing connection ended the shared module connection: \
8191 {module_loop_error:?}"
8192 );
8193 let cotenant = ConnectionId::new(32);
8195 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
8196 let (cotenant_task, cotenant_bind) = relay_route_open(
8197 &handler,
8198 cotenant,
8199 &cotenant_ctx.egress,
8200 &mut module_rx,
8201 102,
8202 "aft",
8203 "closing-cotenant",
8204 )
8205 .await;
8206 handler
8207 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
8208 .await
8209 .unwrap();
8210 assert!(cotenant_task.await.unwrap().is_empty());
8211 let (cotenant_channel, cotenant_epoch) =
8212 published_route(&cotenant_rx.recv().await.unwrap());
8213 assert!(matches!(
8214 forwarding
8215 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
8216 .unwrap(),
8217 DataRoute::Client(DataRouteState::Bound(_))
8218 ));
8219
8220 let goodbye = post_ack_module_frame
8223 .expect("module receives a GOODBYE for the abandoned route channel");
8224 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
8225 assert_eq!(goodbye.header.channel, abandoned_channel);
8226 assert_eq!(goodbye.header.epoch, abandoned_epoch);
8227
8228 assert!(dying_rx.try_recv().is_err());
8232 let second_response = second_task.await.unwrap();
8233 assert_eq!(second_response.len(), 1);
8234 assert_eq!(
8235 parse_error(&second_response[0])["code"],
8236 "target_unavailable"
8237 );
8238 }
8239
8240 #[test]
8247 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
8248 let handler = ControlHandler::default();
8249 let module_connection = ConnectionId::new(30);
8250 let client_connection = ConnectionId::new(31);
8251
8252 handler
8253 .refuse_to_end_module_connection_for_a_client(
8254 module_connection,
8255 77,
8256 ForwardingError::ConnectionClosing {
8257 connection_id: client_connection,
8258 },
8259 )
8260 .expect("a closing client must never end the module connection");
8261
8262 assert!(matches!(
8263 handler.refuse_to_end_module_connection_for_a_client(
8264 module_connection,
8265 78,
8266 ForwardingError::ConnectionClosing {
8267 connection_id: module_connection,
8268 },
8269 ),
8270 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
8271 connection_id
8272 })) if connection_id == module_connection
8273 ));
8274 assert!(matches!(
8275 handler.refuse_to_end_module_connection_for_a_client(
8276 module_connection,
8277 79,
8278 ForwardingError::Poisoned,
8279 ),
8280 Err(RouterError::Forwarding(ForwardingError::Poisoned))
8281 ));
8282 assert!(matches!(
8283 handler.refuse_to_end_module_connection_for_a_client(
8284 module_connection,
8285 80,
8286 ForwardingError::StaleModuleEndpoint,
8287 ),
8288 Err(RouterError::Forwarding(
8289 ForwardingError::StaleModuleEndpoint
8290 ))
8291 ));
8292 }
8293
8294 #[tokio::test]
8307 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
8308 let registry = Arc::new(Registry::default());
8309 let forwarding = Arc::new(ForwardingTable::default());
8310 let supervisor = SupervisorHandle::new();
8311 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8312 let handler =
8313 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8314 .with_supervisor(supervisor);
8315
8316 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
8317 hello_via_sink(
8318 &handler,
8319 &target_ctx,
8320 &mut target_rx,
8321 hello_frame("target", PROTOCOL_VERSION, 1),
8322 )
8323 .await;
8324
8325 let wrong_nonce = handler
8329 .handle_control_frame(
8330 &route_ctx(ConnectionId::new(91)).0,
8331 route_open_frame_with_admission_facts(
8332 20,
8333 "target",
8334 unique_project_root("admission-facts"),
8335 Some(subc_control::ConsumerIdentity {
8336 module_id: "fed".to_string(),
8337 launch_nonce: "not-the-real-nonce".to_string(),
8338 }),
8339 None,
8340 ),
8341 )
8342 .await
8343 .unwrap();
8344 assert_eq!(
8345 parse_error(&wrong_nonce[0])["code"],
8346 "bad_consumer_identity",
8347 "a mismatched launch nonce must be refused, not stamped Reserved"
8348 );
8349
8350 let never_spawned = handler
8354 .handle_control_frame(
8355 &route_ctx(ConnectionId::new(92)).0,
8356 route_open_frame_with_admission_facts(
8357 21,
8358 "target",
8359 unique_project_root("admission-facts"),
8360 Some(subc_control::ConsumerIdentity {
8361 module_id: "never-spawned".to_string(),
8362 launch_nonce: "any-nonce".to_string(),
8363 }),
8364 None,
8365 ),
8366 )
8367 .await
8368 .unwrap();
8369 assert_eq!(
8370 parse_error(&never_spawned[0])["code"],
8371 "bad_consumer_identity",
8372 "an unspawned module_id must be refused rather than accepted for lack of a record"
8373 );
8374 }
8375
8376 #[tokio::test]
8399 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
8400 let registry = Arc::new(Registry::default());
8401 let forwarding = Arc::new(ForwardingTable::default());
8402 let supervisor = SupervisorHandle::new();
8403 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8404 let handler =
8405 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8406 .with_supervisor(supervisor);
8407
8408 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
8409 hello_via_sink(
8410 &handler,
8411 &target_ctx,
8412 &mut target_rx,
8413 hello_frame("target", PROTOCOL_VERSION, 1),
8414 )
8415 .await;
8416
8417 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
8418 let route_handler = handler.clone();
8419 let route_task = tokio::spawn(async move {
8420 route_handler
8421 .handle_control_frame(
8422 &client_ctx,
8423 route_open_frame_with_admission_facts(
8424 30,
8425 "target",
8426 unique_project_root("admission-facts"),
8427 Some(subc_control::ConsumerIdentity {
8428 module_id: "fed".to_string(),
8429 launch_nonce: "fed-nonce".to_string(),
8430 }),
8431 None,
8432 ),
8433 )
8434 .await
8435 .unwrap()
8436 });
8437
8438 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8445 .await
8446 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
8447 .expect("module control channel closed before route.bind");
8448 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8449 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
8450 panic!("expected route.bind")
8451 };
8452 assert_eq!(
8453 principal,
8454 Some(Principal::Reserved {
8455 module_id: "fed".to_string()
8456 }),
8457 "a correctly attested consumer must be stamped Reserved for its own id"
8458 );
8459
8460 handler
8461 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8462 .await
8463 .unwrap();
8464 assert!(route_task.await.unwrap().is_empty());
8465 assert!(
8466 matches!(
8467 serde_json::from_slice::<ClientControlResponse>(
8468 &client_rx.recv().await.unwrap().body
8469 )
8470 .unwrap(),
8471 ClientControlResponse::RouteOpen { .. }
8472 ),
8473 "the route must actually open, not merely avoid an error"
8474 );
8475 }
8476
8477 #[tokio::test(start_paused = true)]
8478 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
8479 let registry = Arc::new(Registry::default());
8480 let forwarding = Arc::new(ForwardingTable::default());
8481 let supervisor = SupervisorHandle::new();
8482 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8483 let handler =
8484 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8485 .with_supervisor(supervisor);
8486
8487 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
8488 hello_via_sink(
8489 &handler,
8490 &target_ctx,
8491 &mut target_rx,
8492 hello_frame("target", PROTOCOL_VERSION, 1),
8493 )
8494 .await;
8495
8496 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
8497 let direct_handler = handler.clone();
8498 let direct_open = tokio::spawn(async move {
8499 direct_handler
8500 .handle_control_frame(
8501 &direct_ctx,
8502 route_open_frame(2, "target", unique_project_root("route-census-direct")),
8503 )
8504 .await
8505 .unwrap()
8506 });
8507 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8508 .await
8509 .expect("no direct route.bind within 5s")
8510 .expect("target control channel closed before direct route.bind");
8511 handler
8512 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
8513 .await
8514 .unwrap();
8515 assert!(direct_open.await.unwrap().is_empty());
8516 let _ = direct_rx.recv().await.unwrap();
8517
8518 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
8519 let reserved_handler = handler.clone();
8520 let reserved_open = tokio::spawn(async move {
8521 reserved_handler
8522 .handle_control_frame(
8523 &reserved_ctx,
8524 route_open_frame_with_admission_facts(
8525 3,
8526 "target",
8527 unique_project_root("admission-facts"),
8528 Some(ConsumerIdentity {
8529 module_id: "fed".to_string(),
8530 launch_nonce: "fed-nonce".to_string(),
8531 }),
8532 None,
8533 ),
8534 )
8535 .await
8536 .unwrap()
8537 });
8538 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8539 .await
8540 .expect("no reserved route.bind within 5s")
8541 .expect("target control channel closed before reserved route.bind");
8542 handler
8543 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
8544 .await
8545 .unwrap();
8546 assert!(reserved_open.await.unwrap().is_empty());
8547 let _ = reserved_rx.recv().await.unwrap();
8548
8549 forwarding
8550 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
8551 .unwrap();
8552 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
8553 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
8554 module_id: Some("target".to_string()),
8555 })
8556 .unwrap();
8557 let census_frame =
8558 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
8559 let response = handler
8560 .handle_control_frame(&census_ctx, census_frame)
8561 .await
8562 .unwrap()
8563 .pop()
8564 .unwrap();
8565 let actual: Value = serde_json::from_slice(&response.body).unwrap();
8566 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
8567 assert!(matches!(
8568 decoded,
8569 ClientControlResponse::SupervisorRoutes { .. }
8570 ));
8571 let routes = actual["modules"][0]["routes"].as_array().unwrap();
8572 assert_eq!(routes.len(), 2);
8573 assert!(routes.iter().all(|route| route["draining"] == true));
8574 assert!(
8577 routes.iter().all(|route| route["drain_reason"] == "reload"),
8578 "draining routes must name the drain's reason: {routes:?}"
8579 );
8580 assert!(routes.iter().any(|route| {
8581 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
8582 }));
8583 assert!(routes.iter().any(|route| {
8584 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
8585 }));
8586
8587 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8588 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
8589 if std::env::var_os("UPDATE_GOLDEN").is_some() {
8590 std::fs::write(
8591 &golden_path,
8592 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
8593 )
8594 .unwrap();
8595 }
8596 let expected: Value =
8597 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
8598 assert_eq!(actual, expected);
8599 }
8600
8601 async fn query_live_roots(
8602 handler: &ControlHandler,
8603 module_ctx: &RouteCtx,
8604 ) -> ModuleControlResponseToModule {
8605 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
8606 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
8607 let response = handler
8608 .handle_control_frame(module_ctx, frame)
8609 .await
8610 .unwrap()
8611 .pop()
8612 .unwrap();
8613 serde_json::from_slice(&response.body).unwrap()
8614 }
8615
8616 #[tokio::test(start_paused = true)]
8617 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
8618 let registry = Arc::new(Registry::default());
8619 let forwarding = Arc::new(ForwardingTable::default());
8620 let handler = ControlHandler::with_forwarding(registry, forwarding);
8621 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
8622 hello_via_sink(
8623 &handler,
8624 &target_ctx,
8625 &mut target_rx,
8626 hello_frame("target", PROTOCOL_VERSION, 1),
8627 )
8628 .await;
8629 let root = unique_project_root("live-roots-known");
8630 let path = ProjectRootId::from_path_allowing_missing(root.path())
8631 .unwrap()
8632 .as_path()
8633 .to_path_buf();
8634 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
8635 let open_handler = handler.clone();
8636 let opened = tokio::spawn(async move {
8637 open_handler
8638 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
8639 .await
8640 .unwrap()
8641 });
8642 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8643 .await
8644 .unwrap()
8645 .unwrap();
8646 handler
8647 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
8648 .await
8649 .unwrap();
8650 assert!(opened.await.unwrap().is_empty());
8651 let _ = client_rx.recv().await.unwrap();
8652
8653 let root = unique_project_root("live-roots-pending");
8654 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
8655 .unwrap()
8656 .as_path()
8657 .to_path_buf();
8658 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
8659 let open_handler = handler.clone();
8660 let pending = tokio::spawn(async move {
8661 open_handler
8662 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
8663 .await
8664 .unwrap()
8665 });
8666 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8667 .await
8668 .unwrap()
8669 .unwrap();
8670 let actual = query_live_roots(&handler, &target_ctx).await;
8671 let ModuleControlResponseToModule::LiveRoots {
8672 roots,
8673 unknown_root_bindings,
8674 total_bindings,
8675 } = actual
8676 else {
8677 panic!("expected live roots")
8678 };
8679 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
8680 assert_eq!(unknown_root_bindings, 0);
8681 assert_eq!(
8682 roots.len(),
8683 2,
8684 "root-known arm must retain each canonical root"
8685 );
8686 assert_eq!(
8687 total_bindings,
8688 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
8689 );
8690 let counts = roots
8691 .iter()
8692 .map(|root| (root.project_root.clone(), root.bound, root.pending))
8693 .collect::<Vec<_>>();
8694 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
8695 expected.sort_by(|a, b| a.0.cmp(&b.0));
8696 assert_eq!(
8697 counts, expected,
8698 "roots must sort by path and count pending separately"
8699 );
8700 handler
8701 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
8702 .await
8703 .unwrap();
8704 assert!(pending.await.unwrap().is_empty());
8705 }
8706
8707 #[tokio::test(start_paused = true)]
8708 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
8709 let forwarding = Arc::new(ForwardingTable::default());
8710 let handler =
8711 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
8712 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
8713 hello_via_sink(
8714 &handler,
8715 &target_ctx,
8716 &mut target_rx,
8717 hello_frame("target", PROTOCOL_VERSION, 1),
8718 )
8719 .await;
8720 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
8721 let pending = forwarding
8722 .begin_route_bind_relay_for_test(
8723 client_ctx.connection_id,
8724 client_ctx.egress.clone(),
8725 2,
8726 "target",
8727 )
8728 .unwrap();
8729 forwarding
8730 .complete_pending_relay(
8731 target_ctx.connection_id,
8732 pending.corr,
8733 RouteBindRelayOutcome::Accepted,
8734 )
8735 .unwrap();
8736 let actual = query_live_roots(&handler, &target_ctx).await;
8737 let ModuleControlResponseToModule::LiveRoots {
8738 roots,
8739 unknown_root_bindings,
8740 total_bindings,
8741 } = actual
8742 else {
8743 panic!("expected live roots")
8744 };
8745 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
8746 assert_eq!(
8747 unknown_root_bindings, 1,
8748 "unknown-root arm must not read as no bindings"
8749 );
8750 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
8751 assert_eq!(
8752 total_bindings,
8753 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
8754 );
8755 }
8756
8757 #[tokio::test(start_paused = true)]
8765 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
8766 let forwarding = Arc::new(ForwardingTable::default());
8767 let handler =
8768 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
8769 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
8770 let replies = handler
8771 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
8772 .await
8773 .unwrap();
8774 let queued_by_hello = module_rx.len();
8775
8776 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
8777 let open_handler = handler.clone();
8778 let open = tokio::spawn(async move {
8779 open_handler
8780 .handle_control_frame(
8781 &client_ctx,
8782 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
8783 )
8784 .await
8785 .unwrap()
8786 });
8787 let mut spins = 0;
8789 while module_rx.len() == queued_by_hello {
8790 spins += 1;
8791 assert!(spins < 10_000, "route.open never queued a route.bind");
8792 tokio::task::yield_now().await;
8793 }
8794
8795 for reply in replies {
8797 module_ctx.egress.send(reply).await.unwrap();
8798 }
8799
8800 let first = module_rx.recv().await.unwrap().frame;
8801 assert_eq!(
8802 first.header.ty,
8803 FrameType::HelloAck,
8804 "the first frame a registering module reads must be its HELLO_ACK"
8805 );
8806 assert_eq!(first.header.corr, 7);
8807 let second = module_rx.recv().await.unwrap().frame;
8808 assert_eq!(second.header.ty, FrameType::Request);
8809 assert!(
8810 matches!(
8811 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
8812 ModuleControlRequest::RouteBind { .. }
8813 ),
8814 "the route.bind follows the ack"
8815 );
8816 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
8817
8818 handler
8819 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
8820 .await
8821 .unwrap();
8822 assert!(open.await.unwrap().is_empty());
8823 let _ = client_rx.recv().await.unwrap();
8824 }
8825
8826 #[tokio::test(start_paused = true)]
8827 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
8828 let handler = ControlHandler::with_forwarding(
8829 Arc::new(Registry::default()),
8830 Arc::new(ForwardingTable::default()),
8831 );
8832 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
8833 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
8834 hello_via_sink(
8835 &handler,
8836 &first_ctx,
8837 &mut first_rx,
8838 hello_frame("first", PROTOCOL_VERSION, 1),
8839 )
8840 .await;
8841 hello_via_sink(
8842 &handler,
8843 &second_ctx,
8844 &mut second_rx,
8845 hello_frame("second", PROTOCOL_VERSION, 2),
8846 )
8847 .await;
8848 let root = unique_project_root("second-only");
8849 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
8850 let cloned = handler.clone();
8851 let open = tokio::spawn(async move {
8852 cloned
8853 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
8854 .await
8855 .unwrap()
8856 });
8857 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
8858 .await
8859 .unwrap()
8860 .unwrap();
8861 let first = query_live_roots(&handler, &first_ctx).await;
8862 let second = query_live_roots(&handler, &second_ctx).await;
8863 assert!(
8864 matches!(
8865 first,
8866 ModuleControlResponseToModule::LiveRoots {
8867 total_bindings: 0,
8868 ..
8869 }
8870 ),
8871 "cross-module scope must not expose another module's roots"
8872 );
8873 assert!(
8874 matches!(
8875 second,
8876 ModuleControlResponseToModule::LiveRoots {
8877 total_bindings: 1,
8878 ..
8879 }
8880 ),
8881 "second module must see its pending route"
8882 );
8883 handler
8884 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
8885 .await
8886 .unwrap();
8887 assert!(open.await.unwrap().is_empty());
8888 }
8889
8890 #[tokio::test(start_paused = true)]
8891 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
8892 let handler = ControlHandler::with_forwarding(
8893 Arc::new(Registry::default()),
8894 Arc::new(ForwardingTable::default()),
8895 );
8896 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
8897 hello_via_sink(
8898 &handler,
8899 &target_ctx,
8900 &mut target_rx,
8901 hello_frame("target", PROTOCOL_VERSION, 1),
8902 )
8903 .await;
8904 let actual = query_live_roots(&handler, &target_ctx).await;
8905 let ModuleControlResponseToModule::LiveRoots {
8906 roots,
8907 unknown_root_bindings,
8908 total_bindings,
8909 } = actual
8910 else {
8911 panic!("expected live roots")
8912 };
8913 assert!(roots.is_empty());
8914 assert_eq!(unknown_root_bindings, 0);
8915 assert_eq!(total_bindings, 0);
8916 assert_eq!(
8917 total_bindings,
8918 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
8919 );
8920 }
8921
8922 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
8929 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8930 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
8931 let text = std::fs::read_to_string(&path)
8932 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
8933 let vectors: Vec<(String, Value)> = text
8934 .lines()
8935 .filter(|line| !line.trim().is_empty())
8936 .map(|line| {
8937 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
8938 let id = entry["corpus_id"]
8939 .as_str()
8940 .expect("every vector carries a corpus_id")
8941 .to_string();
8942 (id, entry["package"].clone())
8943 })
8944 .collect();
8945 assert_eq!(
8949 vectors.len(),
8950 3,
8951 "vendored fed corpus changed size; re-sync from subc-federation"
8952 );
8953
8954 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
8969 let richest = vectors
8970 .iter()
8971 .filter_map(|(_, package)| package.as_object())
8972 .map(|object| {
8973 object
8974 .keys()
8975 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
8976 .count()
8977 })
8978 .max()
8979 .unwrap_or(0);
8980 assert!(
8981 richest >= 2,
8982 "vendored corpus no longer carries a package with unmodelled fields, \
8983 so the relay test can no longer distinguish a verbatim relay from a lossy one"
8984 );
8985
8986 vectors
8987 }
8988
8989 #[tokio::test]
8997 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
8998 for (corpus_id, package) in fed_admission_facts_vectors() {
8999 let registry = Arc::new(Registry::default());
9000 let forwarding = Arc::new(ForwardingTable::default());
9001 let supervisor = SupervisorHandle::new();
9002 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9003 let handler =
9004 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9005 .with_supervisor(supervisor)
9006 .with_admission_facts_config(
9007 Some("fed".to_string()),
9008 Some(vec!["target".to_string()]),
9009 );
9010
9011 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
9012 hello_via_sink(
9013 &handler,
9014 &target_ctx,
9015 &mut target_rx,
9016 hello_frame("target", PROTOCOL_VERSION, 1),
9017 )
9018 .await;
9019
9020 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
9021 let route_handler = handler.clone();
9022 let expected = package.clone();
9023 let route_task = tokio::spawn(async move {
9024 route_handler
9025 .handle_control_frame(
9026 &client_ctx,
9027 route_open_frame_with_admission_facts(
9028 20,
9029 "target",
9030 unique_project_root("admission-facts"),
9031 Some(subc_control::ConsumerIdentity {
9032 module_id: "fed".to_string(),
9033 launch_nonce: "fed-nonce".to_string(),
9034 }),
9035 Some(package),
9036 ),
9037 )
9038 .await
9039 .unwrap()
9040 });
9041
9042 let bind_frame = target_rx.recv().await.unwrap();
9043 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9044 let ModuleControlRequest::RouteBind {
9045 admission_facts, ..
9046 } = bind
9047 else {
9048 panic!("{corpus_id}: expected route.bind")
9049 };
9050 assert_eq!(
9051 admission_facts,
9052 Some(expected),
9053 "{corpus_id}: relay must not add, drop or reshape any field"
9054 );
9055
9056 handler
9057 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
9058 .await
9059 .unwrap();
9060 route_task.await.unwrap();
9061 }
9062 }
9063
9064 #[tokio::test]
9065 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
9066 let registry = Arc::new(Registry::default());
9067 let forwarding = Arc::new(ForwardingTable::default());
9068 let supervisor = SupervisorHandle::new();
9069 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9070 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
9071 let handler =
9072 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9073 .with_supervisor(supervisor)
9074 .with_admission_facts_config(
9075 Some("fed".to_string()),
9076 Some(vec!["target".to_string()]),
9077 );
9078
9079 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
9080 hello_via_sink(
9081 &handler,
9082 &target_ctx,
9083 &mut target_rx,
9084 hello_frame("target", PROTOCOL_VERSION, 1),
9085 )
9086 .await;
9087 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
9088 hello_via_sink(
9089 &handler,
9090 &other_ctx,
9091 &mut other_rx,
9092 hello_frame("other", PROTOCOL_VERSION, 2),
9093 )
9094 .await;
9095
9096 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
9097 let expected_facts = facts.clone();
9098 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
9099 let route_handler = handler.clone();
9100 let route_task = tokio::spawn(async move {
9101 route_handler
9102 .handle_control_frame(
9103 &client_ctx,
9104 route_open_frame_with_admission_facts(
9105 10,
9106 "target",
9107 unique_project_root("admission-facts"),
9108 Some(subc_control::ConsumerIdentity {
9109 module_id: "fed".to_string(),
9110 launch_nonce: "fed-nonce".to_string(),
9111 }),
9112 Some(facts.clone()),
9113 ),
9114 )
9115 .await
9116 .unwrap()
9117 });
9118 let bind_frame = target_rx.recv().await.unwrap();
9119 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9120 let ModuleControlRequest::RouteBind {
9121 admission_facts, ..
9122 } = bind
9123 else {
9124 panic!("expected route.bind")
9125 };
9126 assert_eq!(admission_facts, Some(expected_facts));
9127 handler
9128 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
9129 .await
9130 .unwrap();
9131 assert!(route_task.await.unwrap().is_empty());
9132 assert!(matches!(
9133 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
9134 .unwrap(),
9135 ClientControlResponse::RouteOpen { .. }
9136 ));
9137
9138 let direct = handler
9139 .handle_control_frame(
9140 &route_ctx(ConnectionId::new(73)).0,
9141 route_open_frame_with_admission_facts(
9142 11,
9143 "target",
9144 unique_project_root("admission-facts"),
9145 None,
9146 Some(json!({"x": 1})),
9147 ),
9148 )
9149 .await
9150 .unwrap();
9151 assert_eq!(
9152 parse_error(&direct[0])["code"],
9153 "admission_facts_not_permitted"
9154 );
9155
9156 let different_reserved = handler
9157 .handle_control_frame(
9158 &route_ctx(ConnectionId::new(77)).0,
9159 route_open_frame_with_admission_facts(
9160 15,
9161 "target",
9162 unique_project_root("admission-facts"),
9163 Some(subc_control::ConsumerIdentity {
9164 module_id: "other".to_string(),
9165 launch_nonce: "other-nonce".to_string(),
9166 }),
9167 Some(json!({"x": 1})),
9168 ),
9169 )
9170 .await
9171 .unwrap();
9172 assert_eq!(
9173 parse_error(&different_reserved[0])["code"],
9174 "admission_facts_not_permitted"
9175 );
9176
9177 let other_target = handler
9178 .handle_control_frame(
9179 &route_ctx(ConnectionId::new(74)).0,
9180 route_open_frame_with_admission_facts(
9181 12,
9182 "other",
9183 unique_project_root("admission-facts"),
9184 Some(subc_control::ConsumerIdentity {
9185 module_id: "fed".to_string(),
9186 launch_nonce: "fed-nonce".to_string(),
9187 }),
9188 Some(json!({"x": 1})),
9189 ),
9190 )
9191 .await
9192 .unwrap();
9193 assert_eq!(
9194 parse_error(&other_target[0])["code"],
9195 "admission_facts_target_not_allowed"
9196 );
9197
9198 let nonexistent = handler
9199 .handle_control_frame(
9200 &route_ctx(ConnectionId::new(75)).0,
9201 route_open_frame_with_admission_facts(
9202 13,
9203 "missing",
9204 unique_project_root("admission-facts"),
9205 None,
9206 Some(json!({"x": 1})),
9207 ),
9208 )
9209 .await
9210 .unwrap();
9211 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
9212
9213 let described = handler
9214 .handle_control_frame(
9215 &route_ctx(ConnectionId::new(76)).0,
9216 Frame::build(
9217 FrameType::Request,
9218 control_flags(),
9219 0,
9220 0,
9221 14,
9222 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
9223 )
9224 .unwrap(),
9225 )
9226 .await
9227 .unwrap();
9228 let ClientControlResponse::ServerDescribe { capabilities, .. } =
9229 serde_json::from_slice(&described[0].body).unwrap()
9230 else {
9231 panic!("expected server.describe response")
9232 };
9233 assert!(capabilities
9234 .iter()
9235 .any(|cap| cap == "admission_facts_relay_v1"));
9236 }
9237
9238 #[tokio::test]
9239 async fn admission_facts_without_configured_carrier_are_rejected() {
9240 let registry = Arc::new(Registry::default());
9241 let forwarding = Arc::new(ForwardingTable::default());
9242 let handler = ControlHandler::with_forwarding(registry, forwarding);
9243 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
9244 hello_via_sink(
9245 &handler,
9246 &target_ctx,
9247 &mut target_rx,
9248 hello_frame("target", PROTOCOL_VERSION, 1),
9249 )
9250 .await;
9251
9252 let responses = handler
9253 .handle_control_frame(
9254 &route_ctx(ConnectionId::new(79)).0,
9255 route_open_frame_with_admission_facts(
9256 16,
9257 "target",
9258 unique_project_root("admission-facts"),
9259 None,
9260 Some(json!({"x": 1})),
9261 ),
9262 )
9263 .await
9264 .unwrap();
9265 assert_eq!(
9266 parse_error(&responses[0])["code"],
9267 "admission_facts_not_permitted"
9268 );
9269 }
9270
9271 #[tokio::test]
9272 async fn route_open_relays_consumer_capabilities_verbatim() {
9273 let registry = Arc::new(Registry::default());
9274 let forwarding = Arc::new(ForwardingTable::default());
9275 let handler =
9276 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9277 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
9278 hello_via_sink(
9279 &handler,
9280 &module_ctx,
9281 &mut module_rx,
9282 hello_frame("aft", PROTOCOL_VERSION, 7),
9283 )
9284 .await;
9285
9286 let expected = vec!["elicitation".to_string(), "roots".to_string()];
9287 let expected_for_request = expected.clone();
9288 let project_root = unique_project_root("consumer-capabilities-present");
9289 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
9290 let route_handler = handler.clone();
9291 let route_task = tokio::spawn(async move {
9292 route_handler
9293 .handle_control_frame(
9294 &client_ctx,
9295 route_open_frame_with_consumer_capabilities(
9296 401,
9297 "aft",
9298 project_root,
9299 Some(expected_for_request),
9300 ),
9301 )
9302 .await
9303 .unwrap()
9304 });
9305 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9306 .await
9307 .unwrap()
9308 .unwrap();
9309 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9310 let ModuleControlRequest::RouteBind {
9311 consumer_capabilities,
9312 ..
9313 } = bind
9314 else {
9315 panic!("expected route.bind request, got {bind:?}");
9316 };
9317 assert_eq!(consumer_capabilities, Some(expected.clone()));
9318
9319 handler
9320 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9321 .await
9322 .unwrap();
9323 let route_response = route_task.await.unwrap();
9324 assert!(route_response.is_empty());
9325 let published = client_rx.recv().await.unwrap();
9326 assert!(matches!(
9327 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9328 ClientControlResponse::RouteOpen { .. }
9329 ));
9330 }
9331
9332 #[tokio::test]
9333 async fn route_open_without_consumer_capabilities_relays_none() {
9334 let registry = Arc::new(Registry::default());
9335 let forwarding = Arc::new(ForwardingTable::default());
9336 let handler =
9337 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9338 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
9339 hello_via_sink(
9340 &handler,
9341 &module_ctx,
9342 &mut module_rx,
9343 hello_frame("aft", PROTOCOL_VERSION, 7),
9344 )
9345 .await;
9346
9347 let project_root = unique_project_root("consumer-capabilities-absent");
9348 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
9349 let route_handler = handler.clone();
9350 let route_task = tokio::spawn(async move {
9351 route_handler
9352 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
9353 .await
9354 .unwrap()
9355 });
9356 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9357 .await
9358 .unwrap()
9359 .unwrap();
9360 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9361 let ModuleControlRequest::RouteBind {
9362 consumer_capabilities,
9363 ..
9364 } = bind
9365 else {
9366 panic!("expected route.bind request, got {bind:?}");
9367 };
9368 assert_eq!(consumer_capabilities, None);
9369
9370 handler
9371 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9372 .await
9373 .unwrap();
9374 let route_response = route_task.await.unwrap();
9375 assert!(route_response.is_empty());
9376 let published = client_rx.recv().await.unwrap();
9377 assert!(matches!(
9378 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9379 ClientControlResponse::RouteOpen { .. }
9380 ));
9381 }
9382
9383 async fn bind_role_versions_for(
9387 sent: Option<BTreeMap<String, String>>,
9388 connection: u64,
9389 ) -> Option<BTreeMap<String, String>> {
9390 let registry = Arc::new(Registry::default());
9391 let forwarding = Arc::new(ForwardingTable::default());
9392 let handler =
9393 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9394 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
9395 hello_via_sink(
9396 &handler,
9397 &module_ctx,
9398 &mut module_rx,
9399 hello_frame("aft", PROTOCOL_VERSION, 7),
9400 )
9401 .await;
9402 let project_root = unique_project_root("role-versions");
9403 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
9404 let route_handler = handler.clone();
9405 let route_task = tokio::spawn(async move {
9406 route_handler
9407 .handle_control_frame(
9408 &client_ctx,
9409 route_open_frame_with_role_versions(403, "aft", project_root, sent),
9410 )
9411 .await
9412 .unwrap()
9413 });
9414 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9415 .await
9416 .expect("a well-formed route.open reaches the module as a bind")
9417 .unwrap();
9418 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9419 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
9420 panic!("expected route.bind request, got {bind:?}");
9421 };
9422 handler
9423 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9424 .await
9425 .unwrap();
9426 assert!(route_task.await.unwrap().is_empty());
9427 let published = client_rx.recv().await.unwrap();
9428 assert!(matches!(
9429 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9430 ClientControlResponse::RouteOpen { .. }
9431 ));
9432 role_versions
9433 }
9434
9435 #[tokio::test]
9436 async fn route_open_relays_role_versions_verbatim() {
9437 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
9438 assert_eq!(
9439 bind_role_versions_for(Some(sent.clone()), 141).await,
9440 Some(sent)
9441 );
9442 }
9443
9444 #[tokio::test]
9447 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
9448 assert_eq!(bind_role_versions_for(None, 143).await, None);
9449 assert_eq!(
9450 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
9451 None
9452 );
9453 }
9454
9455 #[tokio::test]
9456 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
9457 let registry = Arc::new(Registry::default());
9458 let forwarding = Arc::new(ForwardingTable::default());
9459 let handler =
9460 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9461 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
9462 hello_via_sink(
9463 &handler,
9464 &module_ctx,
9465 &mut module_rx,
9466 hello_frame("aft", PROTOCOL_VERSION, 7),
9467 )
9468 .await;
9469
9470 let nine: BTreeMap<String, String> = (0..9)
9471 .map(|index| (format!("role-{index}"), "v1".to_string()))
9472 .collect();
9473 for (label, malformed) in [
9474 (
9475 "invalid role name",
9476 role_versions(&[("Tool_Provider", "v1")]),
9477 ),
9478 ("invalid version", role_versions(&[("tool-provider", "v0")])),
9479 ("nine entries", nine),
9480 ] {
9481 let responses = tokio::time::timeout(
9484 Duration::from_secs(1),
9485 handler.handle_control_frame(
9486 &route_ctx(ConnectionId::new(148)).0,
9487 route_open_frame_with_role_versions(
9488 404,
9489 "aft",
9490 unique_project_root("role-versions-malformed"),
9491 Some(malformed),
9492 ),
9493 ),
9494 )
9495 .await
9496 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
9497 .unwrap();
9498 assert_eq!(responses.len(), 1, "{label}");
9499 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
9500 let error = parse_error(&responses[0]);
9501 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
9502 assert_eq!(
9503 error["detail"]["field"], "role_versions",
9504 "{label}: {error}"
9505 );
9506 assert!(
9507 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
9508 "{label}: a malformed declaration is terminal"
9509 );
9510 assert!(
9511 module_rx.try_recv().is_err(),
9512 "{label}: the module must never see a bind"
9513 );
9514 }
9515 }
9516
9517 #[tokio::test]
9521 async fn route_role_versions_capability_is_advertised() {
9522 let handler = ControlHandler::new(Arc::new(Registry::default()));
9523 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
9524 let ack = hello_via_sink(
9525 &handler,
9526 &ctx,
9527 &mut rx,
9528 hello_frame("m", PROTOCOL_VERSION, 1),
9529 )
9530 .await;
9531 let ack = parse_ack(&ack);
9532 assert!(
9533 ack.subc_capabilities
9534 .iter()
9535 .any(|c| c == "route-role-versions/v1"),
9536 "{:?}",
9537 ack.subc_capabilities
9538 );
9539
9540 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
9541 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
9542 let reply = handler
9543 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
9544 .await
9545 .unwrap()
9546 .pop()
9547 .unwrap();
9548 let ClientControlResponse::ServerDescribe { capabilities, .. } =
9549 serde_json::from_slice(&reply.body).unwrap()
9550 else {
9551 panic!("not a server.describe reply");
9552 };
9553 assert!(
9554 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
9555 "{capabilities:?}"
9556 );
9557 }
9558
9559 #[tokio::test]
9560 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
9561 let registry = Arc::new(Registry::default());
9562 let forwarding = Arc::new(ForwardingTable::default());
9563 let handler =
9564 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9565 .with_health_probe_timeout(Duration::from_secs(5));
9566 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
9567 hello_via_sink(
9568 &handler,
9569 &module_ctx,
9570 &mut module_rx,
9571 non_routable_hello_frame_with_control_ops(
9572 "mcp",
9573 300,
9574 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9575 ),
9576 )
9577 .await;
9578 assert!(registry
9579 .get_module("mcp")
9580 .unwrap()
9581 .unwrap()
9582 .manifest
9583 .provides
9584 .is_empty());
9585
9586 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
9587 let route_response = handler
9588 .handle_control_frame(
9589 &route_client_ctx,
9590 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
9591 )
9592 .await
9593 .unwrap();
9594 assert_eq!(route_response[0].header.ty, FrameType::Error);
9595 assert_eq!(
9596 parse_error(&route_response[0])["code"],
9597 "target_unavailable"
9598 );
9599 assert!(parse_error(&route_response[0])["message"]
9600 .as_str()
9601 .unwrap()
9602 .contains("does not provide the requested target"));
9603 assert!(module_rx.try_recv().is_err());
9604
9605 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
9606 let health_handler = handler.clone();
9607 let health_task = tokio::spawn(async move {
9608 health_handler
9609 .handle_control_frame(
9610 &health_client_ctx,
9611 supervisor_health_probe_frame(302, "mcp"),
9612 )
9613 .await
9614 .unwrap()
9615 });
9616 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9617 .await
9618 .unwrap()
9619 .unwrap();
9620 assert_eq!(
9621 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
9622 ModuleControlRequest::HealthCheck {}
9623 );
9624 handler
9625 .handle_control_frame(
9626 &module_ctx,
9627 health_response(health_frame.header.corr, HealthStatus::Ok),
9628 )
9629 .await
9630 .unwrap();
9631 let health_response = health_task.await.unwrap();
9632 assert_eq!(health_response[0].header.ty, FrameType::Response);
9633 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
9634 ClientControlResponse::SupervisorHealthProbe {
9635 module_id, status, ..
9636 } => {
9637 assert_eq!(module_id, "mcp");
9638 assert_eq!(status, HealthStatus::Ok);
9639 }
9640 other => panic!("unexpected health response: {other:?}"),
9641 }
9642
9643 forwarding
9648 .cleanup_connection(module_ctx.connection_id)
9649 .unwrap();
9650 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
9651 let cleanup_response = tokio::time::timeout(
9652 Duration::from_millis(200),
9653 handler.handle_control_frame(
9654 &cleanup_probe_ctx,
9655 supervisor_health_probe_frame(303, "mcp"),
9656 ),
9657 )
9658 .await
9659 .expect("probe should fail immediately when the control lane is gone")
9660 .unwrap();
9661 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
9662 assert_eq!(
9663 parse_error(&cleanup_response[0])["code"],
9664 "target_unavailable"
9665 );
9666 assert!(parse_error(&cleanup_response[0])["message"]
9667 .as_str()
9668 .unwrap()
9669 .contains("no module connection"));
9670
9671 handler
9672 .cleanup_connection(module_ctx.connection_id)
9673 .unwrap();
9674 }
9675
9676 #[tokio::test]
9677 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
9678 let registry = Arc::new(Registry::default());
9679 let supervisor_handle = SupervisorHandle::new();
9680 let supervisor =
9681 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
9682 .with_handle(supervisor_handle.clone())
9683 .with_connection_file_path(
9684 std::env::temp_dir()
9685 .join(format!("subc-route-open-warming-{}", std::process::id())),
9686 );
9687 let module = supervisor
9688 .supervise_configured(
9689 ModuleSpec {
9690 module_id: "warming".to_string(),
9691 program: fake_aft_stub_path(),
9692 args: Vec::new(),
9693 env: Vec::new(),
9694 reserved: false,
9695 reserved_prefixes: Vec::new(),
9696 protocol: ModuleProtocol::Subc,
9697 overlap: Default::default(),
9698 },
9699 true,
9700 )
9701 .unwrap();
9702 assert_eq!(module.state().unwrap(), ModuleState::Running);
9703
9704 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9705 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
9706 let response = handler
9707 .handle_control_frame(
9708 &ctx,
9709 route_open_frame(304, "warming", unique_project_root("warming")),
9710 )
9711 .await
9712 .unwrap();
9713 module.stop().await.unwrap();
9714
9715 assert_eq!(response[0].header.ty, FrameType::Error);
9716 let error = parse_error(&response[0]);
9717 assert_eq!(error["code"], "module_warming");
9718 assert!(error["message"]
9719 .as_str()
9720 .unwrap()
9721 .contains("state=running, enabled=true, live=false"));
9722 }
9723
9724 #[test]
9725 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
9726 let handler = ControlHandler::new(Arc::new(Registry::default()));
9727 let capture = EventCapture::default();
9728 let _subscriber =
9729 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9730 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
9731 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
9732 let pending = (0..limit).collect::<Vec<_>>();
9733 let response = handler
9734 .route_open_capacity_refusal(
9735 &ctx,
9736 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
9737 "busy",
9738 pending.len(),
9739 limit,
9740 )
9741 .unwrap();
9742 assert_eq!(parse_error(&response)["code"], "target_unavailable");
9743 let event = capture
9744 .events()
9745 .into_iter()
9746 .find(|event| {
9747 event.target == "control"
9748 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
9749 })
9750 .expect("connection admission refusal event");
9751 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
9752 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
9753 }
9754
9755 #[test]
9756 fn route_open_target_cap_logs_admission_reason_and_capacity() {
9757 let handler = ControlHandler::new(Arc::new(Registry::default()));
9758 let capture = EventCapture::default();
9759 let _subscriber =
9760 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9761 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
9762 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
9763 let guards = (0..limit)
9764 .map(|_| {
9765 handler
9766 .route_bind_concurrency
9767 .try_admit("busy", limit)
9768 .unwrap()
9769 })
9770 .collect::<Vec<_>>();
9771 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
9772 Err(in_flight) => in_flight,
9773 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
9774 };
9775 let response = handler
9776 .route_open_target_capacity_refusal(
9777 &ctx,
9778 &route_open_frame(397, "busy", unique_project_root("target-cap")),
9779 "busy",
9780 in_flight,
9781 )
9782 .unwrap();
9783 assert_eq!(parse_error(&response)["code"], "target_unavailable");
9784 let event = capture
9785 .events()
9786 .into_iter()
9787 .find(|event| {
9788 event.target == "control"
9789 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
9790 })
9791 .expect("target admission refusal event");
9792 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
9793 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
9794 drop(guards);
9795 }
9796
9797 #[tokio::test]
9802 async fn route_open_refusal_names_the_check_that_refused() {
9803 let handler = ControlHandler::new(Arc::new(Registry::default()));
9804 let capture = EventCapture::default();
9805 let _subscriber =
9806 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9807 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
9808 let response = handler
9809 .handle_control_frame(
9810 &ctx,
9811 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
9812 )
9813 .await
9814 .unwrap();
9815
9816 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
9817 let event = capture
9818 .events()
9819 .into_iter()
9820 .find(|event| {
9821 event.target == "control"
9822 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
9823 })
9824 .expect("route.open refusal event");
9825 assert_eq!(
9826 event.fields.get("reason"),
9827 Some(&"\"not_registered\"".to_string())
9828 );
9829 }
9830
9831 #[tokio::test]
9832 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
9833 let registry = Arc::new(Registry::default());
9834 let supervisor_handle = SupervisorHandle::new();
9835 let supervisor =
9836 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
9837 .with_handle(supervisor_handle.clone())
9838 .with_connection_file_path(std::env::temp_dir().join(format!(
9839 "subc-route-open-refusal-info-{}",
9840 std::process::id()
9841 )));
9842 let module = supervisor
9843 .supervise_configured(
9844 ModuleSpec {
9845 module_id: "warming".to_string(),
9846 program: fake_aft_stub_path(),
9847 args: Vec::new(),
9848 env: Vec::new(),
9849 reserved: false,
9850 reserved_prefixes: Vec::new(),
9851 protocol: ModuleProtocol::Subc,
9852 overlap: Default::default(),
9853 },
9854 true,
9855 )
9856 .unwrap();
9857 assert_eq!(module.state().unwrap(), ModuleState::Running);
9858
9859 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9860 assert!(handler
9861 .counters()
9862 .snapshot()
9863 .get("route_open_refused_by_code")
9864 .is_none());
9865 let capture = EventCapture::default();
9866 let _subscriber =
9867 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9868 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
9869 let response = handler
9870 .handle_control_frame(
9871 &ctx,
9872 route_open_frame(394, "warming", unique_project_root("refusal-info")),
9873 )
9874 .await
9875 .unwrap();
9876 module.stop().await.unwrap();
9877
9878 assert_eq!(parse_error(&response[0])["code"], "module_warming");
9879 let event = capture
9880 .events()
9881 .into_iter()
9882 .find(|event| {
9883 event.target == "control"
9884 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
9885 })
9886 .expect("route.open refusal event");
9887 assert_eq!(
9888 event.fields.get("module_id"),
9889 Some(&"\"warming\"".to_string())
9890 );
9891 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
9892 assert_eq!(
9893 event.fields.get("reason"),
9894 Some(&"\"supervised_not_registered\"".to_string())
9895 );
9896 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
9897 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
9898 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
9899 assert_eq!(
9900 handler.counters().snapshot()["route_open_refused_by_code"],
9901 json!({ "module_warming": 1 })
9902 );
9903 }
9904
9905 const OUTAGE_START: &str = "route.open refusing module: not serving";
9906 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
9907
9908 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
9909 capture
9910 .events()
9911 .into_iter()
9912 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
9913 .collect()
9914 }
9915
9916 fn supervise_stub(
9917 registry: &Arc<Registry>,
9918 module_id: &str,
9919 enabled: bool,
9920 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
9921 let supervisor_handle = SupervisorHandle::new();
9922 let supervisor =
9923 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
9924 .with_handle(supervisor_handle.clone())
9925 .with_connection_file_path(std::env::temp_dir().join(format!(
9926 "subc-route-outage-{module_id}-{}",
9927 std::process::id()
9928 )));
9929 let module = supervisor
9930 .supervise_configured(
9931 ModuleSpec {
9932 module_id: module_id.to_string(),
9933 program: fake_aft_stub_path(),
9934 args: Vec::new(),
9935 env: Vec::new(),
9936 reserved: false,
9937 reserved_prefixes: Vec::new(),
9938 protocol: ModuleProtocol::Subc,
9939 overlap: Default::default(),
9940 },
9941 enabled,
9942 )
9943 .unwrap();
9944 (supervisor_handle, module)
9945 }
9946
9947 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
9948 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
9949 module_id: module_id.to_string(),
9950 drain_timeout_ms: Some(50),
9951 })
9952 .unwrap();
9953 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9954 }
9955
9956 #[test]
9960 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
9961 let registry = Arc::new(Registry::default());
9962 let forwarding = Arc::new(ForwardingTable::default());
9963 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9964 let second = ControlHandler::with_forwarding(registry, forwarding);
9965 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
9966 }
9967
9968 #[tokio::test(flavor = "current_thread")]
9972 async fn route_open_unknown_module_refusals_add_no_outage_state() {
9973 let handler = ControlHandler::new(Arc::new(Registry::default()));
9974 let capture = EventCapture::default();
9975 let _subscriber =
9976 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9977 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
9978 for corr in 0..8 {
9979 let response = handler
9980 .handle_control_frame(
9981 &ctx,
9982 route_open_frame(
9983 380 + corr,
9984 &format!("nobody-{corr}"),
9985 unique_project_root("outage-unknown"),
9986 ),
9987 )
9988 .await
9989 .unwrap();
9990 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
9991 }
9992
9993 assert_eq!(handler.route_outages.tracked_module_count(), 0);
9994 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
9995 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
9996 }
9997
9998 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
10003 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
10004 let registry = Arc::new(Registry::default());
10005 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
10006 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10007 let capture = EventCapture::default();
10008 let _subscriber =
10009 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10010 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
10011 handler.route_outages.record_accepted("outage-restart");
10014
10015 let response = handler
10016 .handle_control_frame(
10017 &ctx,
10018 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
10019 )
10020 .await
10021 .unwrap();
10022 assert_eq!(response[0].header.ty, FrameType::Error);
10023 let starts = outage_lines(&capture, OUTAGE_START);
10024 assert_eq!(starts.len(), 1, "{starts:?}");
10025 assert_eq!(starts[0].level, tracing::Level::WARN);
10026 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
10027 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
10028 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
10029 handler.route_outages.record_accepted("outage-restart");
10030 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
10031
10032 let restart = handler
10033 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
10034 .await
10035 .unwrap();
10036 assert_eq!(
10037 restart[0].header.ty,
10038 FrameType::Response,
10039 "{:?}",
10040 parse_error(&restart[0])
10041 );
10042 handler
10043 .handle_control_frame(
10044 &ctx,
10045 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
10046 )
10047 .await
10048 .unwrap();
10049 module.stop().await.unwrap();
10050
10051 let starts = outage_lines(&capture, OUTAGE_START);
10052 assert_eq!(starts.len(), 2, "{starts:?}");
10053 assert_eq!(starts[1].level, tracing::Level::INFO);
10054 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
10055 }
10056
10057 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
10061 async fn failed_operator_restart_leaves_no_operator_mark() {
10062 let registry = Arc::new(Registry::default());
10063 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
10064 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10065 let capture = EventCapture::default();
10066 let _subscriber =
10067 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10068 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
10069 handler.route_outages.record_accepted("outage-disabled");
10070
10071 let restart = handler
10072 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
10073 .await
10074 .unwrap();
10075 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
10076 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
10077
10078 handler
10079 .handle_control_frame(
10080 &ctx,
10081 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
10082 )
10083 .await
10084 .unwrap();
10085 let starts = outage_lines(&capture, OUTAGE_START);
10086 assert_eq!(starts.len(), 1, "{starts:?}");
10087 assert_eq!(starts[0].level, tracing::Level::WARN);
10088 }
10089
10090 #[tokio::test(flavor = "current_thread")]
10091 async fn route_open_unknown_module_escapes_target_module_id() {
10092 let handler = ControlHandler::new(Arc::new(Registry::default()));
10093 let capture = EventCapture::default();
10094 let _subscriber =
10095 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10096 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
10097 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
10098 let response = handler
10099 .handle_control_frame(
10100 &ctx,
10101 route_open_frame(
10102 395,
10103 hostile_module_id,
10104 unique_project_root("hostile-target-module-id"),
10105 ),
10106 )
10107 .await
10108 .unwrap();
10109
10110 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
10111 let event = capture
10112 .events()
10113 .into_iter()
10114 .find(|event| {
10115 event.target == "control"
10116 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
10117 })
10118 .expect("route.open unknown-module refusal event");
10119 let logged = event.fields.get("module_id").expect("module_id field");
10120 assert!(!logged.bytes().any(|byte| byte < 0x20));
10121 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
10122 }
10123
10124 #[tokio::test(flavor = "current_thread")]
10125 async fn route_open_module_rejection_uses_daemon_counter_key() {
10126 let registry = Arc::new(Registry::default());
10127 let forwarding = Arc::new(ForwardingTable::default());
10128 let handler =
10129 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10130 let module_connection = ConnectionId::new(95);
10131 let (module_ctx, mut module_rx) = route_ctx(module_connection);
10132 hello_via_sink(
10133 &handler,
10134 &module_ctx,
10135 &mut module_rx,
10136 hello_frame("aft", PROTOCOL_VERSION, 395),
10137 )
10138 .await;
10139
10140 let client_connection = ConnectionId::new(96);
10141 let (client_ctx, _client_rx) = route_ctx(client_connection);
10142 let capture = EventCapture::default();
10143 let _subscriber =
10144 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10145 let (route_task, bind) = relay_route_open(
10146 &handler,
10147 client_connection,
10148 &client_ctx.egress,
10149 &mut module_rx,
10150 396,
10151 "aft",
10152 "hostile-module-code",
10153 )
10154 .await;
10155 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
10156 let rejection = Frame::build(
10157 FrameType::Error,
10158 control_flags(),
10159 0,
10160 0,
10161 bind.header.corr,
10162 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
10163 )
10164 .unwrap();
10165 handler
10166 .handle_control_frame(&module_ctx, rejection)
10167 .await
10168 .unwrap();
10169
10170 let response = route_task.await.unwrap();
10171 assert_eq!(parse_error(&response[0])["code"], hostile_code);
10172 let counters = handler.counters().snapshot();
10173 assert_eq!(
10174 counters["route_open_refused_by_code"],
10175 json!({ "module_rejected": 1 })
10176 );
10177 assert!(counters["route_open_refused_by_code"]
10178 .get(hostile_code)
10179 .is_none());
10180
10181 let event = capture
10182 .events()
10183 .into_iter()
10184 .find(|event| {
10185 event.target == "control"
10186 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
10187 })
10188 .expect("route.open module-rejection refusal event");
10189 let logged = event.fields.get("module_code").expect("module_code field");
10190 assert!(!logged.bytes().any(|byte| byte < 0x20));
10191 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
10192 }
10193
10194 #[tokio::test]
10195 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
10196 let registry = Arc::new(Registry::default());
10197 let supervisor_handle = SupervisorHandle::new();
10198 let missing_program = std::env::temp_dir().join(format!(
10199 "subc-route-open-missing-program-{}",
10200 std::process::id()
10201 ));
10202 let supervisor =
10203 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
10204 .with_handle(supervisor_handle.clone());
10205 let module = supervisor
10206 .supervise_configured(
10207 ModuleSpec {
10208 module_id: "failed".to_string(),
10209 program: missing_program,
10210 args: Vec::new(),
10211 env: Vec::new(),
10212 reserved: false,
10213 reserved_prefixes: Vec::new(),
10214 protocol: ModuleProtocol::Subc,
10215 overlap: Default::default(),
10216 },
10217 true,
10218 )
10219 .unwrap();
10220 assert_eq!(module.state().unwrap(), ModuleState::Failed);
10221
10222 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10223 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
10224 let response = handler
10225 .handle_control_frame(
10226 &ctx,
10227 route_open_frame(305, "failed", unique_project_root("failed")),
10228 )
10229 .await
10230 .unwrap();
10231
10232 assert_eq!(response[0].header.ty, FrameType::Error);
10233 let error = parse_error(&response[0]);
10234 assert_eq!(error["code"], "target_unavailable");
10235 assert!(error["message"]
10236 .as_str()
10237 .unwrap()
10238 .contains("state=failed, enabled=true, live=false"));
10239 }
10240
10241 #[tokio::test]
10242 async fn route_open_role_mismatch_remains_target_unavailable() {
10243 let registry = Arc::new(Registry::default());
10244 let handler = ControlHandler::new(Arc::clone(®istry));
10245 handler
10246 .handle_control(
10247 ConnectionId::new(41),
10248 non_routable_hello_frame_with_control_ops("health-only", 306, None),
10249 )
10250 .unwrap();
10251
10252 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
10253 let response = handler
10254 .handle_control_frame(
10255 &ctx,
10256 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
10257 )
10258 .await
10259 .unwrap();
10260
10261 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10262 assert!(parse_error(&response[0])["message"]
10263 .as_str()
10264 .unwrap()
10265 .contains("does not provide the requested target"));
10266 }
10267
10268 #[tokio::test]
10269 async fn route_open_inactive_registration_remains_target_unavailable() {
10270 let registry = Arc::new(Registry::default());
10271 let handler = ControlHandler::new(Arc::clone(®istry));
10272 handler
10273 .handle_control(
10274 ConnectionId::new(43),
10275 hello_frame("inactive", PROTOCOL_VERSION, 308),
10276 )
10277 .unwrap();
10278 assert!(registry
10279 .set_module_state_for_test("inactive", ChannelState::Closed)
10280 .unwrap());
10281
10282 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
10283 let response = handler
10284 .handle_control_frame(
10285 &ctx,
10286 route_open_frame(309, "inactive", unique_project_root("inactive")),
10287 )
10288 .await
10289 .unwrap();
10290
10291 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10292 assert!(parse_error(&response[0])["message"]
10293 .as_str()
10294 .unwrap()
10295 .contains("is not active"));
10296 }
10297
10298 #[tokio::test]
10299 async fn late_health_reply_is_recorded_through_the_module_response_path() {
10300 let registry = Arc::new(Registry::default());
10301 let forwarding = Arc::new(ForwardingTable::default());
10302 let supervisor_handle = SupervisorHandle::new();
10303 let supervisor =
10304 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
10305 .with_forwarding(Arc::clone(&forwarding))
10306 .with_handle(supervisor_handle.clone());
10307 let module = supervisor
10308 .supervise_configured(
10309 crate::ModuleSpec {
10310 module_id: "late-health-response".to_string(),
10311 program: PathBuf::from("disabled-module"),
10312 args: Vec::new(),
10313 env: Vec::new(),
10314 reserved: false,
10315 reserved_prefixes: Vec::new(),
10316 protocol: ModuleProtocol::Subc,
10317 overlap: Default::default(),
10318 },
10319 false,
10320 )
10321 .unwrap();
10322 let handler =
10323 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10324 .with_supervisor(supervisor_handle);
10325 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
10326 handler
10327 .handle_control_frame(
10328 &module_ctx,
10329 hello_frame_with_control_ops(
10330 "late-health-response",
10331 PROTOCOL_VERSION,
10332 7,
10333 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10334 ),
10335 )
10336 .await
10337 .unwrap();
10338 let probe_started_at = Instant::now() - Duration::from_millis(80);
10339 let pending = forwarding
10340 .begin_health_probe_rpc_for(
10341 "late-health-response",
10342 MODULE_CONTROL_OP_HEALTH_CHECK,
10343 probe_started_at,
10344 Instant::now() - Duration::from_millis(1),
10345 )
10346 .unwrap();
10347 assert!(forwarding
10348 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
10349 .unwrap());
10350
10351 let responses = handler
10352 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
10353 .await
10354 .unwrap();
10355
10356 assert!(responses.is_empty());
10357 let health = module.status().unwrap().health;
10358 assert_eq!(health.late_answer_count, 1);
10359 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
10360 }
10361
10362 #[tokio::test]
10363 async fn health_probe_timeout_and_module_death_are_typed() {
10364 let registry = Arc::new(Registry::default());
10365 let forwarding = Arc::new(ForwardingTable::default());
10366 let handler =
10367 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10368 .with_health_probe_timeout(Duration::from_millis(50));
10369 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
10370 hello_via_sink(
10371 &handler,
10372 &module_ctx,
10373 &mut module_rx,
10374 hello_frame_with_control_ops(
10375 "aft",
10376 PROTOCOL_VERSION,
10377 7,
10378 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10379 ),
10380 )
10381 .await;
10382
10383 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10384 let responses = handler
10385 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
10386 .await
10387 .unwrap();
10388 assert_eq!(responses[0].header.ty, FrameType::Error);
10389 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
10390 let _ = module_rx.try_recv();
10391
10392 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
10393 let health_handler = handler.clone();
10394 let death_task = tokio::spawn(async move {
10395 health_handler
10396 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
10397 .await
10398 .unwrap()
10399 });
10400 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10401 .await
10402 .unwrap()
10403 .unwrap();
10404 handler
10405 .cleanup_connection(module_ctx.connection_id)
10406 .unwrap();
10407 let responses = death_task.await.unwrap();
10408 assert_eq!(responses[0].header.ty, FrameType::Error);
10409 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
10410 }
10411
10412 #[test]
10413 fn hello_requires_exact_protocol_version() {
10414 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
10415 let registry = Arc::new(Registry::default());
10416 let handler = ControlHandler::new(Arc::clone(®istry));
10417 let responses = handler
10418 .handle_control(
10419 ConnectionId::new(connection),
10420 hello_frame("aft", offered, 9),
10421 )
10422 .unwrap();
10423
10424 assert_eq!(responses.len(), 1);
10425 assert_eq!(responses[0].header.ty, FrameType::Error);
10426 let error = parse_error(&responses[0]);
10427 assert_eq!(error["code"], "version_unsupported");
10428 assert!(registry.get_module("aft").unwrap().is_none());
10429 assert_eq!(registry.active_registration_count().unwrap(), 0);
10430 }
10431 }
10432
10433 #[test]
10434 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
10435 let registry = Arc::new(Registry::default());
10436 let forwarding = Arc::new(ForwardingTable::default());
10437 let handler =
10438 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10439 let module_connection = ConnectionId::new(301);
10440 let registration = registry
10441 .register_with_control_ops(
10442 manifest("aft-push", PROTOCOL_VERSION),
10443 PROTOCOL_VERSION,
10444 module_connection,
10445 module_baseline_control_ops(),
10446 )
10447 .unwrap();
10448 let (module_tx, _module_rx) = mpsc::channel(8);
10449 let endpoint = forwarding
10450 .register_module_connection(
10451 module_connection,
10452 "aft-push".to_string(),
10453 PROTOCOL_VERSION,
10454 manifest_concurrency(®istration.manifest),
10455 FrameSink::new(module_tx),
10456 )
10457 .unwrap();
10458
10459 let unknown = Frame::build(
10461 FrameType::Push,
10462 control_flags(),
10463 0,
10464 0,
10465 5,
10466 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
10467 )
10468 .unwrap();
10469 let out = handler.handle_status_update(endpoint, unknown).unwrap();
10470 assert!(
10471 out.is_empty(),
10472 "unknown push op must be ignored, got {out:?}"
10473 );
10474
10475 let malformed = Frame::build(
10477 FrameType::Push,
10478 control_flags(),
10479 0,
10480 0,
10481 6,
10482 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
10483 )
10484 .unwrap();
10485 let out = handler.handle_status_update(endpoint, malformed).unwrap();
10486 assert_eq!(out.len(), 1);
10487 assert_eq!(out[0].header.ty, FrameType::Error);
10488 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
10489 }
10490
10491 #[test]
10492 fn hello_rejected_when_connection_already_owns_client_routes() {
10493 let registry = Arc::new(Registry::default());
10494 let forwarding = Arc::new(ForwardingTable::default());
10495 let handler =
10496 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10497 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
10499 let client_connection = ConnectionId::new(202);
10500
10501 let responses = handler
10504 .handle_control(
10505 client_connection,
10506 hello_frame("aft-second", PROTOCOL_VERSION, 9),
10507 )
10508 .unwrap();
10509 assert_eq!(responses[0].header.ty, FrameType::Error);
10510 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
10511 assert!(registry.get_module("aft-second").unwrap().is_none());
10512 }
10513
10514 #[tokio::test]
10515 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
10516 let registry = Arc::new(Registry::default());
10517 let forwarding = Arc::new(ForwardingTable::default());
10518 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
10519 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
10520 hello_via_sink(
10521 &handler,
10522 &module_ctx,
10523 &mut module_rx,
10524 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
10525 )
10526 .await;
10527 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
10528 let pending = forwarding
10529 .begin_route_bind_relay_for_test(
10530 client_ctx.connection_id,
10531 client_ctx.egress.clone(),
10532 2,
10533 "alpha",
10534 )
10535 .unwrap();
10536 forwarding
10537 .complete_pending_relay(
10538 module_ctx.connection_id,
10539 pending.corr,
10540 RouteBindRelayOutcome::Accepted,
10541 )
10542 .unwrap();
10543 client_rx.try_recv().unwrap();
10544 for module_id in ["beta", "alpha"] {
10545 let replies = handler
10546 .handle_control_frame(
10547 &module_ctx,
10548 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
10549 )
10550 .await
10551 .unwrap();
10552 assert_eq!(replies.len(), 1, "second HELLO must be refused");
10553 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
10554 }
10555 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
10556 assert!(registry.get_module("beta").unwrap().is_none());
10557 assert!(matches!(
10558 forwarding
10559 .lookup_data_route(
10560 client_ctx.connection_id,
10561 pending.client_channel,
10562 pending.client_epoch,
10563 )
10564 .unwrap(),
10565 DataRoute::Client(DataRouteState::Bound(_))
10566 ));
10567 assert!(handler
10568 .hello_launch_nonces
10569 .lock()
10570 .unwrap()
10571 .presented(module_ctx.connection_id, Some("alpha-nonce")));
10572 assert!(module_rx.try_recv().is_err());
10573 }
10574
10575 #[test]
10576 fn reserved_module_hello_requires_matching_launch_nonce() {
10577 let registry = Arc::new(Registry::default());
10578 let supervisor = SupervisorHandle::new();
10579 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
10582 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
10583
10584 let no_nonce = handler
10586 .handle_control(
10587 ConnectionId::new(1),
10588 hello_frame("vault", PROTOCOL_VERSION, 1),
10589 )
10590 .unwrap();
10591 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
10592 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
10593 assert!(registry.get_module("vault").unwrap().is_none());
10594
10595 let wrong = handler
10597 .handle_control(
10598 ConnectionId::new(2),
10599 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
10600 )
10601 .unwrap();
10602 assert_eq!(wrong[0].header.ty, FrameType::Error);
10603 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
10604 assert!(registry.get_module("vault").unwrap().is_none());
10605
10606 let ok = handler
10608 .handle_control(
10609 ConnectionId::new(3),
10610 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
10611 )
10612 .unwrap();
10613 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
10614 assert!(registry.get_module("vault").unwrap().is_some());
10615 }
10616
10617 #[test]
10618 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
10619 let registry = Arc::new(Registry::default());
10620 let supervisor = SupervisorHandle::new();
10621 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
10622 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
10623 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
10624
10625 let squat = handler
10626 .handle_control(
10627 ConnectionId::new(1),
10628 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
10629 )
10630 .unwrap();
10631 assert_eq!(squat[0].header.ty, FrameType::Error);
10632 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
10633 assert!(parse_error(&squat[0])["message"]
10634 .as_str()
10635 .unwrap()
10636 .contains("fed:"));
10637
10638 let accepted_peer = handler
10639 .handle_control(
10640 ConnectionId::new(2),
10641 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
10642 )
10643 .unwrap();
10644 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
10645
10646 let accepted_short = handler
10647 .handle_control(
10648 ConnectionId::new(3),
10649 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
10650 )
10651 .unwrap();
10652 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
10653
10654 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
10655 let response = handler
10656 .handle_control(
10657 ConnectionId::new(conn),
10658 hello_frame(module_id, PROTOCOL_VERSION, conn),
10659 )
10660 .unwrap();
10661 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
10662 }
10663 }
10664
10665 #[test]
10666 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
10667 let registry = Arc::new(Registry::default());
10668 let supervisor = SupervisorHandle::new();
10669 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
10670 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
10671 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
10672 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
10673
10674 let owner_nonce = handler
10675 .handle_control(
10676 ConnectionId::new(1),
10677 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
10678 )
10679 .unwrap();
10680 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
10681 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
10682 assert!(registry.get_module("fed:special").unwrap().is_none());
10683
10684 let exact_nonce = handler
10685 .handle_control(
10686 ConnectionId::new(2),
10687 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
10688 )
10689 .unwrap();
10690 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
10691 assert!(registry.get_module("fed:special").unwrap().is_some());
10692 }
10693
10694 #[test]
10695 fn non_reserved_module_ignores_launch_nonce() {
10696 let registry = Arc::new(Registry::default());
10697 let handler = ControlHandler::new(Arc::clone(®istry));
10700 let no_nonce = handler
10701 .handle_control(
10702 ConnectionId::new(1),
10703 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
10704 )
10705 .unwrap();
10706 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
10707 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
10708
10709 let echoed_nonce = handler
10710 .handle_control(
10711 ConnectionId::new(2),
10712 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
10713 )
10714 .unwrap();
10715 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
10716 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
10717 }
10718
10719 #[test]
10720 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
10721 let handler = ControlHandler::default();
10722 let conn = ConnectionId::new(1);
10723 let malformed = Frame::build(
10724 FrameType::Hello,
10725 control_flags(),
10726 0,
10727 0,
10728 3,
10729 b"{not json".to_vec(),
10730 )
10731 .unwrap();
10732
10733 let error = handler.handle_control(conn, malformed).unwrap();
10734 assert_eq!(error[0].header.ty, FrameType::Error);
10735 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
10736
10737 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
10738 let pong = handler.handle_control(conn, ping).unwrap();
10739 assert_eq!(pong[0].header.ty, FrameType::Pong);
10740 assert_eq!(pong[0].header.corr, 4);
10741 }
10742
10743 #[test]
10744 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
10745 let registry = Arc::new(Registry::default());
10746 let handler = ControlHandler::new(Arc::clone(®istry));
10747
10748 handler
10749 .handle_control(
10750 ConnectionId::new(1),
10751 hello_frame("aft", PROTOCOL_VERSION, 1),
10752 )
10753 .unwrap();
10754 let duplicate = handler
10755 .handle_control(
10756 ConnectionId::new(2),
10757 hello_frame("aft", PROTOCOL_VERSION, 2),
10758 )
10759 .unwrap();
10760
10761 assert_eq!(duplicate[0].header.ty, FrameType::Error);
10762 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
10763 let registration = registry.get_module("aft").unwrap().unwrap();
10764 assert_eq!(registration.connection_id, ConnectionId::new(1));
10765 }
10766
10767 #[test]
10768 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
10769 let registry = Arc::new(Registry::default());
10770 let forwarding = Arc::new(ForwardingTable::default());
10771 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
10772 let handler =
10773 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10774 .with_process_liveness(process_liveness);
10775 let (ctx, route_channel, route_epoch) =
10776 bind_liveness_route(®istry, &forwarding, "aft-dead");
10777 let responses = handler
10778 .handle_route_poll(
10779 &ctx,
10780 route_poll_frame(41, PollKind::Liveness, route_channel),
10781 route_channel,
10782 route_epoch,
10783 PollKind::Liveness,
10784 )
10785 .unwrap();
10786
10787 assert_eq!(responses.len(), 1);
10788 assert_eq!(responses[0].header.ty, FrameType::Response);
10789 assert_route_poll_liveness(&responses[0], false);
10790 }
10791
10792 #[test]
10793 fn liveness_poll_without_process_source_uses_bound_route() {
10794 let registry = Arc::new(Registry::default());
10795 let forwarding = Arc::new(ForwardingTable::default());
10796 let handler =
10797 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10798 let (ctx, route_channel, route_epoch) =
10799 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
10800 let responses = handler
10801 .handle_route_poll(
10802 &ctx,
10803 route_poll_frame(42, PollKind::Liveness, route_channel),
10804 route_channel,
10805 route_epoch,
10806 PollKind::Liveness,
10807 )
10808 .unwrap();
10809
10810 assert_route_poll_liveness(&responses[0], true);
10811 }
10812
10813 #[test]
10814 fn liveness_poll_untracked_process_source_uses_bound_route() {
10815 let registry = Arc::new(Registry::default());
10816 let forwarding = Arc::new(ForwardingTable::default());
10817 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
10818 let handler =
10819 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10820 .with_process_liveness(process_liveness);
10821 let (ctx, route_channel, route_epoch) =
10822 bind_liveness_route(®istry, &forwarding, "aft-untracked");
10823 let responses = handler
10824 .handle_route_poll(
10825 &ctx,
10826 route_poll_frame(43, PollKind::Liveness, route_channel),
10827 route_channel,
10828 route_epoch,
10829 PollKind::Liveness,
10830 )
10831 .unwrap();
10832
10833 assert_route_poll_liveness(&responses[0], true);
10834 }
10835
10836 #[tokio::test]
10837 async fn unknown_op_returns_unknown_control_op() {
10838 let handler = ControlHandler::default();
10839 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
10840 let request = Frame::build(
10841 FrameType::Request,
10842 control_flags(),
10843 0,
10844 0,
10845 55,
10846 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
10847 )
10848 .unwrap();
10849
10850 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
10851
10852 assert_eq!(response.len(), 1);
10853 assert_eq!(response[0].header.ty, FrameType::Error);
10854 assert_eq!(response[0].header.corr, 55);
10855 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
10856 }
10857
10858 #[tokio::test]
10859 async fn supervisor_provenance_rejects_unknown_exact_module() {
10860 let handler = ControlHandler::default();
10861 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
10862 let request = Frame::build(
10863 FrameType::Request,
10864 control_flags(),
10865 0,
10866 0,
10867 57,
10868 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
10869 )
10870 .unwrap();
10871
10872 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
10873
10874 assert_eq!(response.len(), 1);
10875 assert_eq!(response[0].header.ty, FrameType::Error);
10876 assert_eq!(response[0].header.corr, 57);
10877 let error = parse_error(&response[0]);
10878 assert_eq!(error["code"], "unknown_module");
10879 assert_eq!(error["message"], "module_id 'missing' is not supervised");
10880 }
10881
10882 #[test]
10883 fn provenance_probe_override_keeps_handler_tests_deterministic() {
10884 let expected = subc_control::RunningImageAgreement::Unavailable {
10885 reason: subc_control::RunningImageUnavailableReason::HashFailed,
10886 };
10887 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
10888 assert_eq!(handler.provenance_probe_override, Some(expected));
10889 }
10890
10891 #[test]
10892 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
10893 let verdict = reload_verdict(
10894 std::path::Path::new("/bin/new"),
10895 Some(std::path::Path::new("/bin/old")),
10896 subc_control::RunningImageAgreement::Unavailable {
10897 reason: subc_control::RunningImageUnavailableReason::HashFailed,
10898 },
10899 );
10900 assert!(matches!(
10901 verdict.path,
10902 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
10903 if configured == std::path::Path::new("/bin/new")
10904 && spawned_from == std::path::Path::new("/bin/old")
10905 ));
10906 }
10907
10908 #[test]
10909 fn reload_verdict_detects_replaced_image_at_same_path() {
10910 let image = subc_control::RunningImageAgreement::Mismatch {
10911 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
10912 digest: "old".into(),
10913 },
10914 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
10915 digest: "new".into(),
10916 },
10917 };
10918 let verdict = reload_verdict(
10919 std::path::Path::new("/bin/same"),
10920 Some(std::path::Path::new("/bin/same")),
10921 image.clone(),
10922 );
10923 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
10924 assert_eq!(verdict.image, image);
10925 }
10926
10927 #[test]
10928 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
10929 let image = subc_control::RunningImageAgreement::Unavailable {
10930 reason: subc_control::RunningImageUnavailableReason::NotRunning,
10931 };
10932 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
10933 assert_eq!(
10934 verdict.path,
10935 subc_control::ReloadPathAgreement::Unavailable {
10936 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
10937 }
10938 );
10939 assert_eq!(verdict.image, image);
10940
10941 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
10942 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
10943 };
10944 let verdict = reload_verdict(
10945 std::path::Path::new("/bin/same"),
10946 Some(std::path::Path::new("/bin/same")),
10947 unconfirmed.clone(),
10948 );
10949 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
10950 assert_eq!(verdict.image, unconfirmed);
10951 }
10952
10953 #[test]
10954 fn reload_verdict_preserves_each_image_unavailability_reason() {
10955 use subc_control::RunningImageUnavailableReason as Reason;
10956
10957 for reason in [
10958 Reason::NotRunning,
10959 Reason::UnsupportedPlatform,
10960 Reason::RunningExecutableUnreadable,
10961 Reason::SpawnedPathUnreadable,
10962 Reason::HashFailed,
10963 Reason::ProcessIdentityUnconfirmed,
10964 Reason::Unknown("future_probe_reason".to_string()),
10965 ] {
10966 let image = subc_control::RunningImageAgreement::Unavailable {
10967 reason: reason.clone(),
10968 };
10969 let verdict = reload_verdict(
10970 std::path::Path::new("/bin/same"),
10971 Some(std::path::Path::new("/bin/same")),
10972 image.clone(),
10973 );
10974 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
10975 assert_eq!(verdict.image, image, "{reason:?}");
10976 }
10977 }
10978
10979 #[tokio::test]
10980 async fn malformed_control_bodies_return_invalid_control_body() {
10981 let handler = ControlHandler::default();
10982 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
10983
10984 for (corr, body) in [
10985 (56, br#"{"route_channel":1}"#.as_slice()),
10986 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
10987 (
10988 58,
10989 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
10990 ),
10991 ] {
10992 let request = Frame::build(
10993 FrameType::Request,
10994 control_flags(),
10995 0,
10996 0,
10997 corr,
10998 body.to_vec(),
10999 )
11000 .unwrap();
11001 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
11002
11003 assert_eq!(response.len(), 1);
11004 assert_eq!(response[0].header.ty, FrameType::Error);
11005 assert_eq!(response[0].header.corr, corr);
11006 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
11007 }
11008 }
11009
11010 #[tokio::test]
11011 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
11012 let registry = Arc::new(Registry::default());
11013 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
11014 let router = Router::with_control_handler(Arc::clone(&control));
11015 let connection = router.begin_connection();
11016 let (ctx, mut rx) = route_ctx(connection.id());
11017
11018 router
11019 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
11020 .await
11021 .unwrap();
11022 let response = rx.recv().await.unwrap();
11023 let ack = parse_ack(&response);
11024 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
11025 let channel = 1;
11026
11027 let goodbye =
11028 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
11029 router.route_for_connection(&ctx, goodbye).await.unwrap();
11030 assert!(rx.try_recv().is_err());
11031 assert!(registry.get_module("aft").unwrap().is_none());
11032
11033 router
11034 .route_for_connection(&ctx, channel_request(channel, 13))
11035 .await
11036 .unwrap();
11037 let error_frame = rx.recv().await.unwrap();
11038 assert_eq!(error_frame.header.ty, FrameType::Error);
11039 assert_eq!(error_frame.header.channel, channel);
11040 }
11041
11042 #[tokio::test]
11043 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
11044 let registry = Arc::new(Registry::default());
11045 let handler = ControlHandler::new(registry).with_capability_config(
11046 [("prov".to_string(), true), ("cons".to_string(), true)],
11047 BTreeMap::new(),
11048 );
11049 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
11050 register_capability_manifest(
11051 &handler,
11052 &provider_ctx,
11053 &mut provider_rx,
11054 capability_manifest("prov", &["thing/v1"], &[]),
11055 1,
11056 )
11057 .await;
11058 let mut consumer = capability_manifest("cons", &[], &[]);
11059 consumer.capabilities.as_mut().unwrap().requires.push(
11060 subc_protocol::manifest::CapabilityRequirement {
11061 capability: "thing/v1".to_string(),
11062 need: subc_protocol::manifest::CapabilityNeed::Required,
11063 },
11064 );
11065 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
11066 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
11067 assert_eq!(
11068 handler.capability_evaluator.verdict("cons", "thing/v1"),
11069 Some(CapabilityVerdict::Provided)
11070 );
11071 let (mut client_rx, _) = open_route_for_capability_test(
11072 &handler,
11073 &provider_ctx,
11074 &mut provider_rx,
11075 703,
11076 3,
11077 "prov",
11078 None,
11079 )
11080 .await;
11081 let goodbye =
11082 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
11083 handler
11084 .handle_control_frame(&provider_ctx, goodbye)
11085 .await
11086 .unwrap();
11087 assert_eq!(
11088 handler.capability_evaluator.verdict("cons", "thing/v1"),
11089 Some(CapabilityVerdict::NeverProvided)
11090 );
11091 let closed = client_rx
11092 .try_recv()
11093 .expect("GOODBYE pushes route.closed before route GOODBYE");
11094 assert!(
11095 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
11096 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
11097 );
11098 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
11099 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
11100 }
11101
11102 #[tokio::test]
11103 async fn dropping_router_connection_releases_registration() {
11104 let registry = Arc::new(Registry::default());
11105 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
11106 let router = Router::with_control_handler(control);
11107 let connection = router.begin_connection();
11108 let (ctx, mut rx) = route_ctx(connection.id());
11109
11110 router
11111 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
11112 .await
11113 .unwrap();
11114 let response = rx.recv().await.unwrap();
11115 let ack = parse_ack(&response);
11116 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
11117 assert!(registry.get_module("aft").unwrap().is_some());
11118
11119 drop(connection);
11120
11121 assert!(registry.get_module("aft").unwrap().is_none());
11122 assert_eq!(registry.active_registration_count().unwrap(), 0);
11123 }
11124
11125 fn capability_manifest(
11126 module_id: &str,
11127 provides: &[&str],
11128 must_never_reach: &[&str],
11129 ) -> ModuleManifest {
11130 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
11131 manifest.capabilities = Some(CapabilityDeclarations {
11132 provides: provides
11133 .iter()
11134 .map(|capability| (*capability).to_string())
11135 .collect(),
11136 requires: Vec::new(),
11137 must_never_reach: must_never_reach
11138 .iter()
11139 .map(|capability| (*capability).to_string())
11140 .collect(),
11141 });
11142 manifest
11143 }
11144
11145 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
11146 Frame::build(
11147 FrameType::Hello,
11148 control_flags(),
11149 0,
11150 0,
11151 corr,
11152 serde_json::to_vec(&ModuleHelloBody {
11153 protocol_ver: manifest.protocol_ver,
11154 manifest,
11155 control_ops: None,
11156 launch_nonce: None,
11157 })
11158 .expect("capability test HELLO serializes"),
11159 )
11160 .expect("capability test HELLO frame builds")
11161 }
11162
11163 fn catalog_update_with_capabilities_frame(
11164 corr: u64,
11165 capabilities: CapabilityDeclarations,
11166 ) -> Frame {
11167 Frame::build(
11168 FrameType::Request,
11169 control_flags(),
11170 0,
11171 0,
11172 corr,
11173 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
11174 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
11175 capabilities: Some(capabilities),
11176 ready: None,
11177 })
11178 .expect("capability catalog.update serializes"),
11179 )
11180 .expect("capability catalog.update frame builds")
11181 }
11182
11183 async fn register_capability_manifest(
11184 handler: &ControlHandler,
11185 ctx: &RouteCtx,
11186 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
11187 manifest: ModuleManifest,
11188 corr: u64,
11189 ) {
11190 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
11191 }
11192
11193 async fn open_route_for_capability_test(
11194 handler: &ControlHandler,
11195 target_ctx: &RouteCtx,
11196 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
11197 client_connection_id: u64,
11198 corr: u64,
11199 target_module_id: &str,
11200 consumer_identity: Option<ConsumerIdentity>,
11201 ) -> (
11202 mpsc::Receiver<crate::router::OutboundFrame>,
11203 ModuleControlRequest,
11204 ) {
11205 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
11206 let route_handler = handler.clone();
11207 let target_module_id = target_module_id.to_string();
11208 let route_task = tokio::spawn(async move {
11209 route_handler
11210 .handle_control_frame(
11211 &client_ctx,
11212 route_open_frame_with_admission_facts(
11213 corr,
11214 &target_module_id,
11215 unique_project_root("admission-facts"),
11216 consumer_identity,
11217 None,
11218 ),
11219 )
11220 .await
11221 .expect("capability test route.open succeeds")
11222 });
11223 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
11224 .await
11225 .expect("capability test route.open must reach route.bind")
11226 .expect("target control receiver stays open");
11227 let bind_request: ModuleControlRequest =
11228 serde_json::from_slice(&bind.body).expect("route.bind decodes");
11229 handler
11230 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
11231 .await
11232 .expect("capability test route.bind ACK succeeds");
11233 assert!(route_task.await.expect("route.open task joins").is_empty());
11234 let opened = client_rx
11235 .recv()
11236 .await
11237 .expect("successful route.open publishes a response");
11238 assert!(matches!(
11239 serde_json::from_slice::<ClientControlResponse>(&opened.body),
11240 Ok(ClientControlResponse::RouteOpen { .. })
11241 ));
11242 (client_rx, bind_request)
11243 }
11244
11245 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
11246 assert_eq!(frame.header.ty, FrameType::Push);
11247 assert_eq!(frame.header.channel, 0);
11248 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
11249 .expect("route.closed control push decodes");
11250 let ClientControlPush::RouteClosed { channels, .. } = &push else {
11251 panic!("expected route.closed");
11252 };
11253 assert_eq!(channels.len(), 1, "exactly one violating route closed");
11254 let channels = channels.clone();
11255 assert_eq!(
11256 push,
11257 ClientControlPush::RouteClosed {
11258 module_id: target_module_id.to_string(),
11259 channels,
11260 reason: RouteCloseReason::CapabilityDenied,
11261 drained: false,
11262 abandoned: 0,
11263 excluded_subscriptions: 0,
11264 terminal: Some(false),
11265 }
11266 );
11267 }
11268
11269 #[tokio::test]
11270 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
11271 let registry = Arc::new(Registry::default());
11272 let forwarding = Arc::new(ForwardingTable::default());
11273 let supervisor = SupervisorHandle::new();
11274 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11275 let handler =
11276 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11277 .with_supervisor(supervisor);
11278 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
11279 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
11280 register_capability_manifest(
11281 &handler,
11282 &target_ctx,
11283 &mut target_rx,
11284 capability_manifest("target", &["credentials-provider/v1"], &[]),
11285 1,
11286 )
11287 .await;
11288 register_capability_manifest(
11289 &handler,
11290 &opener_ctx,
11291 &mut opener_rx,
11292 capability_manifest("opener", &[], &["credentials-provider/v1"]),
11293 2,
11294 )
11295 .await;
11296
11297 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
11298 let replies = handler
11299 .handle_control_frame(
11300 &client_ctx,
11301 route_open_frame_with_admission_facts(
11302 3,
11303 "target",
11304 unique_project_root("admission-facts"),
11305 Some(ConsumerIdentity {
11306 module_id: "opener".to_string(),
11307 launch_nonce: "opener-nonce".to_string(),
11308 }),
11309 None,
11310 ),
11311 )
11312 .await
11313 .expect("denied route.open returns a typed frame");
11314 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
11315 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
11316 assert!(
11317 target_rx.try_recv().is_err(),
11318 "forbidden route.open must not relay route.bind"
11319 );
11320 }
11321
11322 #[tokio::test]
11323 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
11324 let registry = Arc::new(Registry::default());
11325 let forwarding = Arc::new(ForwardingTable::default());
11326 let supervisor = SupervisorHandle::new();
11327 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11328 let handler =
11329 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11330 .with_supervisor(supervisor);
11331 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
11332 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
11333 register_capability_manifest(
11334 &handler,
11335 &target_ctx,
11336 &mut target_rx,
11337 capability_manifest("target", &["credentials-provider/v1"], &[]),
11338 1,
11339 )
11340 .await;
11341 register_capability_manifest(
11342 &handler,
11343 &old_opener_ctx,
11344 &mut old_opener_rx,
11345 capability_manifest("opener", &[], &[]),
11346 2,
11347 )
11348 .await;
11349 let (mut client_rx, _) = open_route_for_capability_test(
11350 &handler,
11351 &target_ctx,
11352 &mut target_rx,
11353 712,
11354 3,
11355 "target",
11356 Some(ConsumerIdentity {
11357 module_id: "opener".to_string(),
11358 launch_nonce: "opener-nonce".to_string(),
11359 }),
11360 )
11361 .await;
11362 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
11363
11364 handler
11365 .cleanup_connection(old_opener_ctx.connection_id)
11366 .expect("old opener registration cleans up");
11367 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
11368 register_capability_manifest(
11369 &handler,
11370 &new_opener_ctx,
11371 &mut new_opener_rx,
11372 capability_manifest("opener", &[], &["credentials-provider/v1"]),
11373 4,
11374 )
11375 .await;
11376
11377 assert_capability_denied_push(
11378 client_rx
11379 .try_recv()
11380 .expect("HELLO deny addition must emit route.closed")
11381 .frame,
11382 "target",
11383 );
11384 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
11385 assert!(matches!(
11386 target_rx.try_recv(),
11387 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
11388 ));
11389 }
11390
11391 #[tokio::test]
11392 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
11393 let registry = Arc::new(Registry::default());
11394 let forwarding = Arc::new(ForwardingTable::default());
11395 let supervisor = SupervisorHandle::new();
11396 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11397 let handler =
11398 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11399 .with_supervisor(supervisor);
11400 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
11401 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
11402 register_capability_manifest(
11403 &handler,
11404 &target_ctx,
11405 &mut target_rx,
11406 capability_manifest("target", &[], &[]),
11407 1,
11408 )
11409 .await;
11410 register_capability_manifest(
11411 &handler,
11412 &opener_ctx,
11413 &mut opener_rx,
11414 capability_manifest("opener", &[], &["credentials-provider/v1"]),
11415 2,
11416 )
11417 .await;
11418 let (mut client_rx, _) = open_route_for_capability_test(
11419 &handler,
11420 &target_ctx,
11421 &mut target_rx,
11422 722,
11423 3,
11424 "target",
11425 Some(ConsumerIdentity {
11426 module_id: "opener".to_string(),
11427 launch_nonce: "opener-nonce".to_string(),
11428 }),
11429 )
11430 .await;
11431 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
11432
11433 let replies = handler
11434 .handle_control_frame(
11435 &target_ctx,
11436 catalog_update_with_capabilities_frame(
11437 4,
11438 CapabilityDeclarations {
11439 provides: vec!["credentials-provider/v1".to_string()],
11440 requires: Vec::new(),
11441 must_never_reach: Vec::new(),
11442 },
11443 ),
11444 )
11445 .await
11446 .expect("claim catalog.update succeeds");
11447 assert!(matches!(
11448 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
11449 Ok(ModuleControlResponseToModule::CatalogUpdate {})
11450 ));
11451 assert_capability_denied_push(
11452 client_rx
11453 .try_recv()
11454 .expect("claim addition must emit route.closed")
11455 .frame,
11456 "target",
11457 );
11458 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
11459 assert!(matches!(
11460 target_rx.try_recv(),
11461 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
11462 ));
11463 }
11464
11465 #[tokio::test]
11466 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
11467 let registry = Arc::new(Registry::default());
11468 let forwarding = Arc::new(ForwardingTable::default());
11469 let supervisor = SupervisorHandle::new();
11470 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11471 let handler =
11472 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11473 .with_supervisor(supervisor);
11474 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
11475 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
11476 register_capability_manifest(
11477 &handler,
11478 &target_ctx,
11479 &mut target_rx,
11480 capability_manifest("target", &["credentials-provider/v1"], &[]),
11481 1,
11482 )
11483 .await;
11484 register_capability_manifest(
11485 &handler,
11486 &opener_ctx,
11487 &mut opener_rx,
11488 capability_manifest("opener", &[], &[]),
11489 2,
11490 )
11491 .await;
11492 let (mut client_rx, _) = open_route_for_capability_test(
11493 &handler,
11494 &target_ctx,
11495 &mut target_rx,
11496 732,
11497 3,
11498 "target",
11499 Some(ConsumerIdentity {
11500 module_id: "opener".to_string(),
11501 launch_nonce: "opener-nonce".to_string(),
11502 }),
11503 )
11504 .await;
11505 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
11506
11507 handler
11508 .handle_control_frame(
11509 &target_ctx,
11510 catalog_update_with_capabilities_frame(
11511 4,
11512 CapabilityDeclarations {
11513 provides: Vec::new(),
11514 requires: Vec::new(),
11515 must_never_reach: Vec::new(),
11516 },
11517 ),
11518 )
11519 .await
11520 .expect("claim removal catalog.update succeeds");
11521 assert_eq!(
11522 forwarding.active_binding_count().unwrap(),
11523 1,
11524 "removing an attested target claim must leave the route census unchanged"
11525 );
11526 assert!(
11527 client_rx.try_recv().is_err(),
11528 "claim removal must not emit route.closed capability_denied"
11529 );
11530 assert!(
11531 target_rx.try_recv().is_err(),
11532 "claim removal must not send the target a route GOODBYE"
11533 );
11534 }
11535
11536 #[tokio::test]
11539 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
11540 let registry = Arc::new(Registry::default());
11541 let forwarding = Arc::new(ForwardingTable::default());
11542 let supervisor = SupervisorHandle::new();
11543 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11544 let handler =
11545 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11546 .with_supervisor(supervisor);
11547 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
11548 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
11549 register_capability_manifest(
11550 &handler,
11551 &target_ctx,
11552 &mut target_rx,
11553 capability_manifest("target", &["credentials-provider/v1"], &[]),
11554 1,
11555 )
11556 .await;
11557 register_capability_manifest(
11558 &handler,
11559 &opener_ctx,
11560 &mut opener_rx,
11561 capability_manifest("opener", &[], &["credentials-provider/v1"]),
11562 2,
11563 )
11564 .await;
11565
11566 let (_client_rx, bind) = open_route_for_capability_test(
11567 &handler,
11568 &target_ctx,
11569 &mut target_rx,
11570 742,
11571 3,
11572 "target",
11573 None,
11574 )
11575 .await;
11576 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
11577 panic!("direct scope-honesty route must bind");
11578 };
11579 assert_eq!(principal, Some(Principal::Direct));
11580 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
11581 }
11582
11583 #[tokio::test]
11586 async fn must_never_reach_self_route_is_capability_forbidden() {
11587 let registry = Arc::new(Registry::default());
11588 let forwarding = Arc::new(ForwardingTable::default());
11589 let supervisor = SupervisorHandle::new();
11590 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
11591 let handler =
11592 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11593 .with_supervisor(supervisor);
11594 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
11595 register_capability_manifest(
11596 &handler,
11597 &self_ctx,
11598 &mut self_rx,
11599 capability_manifest(
11600 "self-provider",
11601 &["credentials-provider/v1"],
11602 &["credentials-provider/v1"],
11603 ),
11604 1,
11605 )
11606 .await;
11607
11608 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
11609 let replies = handler
11610 .handle_control_frame(
11611 &client_ctx,
11612 route_open_frame_with_admission_facts(
11613 2,
11614 "self-provider",
11615 unique_project_root("admission-facts"),
11616 Some(ConsumerIdentity {
11617 module_id: "self-provider".to_string(),
11618 launch_nonce: "self-nonce".to_string(),
11619 }),
11620 None,
11621 ),
11622 )
11623 .await
11624 .expect("self-route refusal returns a typed frame");
11625 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
11626 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
11627 assert!(
11628 self_rx.try_recv().is_err(),
11629 "self denial must not relay route.bind"
11630 );
11631 }
11632
11633 #[test]
11634 fn unsupported_channel_zero_frame_returns_error() {
11635 let handler = ControlHandler::default();
11636 let request = Frame::build(
11637 FrameType::Request,
11638 control_flags(),
11639 0,
11640 0,
11641 21,
11642 b"opaque".to_vec(),
11643 )
11644 .unwrap();
11645
11646 let response = handler
11647 .handle_control(ConnectionId::new(1), request)
11648 .unwrap();
11649
11650 assert_eq!(response[0].header.ty, FrameType::Error);
11651 assert_eq!(
11652 parse_error(&response[0])["code"],
11653 "unsupported_control_frame"
11654 );
11655 }
11656
11657 mod swap {
11662 use super::*;
11663
11664 const INCUMBENT: ConnectionId = ConnectionId::new(30);
11665 const CANDIDATE: ConnectionId = ConnectionId::new(40);
11666
11667 struct Swap {
11668 registry: Arc<Registry>,
11669 forwarding: Arc<ForwardingTable>,
11670 handler: ControlHandler,
11671 incumbent_ctx: RouteCtx,
11672 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
11673 candidate_ctx: RouteCtx,
11674 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
11675 }
11676
11677 async fn swap_with_incumbent() -> Swap {
11678 let registry = Arc::new(Registry::default());
11679 let forwarding = Arc::new(ForwardingTable::default());
11680 let handler =
11681 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11682 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
11683 hello_via_sink(
11684 &handler,
11685 &incumbent_ctx,
11686 &mut incumbent_rx,
11687 hello_frame("aft", PROTOCOL_VERSION, 7),
11688 )
11689 .await;
11690 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
11691 Swap {
11692 registry,
11693 forwarding,
11694 handler,
11695 incumbent_ctx,
11696 incumbent_rx,
11697 candidate_ctx,
11698 candidate_rx,
11699 }
11700 }
11701
11702 fn register_candidate(swap: &Swap, ready: Option<bool>) {
11703 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
11704 candidate_manifest.ready = ready;
11705 let registration = swap
11706 .registry
11707 .register_candidate_with_control_ops(
11708 candidate_manifest,
11709 PROTOCOL_VERSION,
11710 CANDIDATE,
11711 module_baseline_control_ops(),
11712 )
11713 .unwrap();
11714 swap.forwarding
11715 .register_candidate_module_connection(
11716 CANDIDATE,
11717 "aft".to_string(),
11718 PROTOCOL_VERSION,
11719 manifest_concurrency(®istration.manifest),
11720 swap.candidate_ctx.egress.clone(),
11721 )
11722 .unwrap();
11723 }
11724
11725 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
11726 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
11727 swap.registry.promote_candidate("aft").unwrap().unwrap();
11728 cutover.incumbent.unwrap()
11729 }
11730
11731 fn keyed_total(counters: &Value, key: &str) -> u64 {
11732 counters[key]
11733 .as_object()
11734 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
11735 .unwrap_or(0)
11736 }
11737
11738 #[tokio::test]
11744 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
11745 let mut swap = swap_with_incumbent().await;
11746 let handler = swap.handler.clone();
11747
11748 let cotenant = ConnectionId::new(31);
11750 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
11751 let (cotenant_task, cotenant_bind) = relay_route_open(
11752 &handler,
11753 cotenant,
11754 &cotenant_ctx.egress,
11755 &mut swap.incumbent_rx,
11756 100,
11757 "aft",
11758 "swap-cotenant",
11759 )
11760 .await;
11761 handler
11762 .handle_control_frame(
11763 &swap.incumbent_ctx,
11764 route_bind_ack(cotenant_bind.header.corr),
11765 )
11766 .await
11767 .unwrap();
11768 assert!(cotenant_task.await.unwrap().is_empty());
11769 let (cotenant_channel, cotenant_epoch) =
11770 published_route(&cotenant_rx.recv().await.unwrap());
11771
11772 let caller = ConnectionId::new(32);
11774 let (caller_ctx, mut caller_rx) = route_ctx(caller);
11775 let (caller_task, caller_bind) = relay_route_open(
11776 &handler,
11777 caller,
11778 &caller_ctx.egress,
11779 &mut swap.incumbent_rx,
11780 101,
11781 "aft",
11782 "swap-caller",
11783 )
11784 .await;
11785 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
11786
11787 register_candidate(&swap, None);
11788 cutover(&swap);
11789
11790 let ack = handler
11792 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
11793 .await;
11794 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
11795 if module_loop_error.is_some() {
11796 handler.cleanup_connection(INCUMBENT).unwrap();
11799 }
11800
11801 assert!(
11803 cotenant_rx.try_recv().is_err(),
11804 "the co-tenant route on the incumbent was torn down by one late ack: \
11805 {module_loop_error:?}"
11806 );
11807 assert!(matches!(
11808 swap.forwarding
11809 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
11810 .unwrap(),
11811 DataRoute::Client(DataRouteState::Bound(_))
11812 ));
11813 assert_eq!(module_loop_error, None);
11814 assert!(swap
11815 .registry
11816 .get_module_by_connection(INCUMBENT)
11817 .unwrap()
11818 .is_some());
11819
11820 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
11822 .await
11823 .expect("the incumbent is told to drop the abandoned binding")
11824 .unwrap()
11825 .frame;
11826 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11827 assert_eq!(goodbye.header.channel, abandoned_channel);
11828 assert_eq!(goodbye.header.epoch, abandoned_epoch);
11829 assert!(swap.incumbent_rx.try_recv().is_err());
11830
11831 let response = caller_task.await.unwrap();
11833 assert_eq!(response.len(), 1);
11834 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
11835 assert!(caller_rx.try_recv().is_err());
11836
11837 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
11841 let counters = handler.counters().snapshot();
11842 assert_eq!(
11843 keyed_total(&counters, "route_open_accepted_by_principal"),
11844 1
11845 );
11846 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
11847 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
11848 }
11849
11850 #[tokio::test]
11854 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
11855 let mut swap = swap_with_incumbent().await;
11856 register_candidate(&swap, None);
11857 let incumbent = cutover(&swap);
11858 swap.forwarding
11859 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
11860 .unwrap()
11861 .expect("the incumbent is still registered");
11862
11863 let client = ConnectionId::new(33);
11864 let (client_ctx, mut client_rx) = route_ctx(client);
11865 let route_handler = swap.handler.clone();
11866 let open_ctx = RouteCtx {
11867 connection_id: client,
11868 egress: client_ctx.egress.clone(),
11869 };
11870 let mut route_task = tokio::spawn(async move {
11871 route_handler
11872 .handle_control_frame(
11873 &open_ctx,
11874 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
11875 )
11876 .await
11877 .unwrap()
11878 });
11879 let bind = tokio::select! {
11880 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
11881 response = &mut route_task => {
11882 let response = response.unwrap();
11883 panic!(
11884 "post-cutover route.open was refused instead of relayed to the candidate: {}",
11885 parse_error(&response[0])["code"]
11886 );
11887 }
11888 };
11889 swap.handler
11890 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
11891 .await
11892 .unwrap();
11893 assert!(route_task.await.unwrap().is_empty());
11894 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
11895 match swap
11896 .forwarding
11897 .lookup_data_route(client, channel, epoch)
11898 .unwrap()
11899 {
11900 DataRoute::Client(DataRouteState::Bound(route)) => {
11901 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
11902 }
11903 other => panic!("expected a bound route on the candidate, got {other:?}"),
11904 }
11905 assert!(swap.incumbent_rx.try_recv().is_err());
11906 }
11907
11908 #[tokio::test]
11913 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
11914 let swap = swap_with_incumbent().await;
11915 register_candidate(&swap, Some(false));
11916 let update = Frame::build(
11917 FrameType::Request,
11918 control_flags(),
11919 0,
11920 0,
11921 55,
11922 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
11923 provides: manifest("aft", PROTOCOL_VERSION).provides,
11924 capabilities: None,
11925 ready: Some(true),
11926 })
11927 .unwrap(),
11928 )
11929 .unwrap();
11930
11931 let replies = swap
11932 .handler
11933 .handle_control_frame(&swap.candidate_ctx, update)
11934 .await
11935 .unwrap();
11936
11937 assert_eq!(replies.len(), 1);
11938 assert_eq!(
11939 replies[0].header.ty,
11940 FrameType::Response,
11941 "candidate catalog.update was refused: {:?}",
11942 serde_json::from_slice::<Value>(&replies[0].body).ok()
11943 );
11944 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
11945 assert_eq!(
11946 swap.registry
11947 .get_module("aft")
11948 .unwrap()
11949 .unwrap()
11950 .connection_id,
11951 INCUMBENT
11952 );
11953 }
11954 }
11955
11956 mod swap_admission {
11960 use super::*;
11961
11962 const INCUMBENT_NONCE: &str = "incumbent-nonce";
11963 const CANDIDATE_NONCE: &str = "candidate-nonce";
11964
11965 fn handler_with_incumbent(
11966 module_id: &str,
11967 reserved: bool,
11968 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
11969 let registry = Arc::new(Registry::default());
11970 let supervisor = SupervisorHandle::new();
11971 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
11972 if reserved {
11973 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
11974 }
11975 let handler =
11976 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
11977 let incumbent = handler
11978 .handle_control(
11979 ConnectionId::new(1),
11980 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
11981 )
11982 .unwrap();
11983 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
11984 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
11985 (registry, supervisor, handler)
11986 }
11987
11988 #[test]
11994 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
11995 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
11996
11997 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
11998 let replies = handler
11999 .handle_control(
12000 ConnectionId::new(connection),
12001 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
12002 )
12003 .unwrap();
12004 assert_eq!(replies[0].header.ty, FrameType::Error);
12005 assert_eq!(
12006 parse_error(&replies[0])["code"],
12007 "swap_token_invalid",
12008 "nonce {nonce:?}"
12009 );
12010 }
12011 assert!(registry.get_candidate("aft").unwrap().is_none());
12012 assert_eq!(
12013 registry.get_module("aft").unwrap().unwrap().connection_id,
12014 ConnectionId::new(1)
12015 );
12016
12017 let admitted = handler
12020 .handle_control(
12021 ConnectionId::new(4),
12022 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
12023 )
12024 .unwrap();
12025 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
12026 assert_eq!(
12027 registry
12028 .get_candidate("aft")
12029 .unwrap()
12030 .unwrap()
12031 .connection_id,
12032 ConnectionId::new(4)
12033 );
12034 assert_eq!(
12035 registry.get_module("aft").unwrap().unwrap().connection_id,
12036 ConnectionId::new(1),
12037 "the candidate must not take the active slot"
12038 );
12039 let replayed = handler
12040 .handle_control(
12041 ConnectionId::new(5),
12042 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
12043 )
12044 .unwrap();
12045 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
12046
12047 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
12051 let squatter = handler
12052 .handle_control(
12053 ConnectionId::new(6),
12054 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
12055 )
12056 .unwrap();
12057 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
12058 assert!(
12059 registry.get_module("aft").unwrap().is_none(),
12060 "a squatter took the active slot of an id being swapped"
12061 );
12062 }
12063
12064 #[test]
12069 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
12070 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
12071
12072 let replies = handler
12073 .handle_control(
12074 ConnectionId::new(2),
12075 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
12076 )
12077 .unwrap();
12078
12079 assert_eq!(
12080 replies[0].header.ty,
12081 FrameType::HelloAck,
12082 "reserved candidate refused: {:?}",
12083 serde_json::from_slice::<Value>(&replies[0].body).ok()
12084 );
12085 assert_eq!(
12086 registry
12087 .get_candidate("vault")
12088 .unwrap()
12089 .unwrap()
12090 .connection_id,
12091 ConnectionId::new(2)
12092 );
12093 }
12094
12095 #[test]
12098 fn without_an_open_swap_the_ordinary_gates_decide() {
12099 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
12100 supervisor.close_swap("vault");
12101
12102 let candidate = handler
12103 .handle_control(
12104 ConnectionId::new(2),
12105 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
12106 )
12107 .unwrap();
12108 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
12109 let duplicate = handler
12110 .handle_control(
12111 ConnectionId::new(3),
12112 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
12113 )
12114 .unwrap();
12115 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
12116 assert!(registry.get_candidate("vault").unwrap().is_none());
12117 }
12118 }
12119
12120 mod scopes {
12124 use subc_protocol::scope::{
12125 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
12126 ScopeStatus,
12127 };
12128
12129 use super::*;
12130
12131 const OWNER: &str = "prefrontal-core";
12132
12133 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
12134 ScopeRecord {
12135 scope_ref: scope_ref.to_string(),
12136 scope_epoch,
12137 kind: ScopeKind::Head,
12138 parent: None,
12139 child_owners: Vec::new(),
12140 carriers: Vec::new(),
12141 attributes: Default::default(),
12142 }
12143 }
12144
12145 async fn call(
12146 handler: &ControlHandler,
12147 ctx: &RouteCtx,
12148 request: &ModuleControlRequestFromModule,
12149 ) -> Frame {
12150 let body = serde_json::to_vec(request).unwrap();
12151 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
12152 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
12153 assert_eq!(replies.len(), 1, "{replies:?}");
12154 replies.pop().unwrap()
12155 }
12156
12157 async fn sync(
12158 handler: &ControlHandler,
12159 ctx: &RouteCtx,
12160 generation: u64,
12161 scopes: Vec<ScopeRecord>,
12162 ) -> Result<ModuleControlResponseToModule, String> {
12163 let reply = call(
12164 handler,
12165 ctx,
12166 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
12167 )
12168 .await;
12169 match reply.header.ty {
12170 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
12171 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
12172 }
12173 }
12174
12175 async fn describe(
12176 handler: &ControlHandler,
12177 ctx: &RouteCtx,
12178 owner: &str,
12179 scope_ref: &str,
12180 ) -> ModuleControlResponseToModule {
12181 let reply = call(
12182 handler,
12183 ctx,
12184 &ModuleControlRequestFromModule::ScopeDescribe {
12185 owner: Principal::Reserved {
12186 module_id: owner.to_string(),
12187 },
12188 scope_ref: scope_ref.to_string(),
12189 },
12190 )
12191 .await;
12192 assert_eq!(
12193 reply.header.ty,
12194 FrameType::Response,
12195 "{:?}",
12196 parse_error(&reply)
12197 );
12198 serde_json::from_slice(&reply.body).unwrap()
12199 }
12200
12201 async fn module(
12203 handler: &ControlHandler,
12204 connection: u64,
12205 module_id: &str,
12206 nonce: Option<&str>,
12207 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
12208 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
12209 hello_via_sink(
12210 handler,
12211 &ctx,
12212 &mut rx,
12213 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
12214 )
12215 .await;
12216 (ctx, rx)
12217 }
12218
12219 #[tokio::test]
12222 async fn a_client_connection_cannot_sync_or_describe() {
12223 let handler = ControlHandler::new(Arc::new(Registry::default()));
12224 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
12225 for request in [
12226 ModuleControlRequestFromModule::ScopeSync {
12227 generation: 1,
12228 scopes: vec![head("s", 1)],
12229 },
12230 ModuleControlRequestFromModule::ScopeDescribe {
12231 owner: Principal::Direct,
12232 scope_ref: "s".to_string(),
12233 },
12234 ] {
12235 let reply = call(&handler, &ctx, &request).await;
12236 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
12237 }
12238 assert!(
12239 !handler
12240 .scopes
12241 .read()
12242 .unwrap()
12243 .describe(
12244 &Principal::Reserved {
12245 module_id: OWNER.to_string()
12246 },
12247 "s"
12248 )
12249 .owner_synced
12250 );
12251 }
12252
12253 #[tokio::test]
12256 async fn a_module_without_a_supervised_launch_cannot_sync() {
12257 let handler = ControlHandler::new(Arc::new(Registry::default()));
12258 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
12259 assert_eq!(
12260 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
12261 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
12262 );
12263 }
12264
12265 #[tokio::test]
12266 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
12267 let supervisor = SupervisorHandle::new();
12268 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
12269 let handler = ControlHandler::new(Arc::new(Registry::default()))
12270 .with_supervisor(supervisor.clone());
12271 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
12272 sync(&handler, &incumbent, 1, vec![head("s", 1)])
12273 .await
12274 .expect("the current launch syncs");
12275
12276 supervisor.open_swap(OWNER, "n2".to_string());
12279 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
12280 assert_eq!(
12281 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
12282 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
12283 );
12284 sync(&handler, &incumbent, 2, vec![head("s", 1)])
12285 .await
12286 .expect("the serving owner syncs during the swap");
12287
12288 supervisor.close_swap(OWNER);
12291 assert_eq!(
12292 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
12293 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
12294 );
12295 sync(&handler, &incumbent, 3, vec![head("s", 1)])
12296 .await
12297 .expect("the serving owner syncs after the rollback");
12298 handler.cleanup_connection(candidate.connection_id).unwrap();
12299
12300 supervisor.open_swap(OWNER, "n3".to_string());
12305 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
12306 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
12307 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
12308 .await
12309 .expect("the promoted launch takes authority");
12310 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
12311 panic!("unexpected reply {reply:?}");
12312 };
12313 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
12314 assert_eq!(
12315 sync(&handler, &incumbent, 4, Vec::new()).await,
12316 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
12317 );
12318 }
12319
12320 #[tokio::test]
12323 async fn closing_the_authority_connection_frees_sync_authority() {
12324 let supervisor = SupervisorHandle::new();
12325 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
12326 let handler = ControlHandler::new(Arc::new(Registry::default()))
12327 .with_supervisor(supervisor.clone());
12328 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
12329 sync(&handler, &first, 10, vec![head("s", 1)])
12330 .await
12331 .unwrap();
12332 handler.cleanup_connection(first.connection_id).unwrap();
12333
12334 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
12335 sync(&handler, &second, 1, vec![head("s", 1)])
12336 .await
12337 .expect("the next connection takes the released authority");
12338 }
12339
12340 #[tokio::test]
12341 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
12342 let supervisor = SupervisorHandle::new();
12343 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
12344 let handler =
12345 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
12346 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
12347 sync(&handler, &first, 10, vec![head("s", 1)])
12348 .await
12349 .unwrap();
12350 handler
12351 .handle_control_frame(
12352 &first,
12353 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
12354 )
12355 .await
12356 .unwrap();
12357 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
12358 sync(&handler, &second, 1, vec![head("s", 1)])
12359 .await
12360 .expect("GOODBYE releases authority even if the old socket remains open");
12361 }
12362
12363 #[tokio::test]
12364 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
12365 let registry = Arc::new(Registry::default());
12366 let supervisor_handle = SupervisorHandle::new();
12367 let supervisor =
12368 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
12369 .with_handle(supervisor_handle.clone())
12370 .with_daemon_incarnation("incarnation-7".to_string());
12371 supervisor
12374 .supervise_configured(
12375 ModuleSpec {
12376 module_id: OWNER.to_string(),
12377 program: PathBuf::from("/nonexistent/prefrontal-core"),
12378 args: Vec::new(),
12379 env: Vec::new(),
12380 reserved: false,
12381 reserved_prefixes: Vec::new(),
12382 protocol: ModuleProtocol::Subc,
12383 overlap: Default::default(),
12384 },
12385 false,
12386 )
12387 .unwrap();
12388 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
12389 let handler =
12390 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12391 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
12392
12393 let ModuleControlResponseToModule::ScopeDescribe {
12395 status,
12396 daemon_incarnation,
12397 owner_synced,
12398 owner_configured,
12399 scope,
12400 ..
12401 } = describe(&handler, &reader, OWNER, "s").await
12402 else {
12403 panic!("not a describe reply");
12404 };
12405 assert_eq!(status, ScopeStatus::NotLive);
12406 assert_eq!(daemon_incarnation, "incarnation-7");
12407 assert!(!owner_synced);
12408 assert!(owner_configured);
12409 assert!(scope.is_none());
12410
12411 let ModuleControlResponseToModule::ScopeDescribe {
12414 status,
12415 owner_configured,
12416 ..
12417 } = describe(&handler, &reader, "ghost", "s").await
12418 else {
12419 panic!("not a describe reply");
12420 };
12421 assert_eq!(status, ScopeStatus::NotLive);
12422 assert!(!owner_configured);
12423
12424 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
12426 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
12427 let ModuleControlResponseToModule::ScopeDescribe {
12428 status,
12429 scope_epoch,
12430 owner_synced,
12431 scope,
12432 ..
12433 } = describe(&handler, &reader, OWNER, "s").await
12434 else {
12435 panic!("not a describe reply");
12436 };
12437 assert_eq!(status, ScopeStatus::Live);
12438 assert_eq!(scope_epoch, Some(4));
12439 assert!(owner_synced);
12440 let stamp = scope.expect("a live scope carries its stamp");
12441 assert!(
12442 stamp.owner_authorized,
12443 "prefrontal-core is the default authority"
12444 );
12445 assert_eq!(stamp.kind, ScopeKind::Head);
12446 }
12447
12448 #[tokio::test]
12449 async fn scope_authority_owners_decides_owner_authorized() {
12450 let supervisor = SupervisorHandle::new();
12451 supervisor.set_spawn_nonce("broca", "b1".to_string());
12452 let handler = ControlHandler::new(Arc::new(Registry::default()))
12453 .with_supervisor(supervisor)
12454 .with_scope_authority_owners(vec!["broca".to_string()]);
12455 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
12456 let mut gated = head("s", 1);
12457 gated.attributes.agent_id = Some("agent".to_string());
12458 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
12459 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
12460 describe(&handler, &broca, "broca", "s").await
12461 else {
12462 panic!("not a describe reply");
12463 };
12464 assert!(scope.unwrap().owner_authorized);
12465 }
12466
12467 #[tokio::test]
12471 async fn scope_ops_and_the_scopes_capability_are_advertised() {
12472 let handler = ControlHandler::new(Arc::new(Registry::default()));
12473 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
12474 let ack = hello_via_sink(
12475 &handler,
12476 &ctx,
12477 &mut rx,
12478 hello_frame("m", PROTOCOL_VERSION, 1),
12479 )
12480 .await;
12481 let ack = parse_ack(&ack);
12482 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
12483 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
12484 }
12485 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
12486
12487 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
12488 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12489 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12490 let reply = handler
12491 .handle_control_frame(&client, frame)
12492 .await
12493 .unwrap()
12494 .pop()
12495 .unwrap();
12496 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12497 serde_json::from_slice(&reply.body).unwrap()
12498 else {
12499 panic!("not a server.describe reply");
12500 };
12501 assert!(
12502 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
12503 "{capabilities:?}"
12504 );
12505 }
12506
12507 const PLEXUS: &str = "plexus";
12510 const OTHER: &str = "other";
12511 const AFT: &str = "aft";
12512 const BROCA: &str = "broca";
12513 const MAGIC: &str = "magic-context";
12514
12515 fn nonce(module_id: &str) -> String {
12516 format!("nonce-{module_id}")
12517 }
12518
12519 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
12520 let (tx, rx) = mpsc::channel(64);
12521 (
12522 RouteCtx {
12523 connection_id: ConnectionId::new(connection),
12524 egress: FrameSink::new(tx),
12525 },
12526 rx,
12527 )
12528 }
12529
12530 struct Rig {
12534 handler: ControlHandler,
12535 forwarding: Arc<ForwardingTable>,
12536 owner: RouteCtx,
12537 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12538 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
12539 generation: u64,
12540 next_connection: u64,
12541 _supervisor: Supervisor,
12542 }
12543
12544 async fn rig() -> Rig {
12545 rig_with_flow_support(true).await
12546 }
12547
12548 async fn rig_with_flow_support(flow_support: bool) -> Rig {
12549 let registry = Arc::new(Registry::default());
12550 let forwarding = Arc::new(ForwardingTable::default());
12551 let supervisor_handle = SupervisorHandle::new();
12552 let supervisor =
12553 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
12554 .with_handle(supervisor_handle.clone());
12555 supervisor
12556 .supervise_configured(
12557 ModuleSpec {
12558 module_id: OWNER.to_string(),
12559 program: PathBuf::from("/nonexistent/prefrontal-core"),
12560 args: Vec::new(),
12561 env: Vec::new(),
12562 reserved: false,
12563 reserved_prefixes: Vec::new(),
12564 protocol: ModuleProtocol::Subc,
12565 overlap: Default::default(),
12566 },
12567 false,
12568 )
12569 .unwrap();
12570 for module_id in [OWNER, AFT, BROCA, MAGIC] {
12571 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
12572 }
12573 let handler =
12574 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12575 .with_supervisor(supervisor_handle);
12576 let (owner, mut owner_rx) = wide_ctx(1);
12577 hello_via_sink(
12578 &handler,
12579 &owner,
12580 &mut owner_rx,
12581 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
12582 )
12583 .await;
12584 let mut modules = BTreeMap::new();
12585 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
12586 let (ctx, mut rx) = wide_ctx(connection);
12587 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
12588 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
12589 body["manifest"]["provenance"] =
12593 serde_json::json!({"wire_crate_version": "0.29.0"});
12594 if flow_support {
12595 body["manifest"]["capabilities"] =
12596 serde_json::json!({"provides": ["flow-scopes/v1"]});
12597 }
12598 let hello = Frame::build(
12599 FrameType::Hello,
12600 control_flags(),
12601 0,
12602 0,
12603 connection,
12604 serde_json::to_vec(&body).unwrap(),
12605 )
12606 .unwrap();
12607 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
12608 modules.insert(module_id.to_string(), (ctx, rx));
12609 }
12610 Rig {
12611 handler,
12612 forwarding,
12613 owner,
12614 _owner_rx: owner_rx,
12615 modules,
12616 generation: 0,
12617 next_connection: 100,
12618 _supervisor: supervisor,
12619 }
12620 }
12621
12622 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
12623 ScopeCarrier {
12624 principal: Principal::Reserved {
12625 module_id: module_id.to_string(),
12626 },
12627 targets: targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()),
12628 }
12629 }
12630
12631 fn session(scope_epoch: u64) -> ScopeRecord {
12634 let mut record = head("s", scope_epoch);
12635 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
12636 record.attributes.agent_id = Some("agent-1".to_string());
12637 record.attributes.delegates = true;
12638 record
12639 }
12640
12641 impl Rig {
12642 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
12643 self.generation += 1;
12644 sync(&self.handler, &self.owner, self.generation, scopes)
12645 .await
12646 .expect("the owner's sync is accepted");
12647 }
12648
12649 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
12650 ScopeSelector {
12651 owner: Principal::Reserved {
12652 module_id: OWNER.to_string(),
12653 },
12654 scope_ref: scope_ref.to_string(),
12655 scope_epoch,
12656 }
12657 }
12658
12659 fn open_frame(
12660 &mut self,
12661 opener: Option<&str>,
12662 target: &str,
12663 scope: Option<ScopeSelector>,
12664 ) -> (
12665 RouteCtx,
12666 mpsc::Receiver<crate::router::OutboundFrame>,
12667 Frame,
12668 ) {
12669 self.next_connection += 1;
12670 let (ctx, rx) = wide_ctx(self.next_connection);
12671 let root = unique_project_root("scoped-open");
12672 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
12673 target: RouteTarget::ToolProvider {
12674 module_id: target.to_string(),
12675 },
12676 identity: BindIdentity::new(
12677 root.path().to_path_buf(),
12678 "unit".to_string(),
12679 "session".to_string(),
12680 ),
12681 consumer_identity: opener.map(|module_id| ConsumerIdentity {
12682 module_id: module_id.to_string(),
12683 launch_nonce: nonce(module_id),
12684 }),
12685 consumer_capabilities: None,
12686 role_versions: None,
12687 admission_facts: None,
12688 scope,
12689 })
12690 .unwrap();
12691 let frame = Frame::build(
12692 FrameType::Request,
12693 control_flags(),
12694 0,
12695 0,
12696 self.next_connection,
12697 body,
12698 )
12699 .unwrap();
12700 (ctx, rx, frame)
12701 }
12702
12703 async fn refused(
12705 &mut self,
12706 opener: Option<&str>,
12707 target: &str,
12708 scope: Option<ScopeSelector>,
12709 ) -> String {
12710 self.refusal_body(opener, target, scope).await["code"]
12711 .as_str()
12712 .unwrap()
12713 .to_string()
12714 }
12715
12716 async fn refusal_body(
12717 &mut self,
12718 opener: Option<&str>,
12719 target: &str,
12720 scope: Option<ScopeSelector>,
12721 ) -> Value {
12722 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
12723 let replies = tokio::time::timeout(
12724 Duration::from_secs(2),
12725 self.handler.handle_control_frame(&ctx, frame),
12726 )
12727 .await
12728 .expect("the open must be refused before waiting for a bind ack")
12729 .unwrap();
12730 assert_eq!(replies.len(), 1, "{replies:?}");
12731 assert_eq!(replies[0].header.ty, FrameType::Error);
12732 let (_, module_rx) = self.modules.get_mut(target).unwrap();
12733 assert!(
12734 module_rx.try_recv().is_err(),
12735 "a refused open relays nothing"
12736 );
12737 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
12738 parse_error(&replies[0])
12739 }
12740
12741 async fn relayed(
12743 &mut self,
12744 opener: Option<&str>,
12745 target: &str,
12746 scope: Option<ScopeSelector>,
12747 ) -> Relayed {
12748 let (ctx, rx, frame) = self.open_frame(opener, target, scope);
12749 let handler = self.handler.clone();
12750 let task_ctx = ctx.clone();
12751 let task = tokio::spawn(async move {
12752 handler
12753 .handle_control_frame(&task_ctx, frame)
12754 .await
12755 .unwrap()
12756 });
12757 let (_, module_rx) = self.modules.get_mut(target).unwrap();
12758 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
12759 .await
12760 .expect("the target receives the relayed route.bind")
12761 .unwrap()
12762 .frame;
12763 Relayed {
12764 target: target.to_string(),
12765 client: ctx,
12766 client_rx: rx,
12767 task,
12768 bind,
12769 }
12770 }
12771
12772 async fn ack(&self, relayed: &Relayed) {
12773 let (module, _) = &self.modules[&relayed.target];
12774 self.handler
12775 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
12776 .await
12777 .unwrap();
12778 }
12779
12780 async fn bound(
12782 &mut self,
12783 opener: Option<&str>,
12784 target: &str,
12785 scope: Option<ScopeSelector>,
12786 ) -> Bound {
12787 let relayed = self.relayed(opener, target, scope).await;
12788 self.ack(&relayed).await;
12789 let Relayed {
12790 target,
12791 client,
12792 mut client_rx,
12793 task,
12794 bind,
12795 } = relayed;
12796 assert!(
12797 task.await.unwrap().is_empty(),
12798 "the open is answered by commit"
12799 );
12800 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
12801 Bound {
12802 target,
12803 client,
12804 client_rx,
12805 channel,
12806 epoch,
12807 bind,
12808 }
12809 }
12810
12811 fn live(&self, route: &Bound) -> bool {
12812 matches!(
12813 self.forwarding
12814 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
12815 .unwrap(),
12816 DataRoute::Client(DataRouteState::Bound(_))
12817 )
12818 }
12819 }
12820
12821 struct Relayed {
12822 target: String,
12823 client: RouteCtx,
12824 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12825 task: tokio::task::JoinHandle<Vec<Frame>>,
12826 bind: Frame,
12827 }
12828
12829 struct Bound {
12830 target: String,
12831 client: RouteCtx,
12832 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12833 channel: u16,
12834 epoch: u32,
12835 bind: Frame,
12836 }
12837
12838 impl Bound {
12839 fn closed_reason(&mut self) -> RouteCloseReason {
12842 let mut reason = None;
12843 let mut goodbye = false;
12844 while let Ok(outbound) = self.client_rx.try_recv() {
12845 let frame = outbound.frame;
12846 match frame.header.ty {
12847 FrameType::Goodbye => {
12848 assert_eq!(
12849 (frame.header.channel, frame.header.epoch),
12850 (self.channel, self.epoch)
12851 );
12852 goodbye = true;
12853 }
12854 FrameType::Push => {
12855 let ClientControlPush::RouteClosed {
12856 reason: r,
12857 module_id,
12858 ..
12859 } = serde_json::from_slice(&frame.body).unwrap()
12860 else {
12861 panic!("unexpected push");
12862 };
12863 assert_eq!(module_id, self.target);
12864 reason = Some(r);
12865 }
12866 other => panic!("unexpected frame {other:?}"),
12867 }
12868 }
12869 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
12870 reason.expect("the client is told why the route closed")
12871 }
12872
12873 fn untouched(&mut self) -> bool {
12874 self.client_rx.try_recv().is_err()
12875 }
12876
12877 fn stamp(&self) -> Option<ScopeStamp> {
12878 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
12879 ModuleControlRequest::RouteBind { scope, .. } => scope,
12880 other => panic!("expected a route.bind, got {other:?}"),
12881 }
12882 }
12883 }
12884
12885 #[tokio::test]
12886 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
12887 ) {
12888 let mut rig = rig().await;
12889 let mut record = session(1);
12890 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
12891 record.child_owners = vec![Principal::Reserved {
12892 module_id: MAGIC.to_string(),
12893 }];
12894 rig.sync(vec![record]).await;
12895 let scope = || Some(rig_selector("s", Some(1)));
12896
12897 rig.bound(Some(OWNER), PLEXUS, scope()).await;
12900 rig.bound(Some(AFT), OTHER, scope()).await;
12901 rig.bound(Some(BROCA), PLEXUS, scope()).await;
12902
12903 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
12907 assert_eq!(
12908 rig.refused(opener, target, scope()).await,
12909 error_codes::SCOPE_NOT_CARRIER,
12910 "{opener:?} -> {target}"
12911 );
12912 }
12913 }
12914
12915 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
12916 ScopeSelector {
12917 owner: Principal::Reserved {
12918 module_id: OWNER.to_string(),
12919 },
12920 scope_ref: scope_ref.to_string(),
12921 scope_epoch,
12922 }
12923 }
12924
12925 #[tokio::test]
12926 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
12927 let mut rig = rig().await;
12928 rig.sync(vec![session(1)]).await;
12929 for opener in [OWNER, AFT] {
12930 assert_eq!(
12931 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
12932 .await,
12933 error_codes::SCOPE_EPOCH_REQUIRED,
12934 "{opener}"
12935 );
12936 }
12937 }
12938
12939 #[tokio::test]
12940 async fn admission_separates_not_synced_not_live_and_ended() {
12941 let mut rig = rig().await;
12942 let code = rig
12944 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12945 .await;
12946 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
12947 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
12948
12949 let ghost = ScopeSelector {
12951 owner: Principal::Reserved {
12952 module_id: "ghost".to_string(),
12953 },
12954 scope_ref: "s".to_string(),
12955 scope_epoch: Some(1),
12956 };
12957 assert_eq!(
12958 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
12959 error_codes::SCOPE_NOT_LIVE
12960 );
12961
12962 rig.sync(vec![session(2)]).await;
12963 assert_eq!(
12964 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
12965 .await,
12966 error_codes::SCOPE_NOT_LIVE
12967 );
12968 for epoch in [1, 3] {
12969 assert_eq!(
12970 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
12971 .await,
12972 error_codes::SCOPE_ENDED,
12973 "epoch {epoch}"
12974 );
12975 }
12976 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
12978 .await;
12979 }
12980
12981 #[tokio::test]
12982 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
12983 let mut rig = rig().await;
12984 rig.sync(vec![session(1)]).await;
12985 let route = rig
12986 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12987 .await;
12988 let stamp = route.stamp().expect("a scoped bind carries the stamp");
12989 assert_eq!(stamp.scope_ref, "s");
12990 assert_eq!(stamp.scope_epoch, 1);
12991 assert_eq!(stamp.kind, ScopeKind::Head);
12992 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
12993 assert!(stamp.attributes.delegates);
12994 assert!(stamp.owner_authorized);
12995 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
12996 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
12997
12998 let (broca, mut broca_rx) = wide_ctx(50);
13001 hello_via_sink(
13002 &rig.handler,
13003 &broca,
13004 &mut broca_rx,
13005 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
13006 )
13007 .await;
13008 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
13009 .await
13010 .unwrap();
13011 let own = ScopeSelector {
13012 owner: Principal::Reserved {
13013 module_id: BROCA.to_string(),
13014 },
13015 scope_ref: "b".to_string(),
13016 scope_epoch: Some(1),
13017 };
13018 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
13019 assert!(!route.stamp().unwrap().owner_authorized);
13020 }
13021
13022 #[tokio::test]
13023 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
13024 let mut rig = rig().await;
13025 let mut record = head("s", 1);
13026 record.carriers = vec![carrier(AFT, None)];
13027 let flow_id = "Flow:run-7/step_2!~";
13028 record.attributes.flow_id = Some(flow_id.to_string());
13029 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
13030 .await
13031 .unwrap();
13032 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
13033 panic!("not a sync reply");
13034 };
13035 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
13036 let route = rig
13037 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13038 .await;
13039 assert!(rig.live(&route), "the stamped bind committed");
13040 let stamp = route.stamp().expect("a flow scope carries a stamp");
13041 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
13042 assert_eq!(stamp.attributes.agent_id, None);
13043 assert!(!stamp.attributes.delegates);
13044 assert!(stamp.owner_authorized);
13045 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
13046 assert_eq!(unscoped.stamp(), None);
13047 }
13048
13049 #[tokio::test]
13050 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
13051 let mut rig = rig_with_flow_support(false).await;
13052 let mut record = session(1);
13053 record.attributes.flow_id = Some("flow:7".to_string());
13054 rig.sync(vec![record]).await;
13055 for opener in [OWNER, AFT] {
13056 let body = rig
13057 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
13058 .await;
13059 assert_eq!(body["code"], "target_flow_unsupported");
13060 let message = body["message"].as_str().unwrap();
13061 for required in [PLEXUS, "flow-scopes/v1"] {
13062 assert!(message.contains(required), "{message}");
13063 }
13064 }
13065 }
13066
13067 #[tokio::test]
13068 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
13069 let mut rig = rig_with_flow_support(true).await;
13070 let mut record = session(1);
13071 record.attributes.flow_id = Some("flow:7".to_string());
13072 rig.sync(vec![record]).await;
13073 for opener in [OWNER, AFT] {
13074 let route = rig
13075 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
13076 .await;
13077 assert!(rig.live(&route));
13078 assert_eq!(
13079 route.stamp().unwrap().attributes.flow_id.as_deref(),
13080 Some("flow:7")
13081 );
13082 }
13083 }
13084
13085 #[tokio::test]
13086 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
13087 use std::future::Future;
13088
13089 let mut rig = rig_with_flow_support(true).await;
13090 let mut record = session(1);
13091 record.attributes.flow_id = Some("flow:7".to_string());
13092 rig.sync(vec![record]).await;
13093 let (client, mut client_rx, frame) =
13094 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
13095 for _ in 0..64 {
13098 client.egress.try_send(route_bind_ack(1)).unwrap();
13099 }
13100 let handler = rig.handler.clone();
13101 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
13102 std::future::poll_fn(|cx| {
13103 assert!(open.as_mut().poll(cx).is_pending());
13104 std::task::Poll::Ready(())
13105 })
13106 .await;
13107 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
13108
13109 let old_connection = rig.modules[PLEXUS].0.connection_id;
13110 rig.handler.cleanup_connection(old_connection).unwrap();
13111 let (replacement, mut replacement_rx) = wide_ctx(200);
13112 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
13113 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
13114 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
13115 let hello = Frame::build(
13116 FrameType::Hello,
13117 control_flags(),
13118 0,
13119 0,
13120 200,
13121 serde_json::to_vec(&body).unwrap(),
13122 )
13123 .unwrap();
13124 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
13125
13126 client_rx.try_recv().unwrap();
13127 let replies = tokio::time::timeout(Duration::from_secs(2), open)
13128 .await
13129 .expect("the replacement is refused without waiting for a bind ack")
13130 .unwrap();
13131 assert_eq!(replies.len(), 1);
13132 let body = parse_error(&replies[0]);
13133 assert_eq!(body["code"], "target_flow_unsupported");
13134 for required in [PLEXUS, "flow-scopes/v1"] {
13135 assert!(body["message"].as_str().unwrap().contains(required));
13136 }
13137 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
13138 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
13139 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
13140 assert!(rig
13141 .handler
13142 .registry
13143 .get_module_by_connection(replacement.connection_id)
13144 .unwrap()
13145 .is_some());
13146 }
13147
13148 #[tokio::test]
13149 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
13150 {
13151 let mut rig = rig_with_flow_support(false).await;
13152 rig.sync(vec![session(1)]).await;
13153 let route = rig
13154 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13155 .await;
13156 assert!(rig.live(&route));
13157 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
13158 let mut record = session(1);
13159 record.attributes.flow_id = Some("flow:7".to_string());
13160 rig.sync(vec![record]).await;
13161 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
13162 assert!(rig.live(&unscoped));
13163 assert_eq!(unscoped.stamp(), None);
13164 }
13165
13166 #[tokio::test]
13167 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
13168 let mut rig = rig().await;
13169 let mut record = session(1);
13170 record.attributes.flow_id = Some("flow:7".to_string());
13171 rig.sync(vec![record.clone()]).await;
13172 let before = rig.forwarding.published_scope_tag(OWNER, "s");
13173 let mut owner_route = rig
13174 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
13175 .await;
13176 let mut carrier_route = rig
13177 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
13178 .await;
13179 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
13180 rig.sync(vec![record.clone()]).await;
13181 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
13182 assert!(rig.live(&owner_route) && owner_route.untouched());
13183 assert!(rig.live(&carrier_route) && carrier_route.untouched());
13184
13185 record.attributes.flow_id = Some("flow:8".to_string());
13186 rig.sync(vec![record]).await;
13187 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
13188 let before = before.unwrap();
13189 assert_eq!(after.scope_epoch, before.scope_epoch);
13190 assert!(after.version > before.version);
13191 for route in [&mut owner_route, &mut carrier_route] {
13192 assert!(!rig.live(route));
13193 assert_eq!(
13194 route.closed_reason(),
13195 RouteCloseReason::ScopeDelegationChanged
13196 );
13197 }
13198 assert!(rig.live(&unscoped) && unscoped.untouched());
13199 for target in [PLEXUS, OTHER] {
13202 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
13203 let goodbye = module_rx
13204 .try_recv()
13205 .expect("the provider sees the drain")
13206 .frame;
13207 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
13208 assert!(module_rx.try_recv().is_err());
13209 }
13210 let rebound = rig
13211 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13212 .await;
13213 assert_eq!(
13214 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
13215 Some("flow:8")
13216 );
13217 }
13218
13219 #[tokio::test]
13224 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
13225 let mut rig = rig().await;
13226 rig.sync(vec![session(1)]).await;
13227 let mut cotenant = rig
13228 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
13229 .await;
13230
13231 let mut changed = session(1);
13232 changed.child_owners.push(Principal::Reserved {
13233 module_id: MAGIC.to_string(),
13234 });
13235 let mut ended = None;
13236 for (code, next) in [
13237 (error_codes::SCOPE_CHANGED, vec![changed]),
13238 (error_codes::SCOPE_ENDED, Vec::new()),
13239 ] {
13240 let relayed = rig
13241 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13242 .await;
13243 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
13244 ended = Some(next.is_empty());
13245 rig.sync(next).await;
13246 rig.ack(&relayed).await;
13247 let replies = relayed.task.await.unwrap();
13248 assert_eq!(replies.len(), 1, "{replies:?}");
13249 assert_eq!(parse_error(&replies[0])["code"], code);
13250 assert_eq!(
13251 subc_protocol::error_codes::is_retryable_route_open(code),
13252 code == error_codes::SCOPE_CHANGED
13253 );
13254 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
13255 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
13257 let mut goodbyes = Vec::new();
13260 while let Ok(outbound) = plexus_rx.try_recv() {
13261 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
13262 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
13263 }
13264 assert!(
13265 goodbyes.contains(&(bind_channel, bind_epoch)),
13266 "{goodbyes:?}"
13267 );
13268 assert!(rig
13269 .handler
13270 .registry
13271 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
13272 .unwrap()
13273 .is_some());
13274 }
13275 assert_eq!(ended, Some(true));
13276 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
13279 }
13280
13281 #[tokio::test]
13284 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
13285 struct Case {
13286 name: &'static str,
13287 change: fn(&mut ScopeRecord),
13288 closed: [Option<RouteCloseReason>; 4],
13291 }
13292 use RouteCloseReason::*;
13293 let cases = [
13294 Case {
13295 name: "a carrier entry removed",
13296 change: |r| {
13297 r.carriers.retain(|c| {
13298 c.principal
13299 != Principal::Reserved {
13300 module_id: AFT.to_string(),
13301 }
13302 })
13303 },
13304 closed: [None, Some(ScopeCarrierRemoved), None, None],
13305 },
13306 Case {
13307 name: "a target removed from a carrier",
13308 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
13309 closed: [None, None, None, Some(ScopeCarrierRemoved)],
13310 },
13311 Case {
13312 name: "a bare carrier narrowed to targets",
13313 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
13314 closed: [None, Some(ScopeCarrierRemoved), None, None],
13315 },
13316 Case {
13317 name: "delegates turned off",
13318 change: |r| r.attributes.delegates = false,
13319 closed: [Some(ScopeDelegationChanged); 4],
13320 },
13321 Case {
13322 name: "agent_id changed",
13323 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
13324 closed: [Some(ScopeDelegationChanged); 4],
13325 },
13326 Case {
13327 name: "a carrier added, child owners changed, the record re-sent",
13328 change: |r| {
13329 r.carriers.push(carrier(MAGIC, None));
13330 r.child_owners.push(Principal::Reserved {
13331 module_id: MAGIC.to_string(),
13332 });
13333 },
13334 closed: [None; 4],
13335 },
13336 Case {
13337 name: "a target added",
13338 change: |r| {
13339 r.carriers[1]
13340 .targets
13341 .as_mut()
13342 .unwrap()
13343 .push("third".to_string())
13344 },
13345 closed: [None; 4],
13346 },
13347 Case {
13348 name: "delegates turned on",
13349 change: |r| r.attributes.delegates = true,
13350 closed: [None; 4],
13351 },
13352 ];
13353 for case in cases {
13354 let mut rig = rig().await;
13355 rig.sync(vec![session(1)]).await;
13356 let scope = || Some(rig_selector("s", Some(1)));
13357 let mut routes = [
13358 rig.bound(Some(OWNER), PLEXUS, scope()).await,
13359 rig.bound(Some(AFT), PLEXUS, scope()).await,
13360 rig.bound(Some(BROCA), PLEXUS, scope()).await,
13361 rig.bound(Some(BROCA), OTHER, scope()).await,
13362 ];
13363 let mut record = session(1);
13364 (case.change)(&mut record);
13365 rig.sync(vec![record]).await;
13366 for (index, expected) in case.closed.iter().enumerate() {
13367 let route = &mut routes[index];
13368 match expected {
13369 Some(reason) => {
13370 assert!(!rig.live(route), "{}: route {index} still live", case.name);
13371 assert_eq!(
13372 route.closed_reason(),
13373 *reason,
13374 "{}: route {index}",
13375 case.name
13376 );
13377 }
13378 None => {
13379 assert!(rig.live(route), "{}: route {index} closed", case.name);
13380 assert!(
13381 route.untouched(),
13382 "{}: route {index} was told something",
13383 case.name
13384 );
13385 }
13386 }
13387 }
13388 }
13389 }
13390
13391 #[tokio::test]
13392 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
13393 for next in [Vec::new(), vec![session(2)]] {
13395 let mut rig = rig().await;
13396 rig.sync(vec![session(1)]).await;
13397 let mut route = rig
13398 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13399 .await;
13400 rig.sync(next).await;
13401 assert!(!rig.live(&route));
13402 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
13403 }
13404
13405 let mut rig = rig().await;
13408 let mut child = session(1);
13409 child.scope_ref = "child".to_string();
13410 child.kind = ScopeKind::Worker;
13411 child.parent = Some(ScopeParent {
13412 owner: Principal::Reserved {
13413 module_id: OWNER.to_string(),
13414 },
13415 scope_ref: "s".to_string(),
13416 scope_epoch: 1,
13417 });
13418 rig.sync(vec![session(1), child.clone()]).await;
13419 let mut child_route = rig
13420 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
13421 .await;
13422 assert_eq!(
13423 child_route.stamp().unwrap().parent_state,
13424 Some(ParentState::Linked)
13425 );
13426 rig.sync(vec![child]).await;
13427 assert!(!rig.live(&child_route));
13428 assert_eq!(
13429 child_route.closed_reason(),
13430 RouteCloseReason::ScopeParentEnded
13431 );
13432 }
13433
13434 #[tokio::test]
13435 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
13436 ) {
13437 let mut rig = rig().await;
13438 rig.sync(vec![session(1)]).await;
13439 let mut route = rig
13440 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13441 .await;
13442 let before = rig.forwarding.published_scope_tag(OWNER, "s");
13443 rig.sync(vec![session(1)]).await;
13444 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
13445 assert!(rig.live(&route) && route.untouched());
13446
13447 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
13451 .forwarding
13452 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
13453 .unwrap()
13454 else {
13455 panic!("the route is bound");
13456 };
13457 binding.flow.acquire_tagged(9, false).await.unwrap();
13458 let mut widened = session(1);
13459 widened.carriers.push(carrier(MAGIC, None));
13460 rig.sync(vec![widened]).await;
13461 assert!(rig.live(&route) && route.untouched());
13462 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
13463 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
13464 assert_eq!(binding.flow.in_flight(), 1);
13467 binding
13468 .flow
13469 .acquire_tagged(10, false)
13470 .await
13471 .expect("the flow is still open");
13472 }
13473
13474 #[tokio::test]
13477 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
13478 let mut rig = rig().await;
13479 rig.sync(vec![session(1)]).await;
13480 let mut on_incumbent = rig
13481 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13482 .await;
13483
13484 let (candidate, _candidate_rx) = wide_ctx(9);
13487 let registration = rig
13488 .handler
13489 .registry
13490 .register_candidate_with_control_ops(
13491 manifest(PLEXUS, PROTOCOL_VERSION),
13492 PROTOCOL_VERSION,
13493 candidate.connection_id,
13494 module_baseline_control_ops(),
13495 )
13496 .unwrap();
13497 rig.forwarding
13498 .register_candidate_module_connection(
13499 candidate.connection_id,
13500 PLEXUS.to_string(),
13501 PROTOCOL_VERSION,
13502 manifest_concurrency(®istration.manifest),
13503 candidate.egress.clone(),
13504 )
13505 .unwrap();
13506 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
13507 rig.handler
13508 .registry
13509 .promote_candidate(PLEXUS)
13510 .unwrap()
13511 .unwrap();
13512 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
13513
13514 rig.sync(Vec::new()).await;
13515 assert!(!rig.live(&on_incumbent));
13516 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
13517 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
13518 let goodbye = incumbent_rx
13519 .try_recv()
13520 .expect("the superseded endpoint is told")
13521 .frame;
13522 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
13523 }
13524 }
13525}
13526
13527#[cfg(test)]
13528mod concurrency_default_exposure_tests {
13529 use super::*;
13530
13531 fn hello_body(role_json: &str) -> Vec<u8> {
13532 format!(
13533 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
13534 )
13535 .into_bytes()
13536 }
13537
13538 fn manifest_from(body: &[u8]) -> ModuleManifest {
13539 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
13540 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
13541 .expect("manifest parses")
13542 }
13543
13544 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
13545
13546 #[test]
13547 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
13548 let body = hello_body(&format!(
13549 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
13550 ));
13551 let manifest = manifest_from(&body);
13552 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
13555 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
13556 }
13557
13558 #[test]
13559 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
13560 let body = hello_body(&format!(
13561 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
13562 ));
13563 let manifest = manifest_from(&body);
13564 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
13565 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
13566 }
13567
13568 #[test]
13569 fn non_management_roles_are_never_reported() {
13570 let body = hello_body(
13571 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
13572 );
13573 let manifest = manifest_from(&body);
13574 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
13575 }
13576}