1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 path::{Path, PathBuf},
5 sync::{Arc, Mutex, RwLock},
6 time::{Duration, Instant as StdInstant},
7};
8
9use serde::{Deserialize, Serialize};
10use subc_control::{
11 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
12 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
13 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
14 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
15 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
16 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
17 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
18};
19use subc_protocol::{
20 error_codes,
21 manifest::{
22 validate_hello_capability_grammar, validate_hello_self_signal_declarations,
23 CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
24 ProviderRole,
25 },
26 scope::{
27 ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
28 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_DESCRIBE_OP, SCOPE_SYNC_OP,
29 },
30 session::{
31 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
32 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
33 MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE, ROLE_VERSIONS_FIELD,
34 },
35 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
36 Priority, RouteTarget, PROTOCOL_VERSION,
37};
38use tokio::time::{timeout_at, Instant};
39use tracing::{debug, info, warn};
40
41use crate::{
42 capability_requirements::{
43 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
44 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
45 RuntimeModule,
46 },
47 daemon_config::RestartRequiredSection,
48 forwarding::{
49 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
50 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
51 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
52 },
53 observability::{
54 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
55 },
56 provenance::{
57 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
58 },
59 registry::{ChannelState, ConnectionId, Registry, RegistryError},
60 router::{RouteCtx, RouterError},
61 scopes::{BoundScope, HelloLaunchNonces, ScopeTable},
62 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
63 stderr_tail::{CaptureState, TailEntry},
64 supervise::{
65 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
66 SupervisorHandle, SwapHelloAdmission,
67 },
68 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
69};
70
71pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
77
78const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
79const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
80const CAP_PING_PONG: &str = "ping_pong_v1";
81const CAP_SESSION_ATTACH: &str = "session_attach_v1";
82const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
83
84const SUBC_CONTROL_OPS: &[&str] = &[
85 ops::SERVER_DESCRIBE,
86 ops::CATALOG_LIST,
87 ops::ROUTE_OPEN,
88 ops::ROUTE_POLL,
89 ops::ROUTE_CLOSING,
90 ops::ROUTE_CLOSED,
91 ops::SUPERVISOR_LIST,
92 ops::SUPERVISOR_RESTART,
93 ops::SUPERVISOR_SWAP,
94 ops::SUPERVISOR_RELOAD,
95 ops::SUPERVISOR_RESCAN,
96 ops::SUPERVISOR_RELEASE_RESERVED,
97 ops::SUPERVISOR_SET_ENABLED,
98 ops::SUPERVISOR_HEALTH_PROBE,
99 ops::SUPERVISOR_HEALTH,
100 ops::SUPERVISOR_STDERR_TAIL,
101 ops::SUPERVISOR_TERMINALS,
102 ops::SUPERVISOR_ROUTES,
103 ops::SUPERVISOR_PROVENANCE,
104 ops::SUPERVISOR_SPAWN_SNAPSHOT,
105 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
106];
107
108const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
109 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
110 "supervisor.live_roots",
111 SCOPE_SYNC_OP,
112 SCOPE_DESCRIBE_OP,
113];
114
115const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
119
120const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
121
122pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
130
131pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
146
147pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
165
166const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
167const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
168
169fn reload_verdict(
170 configured: &Path,
171 spawned_from: Option<&Path>,
172 image: subc_control::RunningImageAgreement,
173) -> PendingReloadVerdict {
174 let path = match spawned_from {
175 Some(spawned_from) if configured == spawned_from => ReloadPathAgreement::Match,
176 Some(spawned_from) => ReloadPathAgreement::Mismatch {
177 configured: configured.to_path_buf(),
178 spawned_from: spawned_from.to_path_buf(),
179 },
180 None => ReloadPathAgreement::Unavailable {
181 reason: if matches!(
182 image,
183 subc_control::RunningImageAgreement::Unavailable {
184 reason: subc_control::RunningImageUnavailableReason::NotRunning
185 }
186 ) {
187 ReloadPathUnavailableReason::NotRunning
188 } else {
189 ReloadPathUnavailableReason::SpawnedPathUnavailable
190 },
191 },
192 };
193 PendingReloadVerdict { path, image }
194}
195
196#[derive(Clone)]
197struct DaemonProvenanceFacts {
198 build: DaemonBuildProvenance,
199 pid: Option<u32>,
200 started_at_ms: Option<u64>,
201 start_clock: Option<crate::clock::StartClock>,
202 executable_path: Option<PathBuf>,
203 executable_identity: Option<SpawnedFileIdentity>,
204 process_start_time: Option<u64>,
205 probe: ExecutableIdentityProbe,
206}
207
208impl Default for DaemonProvenanceFacts {
209 fn default() -> Self {
210 Self {
211 build: DaemonBuildProvenance {
212 build_git_sha: None,
213 build_lock_digest: None,
214 },
215 pid: None,
216 started_at_ms: None,
217 start_clock: None,
218 executable_path: None,
219 executable_identity: None,
220 process_start_time: None,
221 probe: ExecutableIdentityProbe::default(),
222 }
223 }
224}
225
226#[derive(Debug, Clone)]
227struct SupervisorRescanContext {
228 supervisor: Supervisor,
229 config_path: PathBuf,
230 configured_port: Option<u16>,
231 storage_config: Option<crate::daemon_config::StorageConfig>,
232 admission_facts_carrier_module_id: Option<String>,
233 admission_facts_targets: Option<Vec<String>>,
234 scope_authority_owners: Vec<String>,
235}
236
237const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
258 "reloading",
259 "supervisor_not_live",
260 "registration_not_active",
261 "no_forwarding_connection",
262 "relay_send_failed",
263];
264
265#[derive(Clone)]
267pub struct ControlHandler {
268 registry: Arc<Registry>,
269 forwarding: Arc<ForwardingTable>,
270 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
271 supervisor: SupervisorHandle,
272 subc_capabilities: Arc<[String]>,
273 route_bind_relay_timeout: Duration,
277 route_bind_relay_timeouts: BTreeMap<String, Duration>,
281 route_bind_breakers: RouteBindBreakers,
284 route_bind_concurrency: RouteBindConcurrency,
287 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
292 route_bind_breaker_threshold: u32,
294 route_bind_breaker_cooldown: Duration,
296 health_probe_timeout: Duration,
297 storage_config: Option<crate::daemon_config::StorageConfig>,
300 machine_id: Option<crate::machine_id::MachineId>,
304 admission_facts_carrier_module_id: Option<String>,
305 admission_facts_targets: Option<Vec<String>>,
306 scopes: Arc<RwLock<ScopeTable>>,
310 scope_authority_owners: Vec<String>,
313 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
316 rescan: Option<SupervisorRescanContext>,
317 connected_clients: ConnectedClients,
318 counters: DaemonCounters,
319 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
320 daemon_provenance: DaemonProvenanceFacts,
321 #[cfg(test)]
322 control_dispatch_delay: Option<Duration>,
323 #[cfg(test)]
324 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
325}
326
327impl fmt::Debug for ControlHandler {
328 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
329 f.debug_struct("ControlHandler")
330 .field("registry", &self.registry)
331 .field("forwarding", &self.forwarding)
332 .field("process_liveness", &self.process_liveness.is_some())
333 .field("supervisor", &self.supervisor)
334 .field("subc_capabilities", &self.subc_capabilities)
335 .finish()
336 }
337}
338
339struct RouteOpenRequest {
340 target: RouteTarget,
341 identity: BindIdentity,
342 consumer_identity: Option<ConsumerIdentity>,
343 consumer_capabilities: Option<Vec<String>>,
344 role_versions: Option<BTreeMap<String, String>>,
345 admission_facts: Option<serde_json::Value>,
346 scope: Option<ScopeSelector>,
347}
348
349struct RouteBindReservationGuard {
350 forwarding: Arc<ForwardingTable>,
351 endpoint: ModuleEndpointId,
352 relay_corr: u64,
353 armed: bool,
354}
355
356struct ModuleControlRpcGuard {
357 forwarding: Arc<ForwardingTable>,
358 endpoint: ModuleEndpointId,
359 corr: u64,
360 armed: bool,
361}
362
363impl ModuleControlRpcGuard {
364 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
365 Self {
366 forwarding,
367 endpoint,
368 corr,
369 armed: true,
370 }
371 }
372
373 fn disarm(&mut self) {
374 self.armed = false;
375 }
376}
377
378impl Drop for ModuleControlRpcGuard {
379 fn drop(&mut self) {
380 if self.armed {
381 let _ = self
382 .forwarding
383 .cancel_module_control_rpc(self.endpoint, self.corr);
384 }
385 }
386}
387
388impl RouteBindReservationGuard {
389 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
390 Self {
391 forwarding,
392 endpoint,
393 relay_corr,
394 armed: true,
395 }
396 }
397
398 fn release_and_disarm(&mut self) {
399 if !self.armed {
400 return;
401 }
402 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
403 self.endpoint,
404 self.relay_corr,
405 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
406 ) {
407 send_goodbye_target_best_effort(
408 &self.forwarding.counters(),
409 &target,
410 "canceled route.bind",
411 );
412 }
413 self.armed = false;
414 }
415
416 fn disarm(&mut self) {
417 self.armed = false;
418 }
419}
420
421impl Drop for RouteBindReservationGuard {
422 fn drop(&mut self) {
423 self.release_and_disarm();
424 }
425}
426
427#[derive(Debug, Clone, Default)]
457pub(crate) struct RouteBindBreakers {
458 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
459}
460
461#[derive(Debug, Clone, Default)]
462pub(crate) struct RouteBindConcurrency {
463 modules: Arc<Mutex<HashMap<String, usize>>>,
464}
465
466struct RouteBindConcurrencyGuard {
467 concurrency: RouteBindConcurrency,
468 module_id: String,
469}
470
471impl RouteBindConcurrency {
472 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
475 let mut modules = self
476 .modules
477 .lock()
478 .expect("route.bind concurrency mutex poisoned");
479 let in_flight = modules.entry(module_id.to_string()).or_default();
480 if *in_flight >= limit {
481 return Err(*in_flight);
482 }
483 *in_flight += 1;
484 Ok(RouteBindConcurrencyGuard {
485 concurrency: self.clone(),
486 module_id: module_id.to_string(),
487 })
488 }
489}
490
491impl Drop for RouteBindConcurrencyGuard {
492 fn drop(&mut self) {
493 let mut modules = self
494 .concurrency
495 .modules
496 .lock()
497 .expect("route.bind concurrency mutex poisoned");
498 let remove = {
499 let in_flight = modules
500 .get_mut(&self.module_id)
501 .expect("admitted route.bind has a concurrency entry");
502 *in_flight -= 1;
503 *in_flight == 0
504 };
505 if remove {
506 modules.remove(&self.module_id);
507 }
508 }
509}
510
511#[derive(Debug, Default)]
512struct ModuleBreakerState {
513 consecutive_timeouts: u32,
515 cooldown_until: Option<Instant>,
518 probe_in_flight: Option<Arc<()>>,
523}
524
525enum RouteBindAdmission<'a> {
527 Admitted {
528 guard: RouteBindBreakerGuard<'a>,
529 probe: bool,
532 },
533 Refused {
534 consecutive_timeouts: u32,
535 retry_in: Duration,
539 probe_in_flight: bool,
540 },
541}
542
543struct RouteBindBreakerGuard<'a> {
550 breakers: RouteBindBreakers,
551 module_id: &'a str,
552 probe_token: Option<Arc<()>>,
553 settled: bool,
554}
555
556impl RouteBindBreakerGuard<'_> {
557 fn record_accepted(&mut self) -> bool {
561 self.settled = true;
562 self.breakers.record_accepted(self.module_id)
563 }
564
565 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
568 self.settled = true;
569 self.breakers.record_timeout(
570 self.module_id,
571 self.probe_token.as_ref(),
572 threshold,
573 cooldown,
574 )
575 }
576
577 fn record_inconclusive(&mut self) {
586 self.settled = true;
587 self.breakers
588 .record_inconclusive(self.module_id, self.probe_token.as_ref());
589 }
590}
591
592impl Drop for RouteBindBreakerGuard<'_> {
593 fn drop(&mut self) {
594 if !self.settled {
595 self.breakers
596 .record_inconclusive(self.module_id, self.probe_token.as_ref());
597 }
598 }
599}
600
601struct BreakerOpened {
605 consecutive_timeouts: u32,
606 reopened_after_probe: bool,
609}
610
611impl RouteBindBreakers {
612 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
613 self.modules
614 .lock()
615 .expect("route.bind breaker mutex poisoned")
616 }
617
618 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
621 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
622 probe: probe_token.is_some(),
623 guard: RouteBindBreakerGuard {
624 breakers: self.clone(),
625 module_id,
626 probe_token,
627 settled: false,
628 },
629 };
630
631 let mut modules = self.lock();
632 let Some(state) = modules.get_mut(module_id) else {
633 return admitted(None);
634 };
635 let Some(cooldown_until) = state.cooldown_until else {
636 return admitted(None);
637 };
638 if state.probe_in_flight.is_some() {
639 return RouteBindAdmission::Refused {
640 consecutive_timeouts: state.consecutive_timeouts,
641 retry_in: Duration::ZERO,
642 probe_in_flight: true,
643 };
644 }
645 let now = Instant::now();
646 if now < cooldown_until {
647 return RouteBindAdmission::Refused {
648 consecutive_timeouts: state.consecutive_timeouts,
649 retry_in: cooldown_until - now,
650 probe_in_flight: false,
651 };
652 }
653 let token = Arc::new(());
654 state.probe_in_flight = Some(Arc::clone(&token));
655 admitted(Some(token))
656 }
657
658 fn record_accepted(&self, module_id: &str) -> bool {
659 self.lock()
660 .remove(module_id)
661 .is_some_and(|state| state.cooldown_until.is_some())
662 }
663
664 fn record_timeout(
665 &self,
666 module_id: &str,
667 probe_token: Option<&Arc<()>>,
668 threshold: u32,
669 cooldown: Duration,
670 ) -> Option<BreakerOpened> {
671 let mut modules = self.lock();
672 let state = modules.entry(module_id.to_string()).or_default();
673 let was_open = state.cooldown_until.is_some();
674 let was_probe = Self::owns_probe(state, probe_token);
675 if was_probe {
676 state.probe_in_flight = None;
677 }
678 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
679 if state.consecutive_timeouts < threshold {
680 return None;
681 }
682 state.cooldown_until = Some(Instant::now() + cooldown);
683 Some(BreakerOpened {
684 consecutive_timeouts: state.consecutive_timeouts,
685 reopened_after_probe: was_open && was_probe,
686 })
687 }
688
689 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
690 state
694 .probe_in_flight
695 .as_ref()
696 .zip(token)
697 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
698 }
699
700 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
701 if let Some(state) = self.lock().get_mut(module_id) {
702 if Self::owns_probe(state, probe_token) {
703 state.probe_in_flight = None;
704 }
705 }
706 }
707
708 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
723 self.lock()
724 .remove(module_id)
725 .map(|state| state.consecutive_timeouts)
726 .filter(|discarded| *discarded > 0)
727 }
728
729 fn open_snapshot(&self) -> Option<serde_json::Value> {
736 let now = Instant::now();
737 let modules = self.lock();
738 let open = modules
739 .iter()
740 .filter_map(|(module_id, state)| {
741 let cooldown_until = state.cooldown_until?;
742 Some((
743 module_id.clone(),
744 serde_json::json!({
745 "consecutive_timeouts": state.consecutive_timeouts,
746 "cooldown_remaining_ms":
747 cooldown_until.saturating_duration_since(now).as_millis() as u64,
748 "probe_in_flight": state.probe_in_flight.is_some(),
749 }),
750 ))
751 })
752 .collect::<serde_json::Map<String, serde_json::Value>>();
753 (!open.is_empty()).then_some(serde_json::Value::Object(open))
754 }
755}
756
757impl ControlHandler {
758 pub fn new(registry: Arc<Registry>) -> Self {
759 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
760 }
761
762 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
763 let counters = forwarding.counters();
764 let route_bind_breakers = forwarding.route_bind_breakers();
768 let route_bind_concurrency = forwarding.route_bind_concurrency();
769 let route_outages = forwarding.route_outages();
770 Self {
771 registry,
772 forwarding,
773 process_liveness: None,
774 supervisor: SupervisorHandle::new(),
775 subc_capabilities: Arc::from([
776 CAP_MANIFEST_REGISTRATION.to_string(),
777 CAP_CHANNEL_LIFECYCLE.to_string(),
778 CAP_PING_PONG.to_string(),
779 CAP_SESSION_ATTACH.to_string(),
780 CAP_ADMISSION_FACTS_RELAY.to_string(),
781 CAP_SCOPES_V1.to_string(),
782 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
783 ]),
784 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
785 route_bind_relay_timeouts: BTreeMap::new(),
786 route_bind_breakers,
787 route_bind_concurrency,
788 route_outages,
789 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
790 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
791 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
792 storage_config: None,
793 machine_id: None,
794 admission_facts_carrier_module_id: None,
795 admission_facts_targets: None,
796 scopes: Arc::new(RwLock::new(ScopeTable::new(
797 crate::daemon_config::default_scope_authority_owners(),
798 ))),
799 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
800 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
801 rescan: None,
802 connected_clients: ConnectedClients::new(),
803 counters,
804 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
805 daemon_provenance: DaemonProvenanceFacts::default(),
806 #[cfg(test)]
807 control_dispatch_delay: None,
808 #[cfg(test)]
809 provenance_probe_override: None,
810 }
811 }
812
813 pub fn with_storage_config(
816 mut self,
817 storage_config: Option<crate::daemon_config::StorageConfig>,
818 ) -> Self {
819 self.storage_config = storage_config;
820 self
821 }
822
823 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
826 self.machine_id = machine_id;
827 self
828 }
829
830 pub fn with_admission_facts_config(
834 mut self,
835 carrier_module_id: Option<String>,
836 targets: Option<Vec<String>>,
837 ) -> Self {
838 self.admission_facts_carrier_module_id = carrier_module_id;
839 self.admission_facts_targets = targets;
840 self
841 }
842
843 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
847 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
848 self.scope_authority_owners = owners;
849 self
850 }
851
852 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
855 self.route_bind_relay_timeout = timeout;
856 self
857 }
858
859 pub fn with_route_bind_relay_timeouts(
865 mut self,
866 timeouts: impl IntoIterator<Item = (String, Duration)>,
867 ) -> Self {
868 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
869 self
870 }
871
872 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
878 self.route_bind_relay_timeouts
879 .get(module_id)
880 .copied()
881 .unwrap_or(self.route_bind_relay_timeout)
882 }
883
884 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
892 self.route_bind_breaker_threshold = threshold.max(1);
893 self.route_bind_breaker_cooldown = cooldown;
894 self
895 }
896
897 #[cfg(test)]
898 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
899 self.health_probe_timeout = timeout;
900 self
901 }
902
903 #[cfg(test)]
904 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
905 self.control_dispatch_delay = Some(delay);
906 self
907 }
908
909 pub fn with_process_liveness(
910 mut self,
911 process_liveness: Arc<dyn ModuleProcessLiveness>,
912 ) -> Self {
913 self.process_liveness = Some(process_liveness);
914 self
915 }
916
917 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
918 self.supervisor = supervisor;
919 self
920 }
921
922 pub fn with_daemon_provenance(
923 mut self,
924 pid: u32,
925 started_at_ms: u64,
926 executable_path: Option<PathBuf>,
927 build_git_sha: Option<String>,
928 build_lock_digest: Option<String>,
929 ) -> Self {
930 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
931 let process_start_time = process_start_time(pid);
932 self.daemon_provenance = DaemonProvenanceFacts {
933 build: DaemonBuildProvenance {
934 build_git_sha,
935 build_lock_digest,
936 },
937 pid: Some(pid),
938 started_at_ms: Some(started_at_ms),
939 start_clock: None,
940 executable_path,
941 executable_identity,
942 process_start_time,
943 probe: ExecutableIdentityProbe::default(),
944 };
945 self
946 }
947
948 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
949 self.daemon_provenance.start_clock = Some(clock);
950 self
951 }
952
953 #[cfg(test)]
954 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
955 self.provenance_probe_override = Some(result);
956 self
957 }
958
959 pub fn with_capability_config(
963 self,
964 modules: impl IntoIterator<Item = (String, bool)>,
965 reserved_capabilities: BTreeMap<String, String>,
966 ) -> Self {
967 self.capability_evaluator
968 .configure(modules, reserved_capabilities);
969 self
970 }
971
972 pub fn with_supervisor_rescan(
973 mut self,
974 supervisor: Supervisor,
975 config_path: impl Into<PathBuf>,
976 configured_port: Option<u16>,
977 ) -> Self {
978 self.rescan = Some(SupervisorRescanContext {
979 supervisor,
980 config_path: config_path.into(),
981 configured_port,
982 storage_config: self.storage_config.clone(),
983 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
984 admission_facts_targets: self.admission_facts_targets.clone(),
985 scope_authority_owners: self.scope_authority_owners.clone(),
986 });
987 self
988 }
989
990 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
991 self.connected_clients = connected_clients;
992 self
993 }
994
995 pub fn forwarding(&self) -> Arc<ForwardingTable> {
996 Arc::clone(&self.forwarding)
997 }
998
999 pub(crate) fn counters(&self) -> DaemonCounters {
1000 self.counters.clone()
1001 }
1002
1003 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1006 tokio::spawn(async move {
1007 loop {
1008 self.capability_evaluator
1009 .wait_for_change_or_deadline()
1010 .await;
1011 self.refresh_capability_requirements();
1012 }
1013 });
1014 }
1015
1016 fn runtime_capability_snapshot(
1017 &self,
1018 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1019 let runtime = self
1020 .supervisor
1021 .list()
1022 .into_iter()
1023 .map(|module| {
1024 let status = module.status().map_err(|err| {
1025 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1026 })?;
1027 Ok(RuntimeModule {
1028 module_id: status.module_id,
1029 state: status.state,
1030 enabled: status.enabled,
1031 })
1032 })
1033 .collect::<Result<Vec<_>, RouterError>>()?;
1034 let (_, registrations) = self.registry.list_modules().map_err(|err| {
1035 RouterError::backend(
1036 0,
1037 0,
1038 format!("failed to list capability registrations: {err}"),
1039 )
1040 })?;
1041 let registrations = registrations
1042 .into_iter()
1043 .map(|registration| RegisteredModule {
1044 module_id: registration.manifest.module_id,
1045 module_version: registration.manifest.module_version,
1046 capabilities: registration.manifest.capabilities,
1047 })
1048 .collect();
1049 Ok((runtime, registrations))
1050 }
1051
1052 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1059 let cached_registration = RegisteredModule {
1060 module_id: registration.manifest.module_id.clone(),
1061 module_version: registration.manifest.module_version.clone(),
1062 capabilities: registration.manifest.capabilities.clone(),
1063 };
1064 if self.capability_evaluator.record_hello(&cached_registration) {
1065 warn!(
1066 module_id = %cached_registration.module_id,
1067 "capability claims drifted from the cached manifest"
1068 );
1069 }
1070 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1071 self.enforce_capability_denies();
1072 }
1073 self.refresh_capability_requirements();
1074 }
1075
1076 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1080 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1081 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1082 self.supervisor.set_swap_promotion_observer(observer);
1083 }
1084
1085 pub fn refresh_capability_requirements(&self) {
1086 match self.runtime_capability_snapshot() {
1087 Ok((runtime, registrations)) => {
1088 log_requirement_events(
1089 self.capability_evaluator
1090 .evaluate_now(&runtime, ®istrations),
1091 );
1092 }
1093 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1094 }
1095 }
1096
1097 fn enforce_capability_denies(&self) {
1101 let (_, registrations) = match self.registry.list_modules() {
1102 Ok(snapshot) => snapshot,
1103 Err(err) => {
1104 warn!(error = %err, "failed to read registrations for capability deny census");
1105 return;
1106 }
1107 };
1108 let manifests = registrations
1109 .into_iter()
1110 .map(|registration| {
1111 (
1112 registration.manifest.module_id.clone(),
1113 registration.manifest,
1114 )
1115 })
1116 .collect::<BTreeMap<_, _>>();
1117 let census = match self.forwarding.route_census(None) {
1118 Ok(census) => census,
1119 Err(err) => {
1120 warn!(error = %err, "failed to read route census for capability deny enforcement");
1121 return;
1122 }
1123 };
1124
1125 for (target_module_id, routes) in census {
1126 let Some(target_manifest) = manifests.get(&target_module_id) else {
1127 continue;
1128 };
1129 let mut closed_routes = Vec::new();
1130 let mut module_goodbyes = Vec::new();
1131 for route in routes {
1132 let Principal::Reserved {
1133 module_id: opening_module_id,
1134 } = &route.principal
1135 else {
1136 continue;
1137 };
1138 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1139 continue;
1140 };
1141 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1142 continue;
1143 };
1144
1145 match self.forwarding.release_client_route(
1146 route.goodbye_target.connection_id,
1147 route.goodbye_target.channel,
1148 route.goodbye_target.epoch,
1149 ) {
1150 Ok(RouteRelease::Removed(module_goodbye)) => {
1151 warn!(
1152 opening_module_id,
1153 target_module_id,
1154 capability,
1155 "force-closing route because an attested capability deny edge now matches"
1156 );
1157 closed_routes.push(route);
1158 module_goodbyes.push(module_goodbye);
1159 }
1160 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1161 Err(err) => warn!(
1162 opening_module_id,
1163 target_module_id,
1164 capability,
1165 error = %err,
1166 "failed to force-close capability-denied route"
1167 ),
1168 }
1169 }
1170
1171 if closed_routes.is_empty() {
1172 continue;
1173 }
1174 send_route_control_pushes(
1175 &self.forwarding,
1176 closed_routes,
1177 ClientControlPush::RouteClosed {
1178 module_id: target_module_id,
1179 channels: Vec::new(),
1180 reason: RouteCloseReason::CapabilityDenied,
1181 drained: false,
1182 abandoned: 0,
1183 excluded_subscriptions: 0,
1184 terminal: Some(false),
1185 },
1186 );
1187 self.emit_route_goodbyes(module_goodbyes);
1188 }
1189 }
1190
1191 fn not_ready_reason(
1197 &self,
1198 registration: &crate::registry::ModuleRegistration,
1199 ) -> Option<NotReadyReason> {
1200 if !registration.ready {
1201 return Some(NotReadyReason {
1202 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1203 capability: None,
1204 });
1205 }
1206 self.first_unprovided_required_capability(registration)
1207 .map(|capability| NotReadyReason {
1208 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1209 capability: Some(capability),
1210 })
1211 }
1212
1213 fn first_unprovided_required_capability(
1231 &self,
1232 registration: &crate::registry::ModuleRegistration,
1233 ) -> Option<String> {
1234 let required = registration
1235 .manifest
1236 .capabilities
1237 .iter()
1238 .flat_map(|declarations| declarations.requires.iter())
1239 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1240 .map(|requirement| requirement.capability.as_str())
1241 .collect::<BTreeSet<_>>();
1242 if required.is_empty() {
1243 return None;
1244 }
1245 let module_id = registration.manifest.module_id.as_str();
1246 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1247 if required
1248 .iter()
1249 .any(|capability| verdict(capability).is_none())
1250 {
1251 self.refresh_capability_requirements();
1252 }
1253 required
1254 .into_iter()
1255 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1256 .map(str::to_string)
1257 }
1258
1259 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1260 self.capability_evaluator
1261 .statuses()
1262 .into_iter()
1263 .map(capability_requirement_status)
1264 .collect()
1265 }
1266
1267 fn deregister_connection(
1274 &self,
1275 connection_id: ConnectionId,
1276 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1277 self.registry.deregister_connection(connection_id)
1278 }
1279
1280 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1281 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1282 return None;
1283 }
1284 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1285 parse_client_control_request(&frame.body)
1286 else {
1287 return None;
1288 };
1289 Some(target_module_id(&target).to_string())
1290 }
1291
1292 pub(crate) fn route_open_capacity_refusal(
1293 &self,
1294 ctx: &RouteCtx,
1295 frame: &Frame,
1296 target_module_id: &str,
1297 in_flight: usize,
1298 limit: usize,
1299 ) -> Result<Frame, RouterError> {
1300 self.route_open_admission_refusal_frame(
1301 ctx,
1302 frame,
1303 target_module_id,
1304 "open_admission_full",
1305 (in_flight, limit),
1306 format!(
1307 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1308 ),
1309 )
1310 }
1311
1312 fn route_open_target_capacity_refusal(
1313 &self,
1314 ctx: &RouteCtx,
1315 frame: &Frame,
1316 target_module_id: &str,
1317 in_flight: usize,
1318 ) -> Result<Frame, RouterError> {
1319 self.route_open_admission_refusal_frame(
1320 ctx,
1321 frame,
1322 target_module_id,
1323 "target_binds_full",
1324 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1325 format!(
1326 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1327 ),
1328 )
1329 }
1330
1331 fn route_open_admission_refusal_frame(
1339 &self,
1340 ctx: &RouteCtx,
1341 frame: &Frame,
1342 target_module_id: &str,
1343 reason: &'static str,
1344 (in_flight, limit): (usize, usize),
1345 message: impl Into<String>,
1346 ) -> Result<Frame, RouterError> {
1347 let code = error_codes::TARGET_UNAVAILABLE;
1348 self.counters.increment_route_open_refused(code);
1349 info!(
1350 target: "control",
1351 code,
1352 reason,
1353 module_id = ?target_module_id,
1354 connection_id = ctx.connection_id.get(),
1355 in_flight,
1356 limit,
1357 "route.open refused"
1358 );
1359 control_error_frame(frame, code, message.into())
1360 }
1361
1362 #[cfg(test)]
1368 pub fn handle_control(
1369 &self,
1370 connection_id: ConnectionId,
1371 frame: Frame,
1372 ) -> Result<Vec<Frame>, RouterError> {
1373 match frame.header.ty {
1374 FrameType::Ping => Ok(vec![pong(&frame)?]),
1375 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1376 FrameType::Goodbye => self.handle_goodbye(connection_id),
1377 ty => Ok(vec![control_error_frame(
1378 &frame,
1379 "unsupported_control_frame",
1380 format!("unsupported channel-0 frame {ty:?}"),
1381 )?]),
1382 }
1383 }
1384
1385 pub async fn handle_control_frame(
1386 &self,
1387 ctx: &RouteCtx,
1388 frame: Frame,
1389 ) -> Result<Vec<Frame>, RouterError> {
1390 self.handle_control_frame_timed(ctx, frame, None).await
1391 }
1392
1393 pub(crate) async fn handle_control_frame_timed(
1394 &self,
1395 ctx: &RouteCtx,
1396 frame: Frame,
1397 dispatch_started_at: Option<StdInstant>,
1398 ) -> Result<Vec<Frame>, RouterError> {
1399 match frame.header.ty {
1400 FrameType::Ping => Ok(vec![pong(&frame)?]),
1401 FrameType::Hello => {
1402 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1403 }
1404 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1405 FrameType::Cancel => {
1406 if self
1407 .supervisor
1408 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1409 {
1410 Ok(Vec::new())
1411 } else {
1412 Ok(vec![control_error_frame(
1413 &frame,
1414 "unknown_subscription",
1415 "no supervisor spawn subscription has this correlation id",
1416 )?])
1417 }
1418 }
1419 FrameType::Request => {
1420 if self
1421 .forwarding
1422 .module_endpoint_for_connection(ctx.connection_id)
1423 .map_err(RouterError::Forwarding)?
1424 .is_some()
1425 {
1426 if !is_known_module_request_op(&frame.body) {
1427 return Ok(vec![control_error_frame(
1428 &frame,
1429 "unsupported_control_frame",
1430 "module-originated channel-0 REQUEST is not supported",
1431 )?]);
1432 }
1433 let request = match parse_module_control_request_from_module(&frame.body) {
1434 Ok(request) => request,
1435 Err((err, ControlRequestBodyError::UnknownOp)) => {
1436 return Ok(vec![control_error_frame(
1437 &frame,
1438 "unsupported_control_frame",
1439 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1440 )?])
1441 }
1442 Err((err, ControlRequestBodyError::InvalidBody)) => {
1443 return Ok(vec![control_error_frame(
1444 &frame,
1445 "invalid_control_body",
1446 format!("malformed module control body: {err}"),
1447 )?])
1448 }
1449 };
1450 let op = module_control_request_op(&request);
1451 let corr = frame.header.corr;
1452 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1453 let result =
1454 self.handle_module_control_request(ctx.connection_id, frame, request);
1455 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1456 return result;
1457 }
1458
1459 if is_known_module_request_op(&frame.body) {
1460 return Ok(vec![control_error_frame(
1461 &frame,
1462 "not_registered",
1463 "catalog.update requires an active module registration owned by this connection",
1464 )?]);
1465 }
1466
1467 let request = match parse_client_control_request(&frame.body) {
1468 Ok(request) => request,
1469 Err((err, ControlRequestBodyError::UnknownOp)) => {
1470 return Ok(vec![control_error_frame(
1471 &frame,
1472 "unknown_control_op",
1473 format!("unknown client control op: {err}"),
1474 )?])
1475 }
1476 Err((err, ControlRequestBodyError::InvalidBody)) => {
1477 return Ok(vec![control_error_frame(
1478 &frame,
1479 "invalid_control_body",
1480 format!("malformed client control body: {err}"),
1481 )?])
1482 }
1483 };
1484 let op = client_control_request_op(&request);
1485 let corr = frame.header.corr;
1486 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1487 #[cfg(test)]
1488 if let Some(delay) = self.control_dispatch_delay {
1489 tokio::time::sleep(delay).await;
1490 }
1491 let result = self
1492 .handle_client_control_request(ctx, frame, request)
1493 .await;
1494 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1495 result
1496 }
1497 FrameType::Push => {
1498 let Some(endpoint) = self
1499 .forwarding
1500 .module_endpoint_for_connection(ctx.connection_id)
1501 .map_err(RouterError::Forwarding)?
1502 else {
1503 return Ok(vec![control_error_frame(
1504 &frame,
1505 "unsupported_control_frame",
1506 "client-originated channel-0 PUSH is not supported",
1507 )?]);
1508 };
1509 self.handle_status_update(endpoint, frame)
1510 }
1511 FrameType::Response | FrameType::Error
1512 if self
1513 .forwarding
1514 .module_endpoint_for_connection(ctx.connection_id)
1515 .map_err(RouterError::Forwarding)?
1516 .is_some() =>
1517 {
1518 self.handle_module_relay_response(ctx.connection_id, frame)
1519 }
1520 ty => Ok(vec![control_error_frame(
1521 &frame,
1522 "unsupported_control_frame",
1523 format!("unsupported channel-0 frame {ty:?}"),
1524 )?]),
1525 }
1526 }
1527
1528 pub fn cleanup_connection(
1529 &self,
1530 connection_id: ConnectionId,
1531 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1532 let crash_closed = self
1533 .registry
1534 .get_module_by_connection(connection_id)?
1535 .and_then(|registration| {
1536 self.forwarding
1537 .module_endpoint_for_connection(connection_id)
1538 .ok()
1539 .flatten()
1540 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1541 .map(|routes| (registration.manifest.module_id, routes))
1542 });
1543 let crash_closed = crash_closed.map(|(module_id, routes)| {
1544 let terminal = match self.supervisor.get(&module_id) {
1545 None => false,
1546 Some(module) => match module.will_recover_after_connection_loss() {
1547 Ok(will_recover) => !will_recover,
1548 Err(err) => {
1549 warn!(
1550 %module_id,
1551 error = %err,
1552 "failed to read crash recovery verdict; reporting non-terminal conservatively"
1553 );
1554 false
1555 }
1556 },
1557 };
1558 let reason = match self.forwarding.is_daemon_draining() {
1562 Ok(true) => RouteCloseReason::Restart,
1563 Ok(false) => RouteCloseReason::Crash,
1564 Err(err) => {
1565 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
1566 RouteCloseReason::Crash
1567 }
1568 };
1569 (module_id, routes, reason, terminal)
1570 });
1571 let registrations = self.deregister_connection(connection_id);
1572 let cleanup = if crash_closed.is_some() {
1573 self.forwarding.cleanup_connection_counted(connection_id)
1574 } else {
1575 self.forwarding
1581 .cleanup_connection(connection_id)
1582 .map(|released| crate::forwarding::ConnectionCleanup {
1583 released,
1584 abandoned_relays: 0,
1585 })
1586 };
1587 if let Some((module_id, routes, reason, terminal)) = crash_closed {
1592 let abandoned = cleanup
1593 .as_ref()
1594 .map_or(0, |cleanup| cleanup.abandoned_relays);
1595 send_route_control_pushes(
1596 &self.forwarding,
1597 routes,
1598 ClientControlPush::RouteClosed {
1599 module_id,
1600 channels: Vec::new(),
1601 reason,
1602 drained: false,
1603 abandoned,
1604 excluded_subscriptions: 0,
1605 terminal: Some(terminal),
1606 },
1607 );
1608 }
1609 if let Ok(cleanup) = cleanup {
1610 self.emit_route_goodbyes(cleanup.released);
1611 }
1612 if matches!(®istrations, Ok(r) if !r.is_empty()) {
1616 crate::supervise::notify_registration_release();
1617 self.capability_evaluator.wake_deadline_loop();
1618 self.refresh_capability_requirements();
1619 }
1620 self.supervisor.remove_spawn_subscribers(connection_id);
1621 self.hello_launch_nonces
1624 .lock()
1625 .unwrap_or_else(|poisoned| poisoned.into_inner())
1626 .forget(connection_id);
1627 self.scopes
1628 .write()
1629 .unwrap_or_else(|poisoned| poisoned.into_inner())
1630 .release_connection(connection_id);
1631 registrations
1632 }
1633
1634 pub(crate) fn handle_route_goodbye(
1635 &self,
1636 connection_id: ConnectionId,
1637 route_channel: u16,
1638 route_epoch: u32,
1639 ) -> Result<bool, RouterError> {
1640 debug!(
1641 connection_id = connection_id.get(),
1642 route_channel, route_epoch, "handling route GOODBYE"
1643 );
1644 let RouteRelease::Removed(released_route) = self
1645 .forwarding
1646 .release_client_route(connection_id, route_channel, route_epoch)
1647 .map_err(RouterError::Forwarding)?
1648 else {
1649 return Ok(false);
1650 };
1651 self.emit_route_goodbyes(vec![released_route]);
1652 Ok(true)
1653 }
1654
1655 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
1656 for released in released_routes {
1657 let frame = match Frame::build_with_version(
1658 released.negotiated_ver,
1659 FrameType::Goodbye,
1660 control_flags(),
1661 released.channel,
1662 released.epoch,
1663 0,
1664 Vec::new(),
1665 ) {
1666 Ok(frame) => frame,
1667 Err(err) => {
1668 warn!(
1669 route_channel = released.channel,
1670 error = %err,
1671 "failed to build route GOODBYE frame"
1672 );
1673 continue;
1674 }
1675 };
1676 if !released.close_on_delivery_failure() {
1677 crate::forwarding::send_module_route_goodbye(
1678 &self.counters,
1679 &released.sink,
1680 frame,
1681 released.module_id.as_deref(),
1682 "client route released",
1683 );
1684 continue;
1685 }
1686 if let Err(err) = released.sink.try_send(frame) {
1687 warn!(
1688 target_connection_id = released.connection_id.get(),
1689 route_channel = released.channel,
1690 error = %err,
1691 "route GOODBYE was not delivered to client; closing target connection"
1692 );
1693 if self
1694 .forwarding
1695 .escalate_client_delivery_failure(
1696 released.connection_id,
1697 released.channel,
1698 released.epoch,
1699 CloseReason::new(
1700 "route_goodbye_delivery_failed",
1701 format!(
1702 "failed to enqueue route GOODBYE for channel {}: {err}",
1703 released.channel
1704 ),
1705 ),
1706 crate::forwarding::UndeliveredFrame {
1707 module_id: released.module_id.as_deref(),
1708 sink: &released.sink,
1709 },
1710 )
1711 .unwrap_or(false)
1712 {
1713 self.counters.increment_goodbye_relay_client_failed();
1714 }
1715 }
1716 }
1717 }
1718
1719 fn send_abandoned_route_bind_goodbye(
1731 &self,
1732 module_sink: &crate::FrameSink,
1733 negotiated_ver: u8,
1734 module_channel: u16,
1735 module_epoch: u32,
1736 ) {
1737 let frame = match Frame::build_with_version(
1738 negotiated_ver,
1739 FrameType::Goodbye,
1740 control_flags(),
1741 module_channel,
1742 module_epoch,
1743 0,
1744 Vec::new(),
1745 ) {
1746 Ok(frame) => frame,
1747 Err(err) => {
1748 warn!(
1749 route_channel = module_channel,
1750 error = %err,
1751 "failed to build GOODBYE for abandoned route.bind"
1752 );
1753 return;
1754 }
1755 };
1756 crate::forwarding::send_module_route_goodbye(
1757 &self.counters,
1758 module_sink,
1759 frame,
1760 None,
1761 "abandoned route.bind",
1762 );
1763 }
1764
1765 fn handle_hello(
1766 &self,
1767 connection_id: ConnectionId,
1768 sink: Option<crate::FrameSink>,
1769 frame: Frame,
1770 ) -> Result<Vec<Frame>, RouterError> {
1771 debug!(
1772 connection_id = connection_id.get(),
1773 corr = frame.header.corr,
1774 "handling HELLO"
1775 );
1776 if self
1780 .registry
1781 .get_module_by_connection(connection_id)
1782 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
1783 .is_some()
1784 {
1785 return Ok(vec![control_error_frame(
1786 &frame,
1787 "invalid_hello",
1788 "connection is already registered as a module",
1789 )?]);
1790 }
1791 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
1792 Ok(value) => value,
1793 Err(err) => {
1794 return Ok(vec![control_error_frame(
1795 &frame,
1796 "invalid_hello",
1797 format!("malformed HELLO body: {err}"),
1798 )?])
1799 }
1800 };
1801 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
1802 return Ok(vec![control_error_frame(
1803 &frame,
1804 "invalid_capability_grammar",
1805 err.to_string(),
1806 )?]);
1807 }
1808 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
1809 return Ok(vec![control_error_frame(
1810 &frame,
1811 "invalid_manifest",
1812 err.to_string(),
1813 )?]);
1814 }
1815 if let Some(provenance) = hello_value
1816 .get("manifest")
1817 .and_then(|manifest| manifest.get("provenance"))
1818 {
1819 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
1820 return Ok(vec![control_error_frame(
1821 &frame,
1822 "invalid_manifest",
1823 format!("malformed manifest provenance: {err}"),
1824 )?]);
1825 }
1826 }
1827 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
1828 Ok(hello) => hello,
1829 Err(err) => {
1830 return Ok(vec![control_error_frame(
1831 &frame,
1832 "invalid_hello",
1833 format!("malformed HELLO body: {err}"),
1834 )?])
1835 }
1836 };
1837
1838 if hello.protocol_ver != hello.manifest.protocol_ver {
1839 return Ok(vec![control_error_frame(
1840 &frame,
1841 "invalid_manifest",
1842 format!(
1843 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
1844 hello.protocol_ver, hello.manifest.protocol_ver
1845 ),
1846 )?]);
1847 }
1848
1849 if hello.manifest.module_id.trim().is_empty() {
1850 return Ok(vec![control_error_frame(
1851 &frame,
1852 "invalid_manifest",
1853 "manifest module_id must not be empty",
1854 )?]);
1855 }
1856
1857 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
1858 Ok(negotiated_ver) => negotiated_ver,
1859 Err(message) => {
1860 return Ok(vec![control_error_frame(
1861 &frame,
1862 "version_unsupported",
1863 message,
1864 )?])
1865 }
1866 };
1867
1868 let swap_admission = self
1876 .supervisor
1877 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
1878 if swap_admission == SwapHelloAdmission::Refused {
1879 warn!(
1880 module_id = %hello.manifest.module_id,
1881 connection_id = connection_id.get(),
1882 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
1883 );
1884 return Ok(vec![control_error_frame(
1885 &frame,
1886 "swap_token_invalid",
1887 format!(
1888 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
1889 hello.manifest.module_id
1890 ),
1891 )?]);
1892 }
1893 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
1894
1895 if let Some(rejection) = (!swap_candidate)
1903 .then(|| {
1904 self.supervisor.reserved_hello_rejection(
1905 &hello.manifest.module_id,
1906 hello.launch_nonce.as_deref(),
1907 )
1908 })
1909 .flatten()
1910 {
1911 let message = match rejection {
1912 ReservedHelloRejection::Exact { module_id } => format!(
1913 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
1914 ),
1915 ReservedHelloRejection::Prefix {
1916 prefix,
1917 owner_module_id,
1918 } => format!(
1919 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
1920 hello.manifest.module_id
1921 ),
1922 };
1923 return Ok(vec![control_error_frame(
1924 &frame,
1925 "reserved_module",
1926 message,
1927 )?]);
1928 }
1929
1930 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
1931 &hello.manifest.module_id,
1932 hello.manifest.capabilities.as_ref(),
1933 );
1934 if let Some(refusal) = reserved_capability_refusals.first() {
1935 let capability = refusal.capability.clone();
1936 let bound_module = refusal.claimants[0].clone();
1937 log_duplicate_claim_events(reserved_capability_refusals);
1938 return Ok(vec![control_error_frame(
1939 &frame,
1940 "reserved_capability",
1941 format!(
1942 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
1943 capability, bound_module, hello.manifest.module_id
1944 ),
1945 )?]);
1946 }
1947
1948 if self
1951 .forwarding
1952 .connection_has_client_routes(connection_id)
1953 .map_err(RouterError::Forwarding)?
1954 {
1955 return Ok(vec![control_error_frame(
1956 &frame,
1957 "invalid_hello",
1958 "connection has open client routes and cannot also register as a module",
1959 )?]);
1960 }
1961
1962 self.hello_launch_nonces
1967 .lock()
1968 .unwrap_or_else(|poisoned| poisoned.into_inner())
1969 .record(connection_id, hello.launch_nonce.as_deref());
1970 let control_ops = effective_module_control_ops(hello.control_ops);
1971 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
1974 if swap_candidate {
1975 return self.register_swap_candidate(
1976 connection_id,
1977 sink,
1978 &frame,
1979 hello.manifest,
1980 negotiated_ver,
1981 control_ops,
1982 hello_ack,
1983 );
1984 }
1985 let registration = match self.registry.register_with_control_ops(
1986 hello.manifest,
1987 negotiated_ver,
1988 connection_id,
1989 control_ops,
1990 ) {
1991 Ok(registration) => registration,
1992 Err(RegistryError::DuplicateModuleId { module_id }) => {
1993 return Ok(vec![control_error_frame(
1994 &frame,
1995 "duplicate_module_id",
1996 format!(
1997 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
1998 ),
1999 )?])
2000 }
2001 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2002 return Ok(vec![control_error_frame(
2003 &frame,
2004 "invalid_module_id",
2005 err.to_string(),
2006 )?])
2007 }
2008 Err(err) => {
2009 return Ok(vec![control_error_frame(
2010 &frame,
2011 "registry_error",
2012 err.to_string(),
2013 )?])
2014 }
2015 };
2016
2017 let reply = if let Some(sink) = sink {
2018 let concurrency = manifest_concurrency(®istration.manifest);
2034 if let Err(err) = self.forwarding.register_module_connection_acked(
2035 connection_id,
2036 registration.manifest.module_id.clone(),
2037 negotiated_ver,
2038 concurrency,
2039 sink,
2040 hello_ack,
2041 ) {
2042 if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
2046 crate::supervise::notify_registration_release();
2047 }
2048 return Ok(vec![control_error_frame(
2049 &frame,
2050 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2051 "invalid_hello"
2052 } else {
2053 forwarding_error_code(&err)
2054 },
2055 err.to_string(),
2056 )?]);
2057 }
2058 Vec::new()
2059 } else {
2060 vec![hello_ack]
2063 };
2064
2065 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2074 info!(
2075 module_id = %registration.manifest.module_id,
2076 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2077 );
2078 }
2079
2080 self.apply_registration_capabilities(®istration);
2081
2082 info!(
2083 module_id = %registration.manifest.module_id,
2084 module_version = %registration.manifest.module_version,
2085 negotiated_ver,
2086 routable_provider = manifest_provides_routable_role(®istration.manifest),
2087 connection_id = connection_id.get(),
2088 "module registered"
2089 );
2090
2091 Ok(reply)
2092 }
2093
2094 #[allow(clippy::too_many_arguments)]
2104 fn register_swap_candidate(
2105 &self,
2106 connection_id: ConnectionId,
2107 sink: Option<crate::FrameSink>,
2108 frame: &Frame,
2109 manifest: ModuleManifest,
2110 negotiated_ver: u8,
2111 control_ops: Vec<String>,
2112 hello_ack: Frame,
2113 ) -> Result<Vec<Frame>, RouterError> {
2114 let module_id = manifest.module_id.clone();
2115 let registration = match self.registry.register_candidate_with_control_ops(
2116 manifest,
2117 negotiated_ver,
2118 connection_id,
2119 control_ops,
2120 ) {
2121 Ok(registration) => registration,
2122 Err(RegistryError::DuplicateModuleId { module_id }) => {
2123 return Ok(vec![control_error_frame(
2124 frame,
2125 "duplicate_module_id",
2126 format!(
2127 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2128 ),
2129 )?])
2130 }
2131 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2132 return Ok(vec![control_error_frame(
2133 frame,
2134 "invalid_module_id",
2135 err.to_string(),
2136 )?])
2137 }
2138 Err(err) => {
2139 return Ok(vec![control_error_frame(
2140 frame,
2141 "registry_error",
2142 err.to_string(),
2143 )?])
2144 }
2145 };
2146 let reply = if let Some(sink) = sink {
2147 let concurrency = manifest_concurrency(®istration.manifest);
2151 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2152 connection_id,
2153 module_id.clone(),
2154 negotiated_ver,
2155 concurrency,
2156 sink,
2157 hello_ack,
2158 ) {
2159 if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
2160 crate::supervise::notify_registration_release();
2161 }
2162 return Ok(vec![control_error_frame(
2163 frame,
2164 forwarding_error_code(&err),
2165 err.to_string(),
2166 )?]);
2167 }
2168 Vec::new()
2169 } else {
2170 vec![hello_ack]
2171 };
2172 self.supervisor.mark_swap_candidate_admitted(&module_id);
2173 info!(
2174 module_id = %module_id,
2175 module_version = %registration.manifest.module_version,
2176 negotiated_ver,
2177 ready = registration.ready,
2178 connection_id = connection_id.get(),
2179 "swap candidate registered; not routable until cutover"
2180 );
2181 Ok(reply)
2182 }
2183
2184 fn build_hello_ack(
2185 &self,
2186 frame: &Frame,
2187 negotiated_ver: u8,
2188 module_id: &str,
2189 ) -> Result<Frame, RouterError> {
2190 let ack = ModuleHelloAckBody {
2191 negotiated_ver,
2192 subc_ops: module_subc_ops(),
2193 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2194 storage: self
2195 .storage_config
2196 .as_ref()
2197 .map(|cfg| cfg.descriptor_for(module_id)),
2198 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2199 };
2200 let body = serde_json::to_vec(&ack).map_err(|err| {
2201 RouterError::backend(
2202 0,
2203 frame.header.corr,
2204 format!("failed to encode HELLO_ACK: {err}"),
2205 )
2206 })?;
2207
2208 Frame::build_with_version(
2209 negotiated_ver,
2210 FrameType::HelloAck,
2211 control_flags(),
2212 0,
2213 0,
2214 frame.header.corr,
2215 body,
2216 )
2217 .map_err(RouterError::FrameBuild)
2218 }
2219
2220 async fn handle_client_control_request(
2221 &self,
2222 ctx: &RouteCtx,
2223 frame: Frame,
2224 request: ClientControlRequest,
2225 ) -> Result<Vec<Frame>, RouterError> {
2226 match request {
2227 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2228 ClientControlRequest::CatalogList { module_id } => {
2229 self.handle_catalog_list(frame, module_id)
2230 }
2231 ClientControlRequest::RouteOpen {
2232 target,
2233 identity,
2234 consumer_identity,
2235 consumer_capabilities,
2236 role_versions,
2237 admission_facts,
2238 scope,
2239 } => {
2240 self.handle_route_open(
2241 ctx,
2242 frame,
2243 RouteOpenRequest {
2244 target,
2245 identity,
2246 consumer_identity,
2247 consumer_capabilities,
2248 role_versions,
2249 admission_facts,
2250 scope,
2251 },
2252 )
2253 .await
2254 }
2255 ClientControlRequest::RoutePoll {
2256 route_channel,
2257 route_epoch,
2258 kind,
2259 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2260 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2261 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2262 self.handle_supervisor_spawn_snapshot(frame)
2263 }
2264 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2265 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2266 }
2267 ClientControlRequest::SupervisorRestart {
2268 module_id,
2269 drain_timeout_ms,
2270 } => {
2271 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms)
2272 .await
2273 }
2274 ClientControlRequest::SupervisorSwap {
2275 module_id,
2276 ready_timeout_ms,
2277 } => {
2278 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms)
2279 .await
2280 }
2281 ClientControlRequest::SupervisorReload { module_id } => {
2282 self.handle_supervisor_reload(frame, module_id).await
2283 }
2284 ClientControlRequest::SupervisorRescan { preview } => {
2285 self.handle_supervisor_rescan(frame, preview).await
2286 }
2287 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2288 self.handle_supervisor_release_reserved(frame, module_id)
2289 .await
2290 }
2291 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2292 self.handle_supervisor_set_enabled(frame, module_id, enabled)
2293 .await
2294 }
2295 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2296 self.handle_supervisor_health_probe(frame, module_id).await
2297 }
2298 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2299 ClientControlRequest::SupervisorRoutes { module_id } => {
2300 self.handle_supervisor_routes(frame, module_id)
2301 }
2302 ClientControlRequest::SupervisorProvenance { module_id } => {
2303 self.handle_supervisor_provenance(frame, module_id).await
2304 }
2305 ClientControlRequest::SupervisorStderrTail {
2306 module_id,
2307 max_lines,
2308 max_bytes,
2309 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2310 ClientControlRequest::SupervisorTerminals { module_id } => {
2311 self.handle_supervisor_terminals(frame, module_id).await
2312 }
2313 }
2314 }
2315
2316 fn handle_module_control_request(
2317 &self,
2318 connection_id: ConnectionId,
2319 frame: Frame,
2320 request: ModuleControlRequestFromModule,
2321 ) -> Result<Vec<Frame>, RouterError> {
2322 match request {
2323 ModuleControlRequestFromModule::CatalogUpdate {
2324 provides,
2325 capabilities,
2326 ready,
2327 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2328 ModuleControlRequestFromModule::LiveRoots {} => {
2329 let registered = self
2330 .registry
2331 .get_module_by_connection(connection_id)
2332 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2333 let Some(registration) = registered else {
2334 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2335 };
2336 let response = self
2337 .forwarding
2338 .live_roots(®istration.manifest.module_id)
2339 .map_err(RouterError::Forwarding)?;
2340 Ok(vec![control_response_body_frame(
2341 &frame,
2342 &response,
2343 "ModuleControlResponseToModule::LiveRoots",
2344 )?])
2345 }
2346 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
2347 self.handle_scope_sync(connection_id, frame, generation, scopes)
2348 }
2349 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
2350 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
2351 }
2352 }
2353 }
2354
2355 fn handle_scope_sync(
2359 &self,
2360 connection_id: ConnectionId,
2361 frame: Frame,
2362 generation: u64,
2363 scopes: Vec<ScopeRecord>,
2364 ) -> Result<Vec<Frame>, RouterError> {
2365 let Some(registration) = self
2366 .registry
2367 .get_module_by_connection(connection_id)
2368 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2369 else {
2370 return Ok(vec![control_error_frame(
2371 &frame,
2372 "not_registered",
2373 "scope.sync requires an active module registration owned by this connection",
2374 )?]);
2375 };
2376 let owner = registration.manifest.module_id;
2377 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
2378 let is_current_launch = |connection: ConnectionId| {
2379 self.hello_launch_nonces
2380 .lock()
2381 .unwrap_or_else(|poisoned| poisoned.into_inner())
2382 .presented(connection, current_nonce.as_deref())
2383 };
2384 let mut table = self
2389 .scopes
2390 .write()
2391 .unwrap_or_else(|poisoned| poisoned.into_inner());
2392 let outcome = table.sync(&owner, connection_id, is_current_launch, generation, scopes);
2393 let drained = match &outcome {
2394 Ok(applied) => self
2395 .forwarding
2396 .publish_scope_changes(&applied.tag_changes)
2397 .map_err(RouterError::Forwarding)?,
2398 Err(_) => Vec::new(),
2399 };
2400 drop(table);
2401 match outcome {
2402 Ok(applied) => {
2403 let counts = ScopeOutcomeCounts::of(&applied.results);
2404 info!(
2405 owner = %owner,
2406 generation,
2407 records = applied.results.len(),
2408 created = counts.created,
2409 replaced = counts.replaced,
2410 updated = counts.updated,
2411 unchanged = counts.unchanged,
2412 refused = counts.refused,
2413 ended = applied.ended.len(),
2414 tag_changes = applied.tag_changes.len(),
2415 routes_closed = drained.len(),
2416 "scope sync accepted"
2417 );
2418 for refused in applied
2424 .results
2425 .iter()
2426 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
2427 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
2428 {
2429 warn!(
2430 owner = %owner,
2431 generation,
2432 scope_ref = %refused.scope_ref,
2433 scope_epoch = refused.scope_epoch,
2434 code = refused.code.as_deref().unwrap_or(""),
2435 "scope record refused"
2436 );
2437 }
2438 self.close_scope_drained_routes(drained);
2439 let response = ModuleControlResponseToModule::ScopeSync {
2440 generation,
2441 results: applied.results,
2442 ended: applied.ended,
2443 };
2444 Ok(vec![control_response_body_frame(
2445 &frame,
2446 &response,
2447 "ModuleControlResponseToModule::ScopeSync",
2448 )?])
2449 }
2450 Err(refusal) => {
2451 info!(
2452 owner = %owner,
2453 generation,
2454 code = refusal.code,
2455 "scope sync refused"
2456 );
2457 Ok(vec![control_error_frame(
2458 &frame,
2459 refusal.code,
2460 refusal.message,
2461 )?])
2462 }
2463 }
2464 }
2465
2466 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
2472 if drained.is_empty() {
2473 return;
2474 }
2475 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
2476 BTreeMap::new();
2477 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
2478 for route in drained {
2479 warn!(
2480 module_id = %route.module_id,
2481 reason = ?route.reason,
2482 client_connection_id = route.client.connection_id.get(),
2483 route_channel = route.client.channel,
2484 "closing route because its scope changed"
2485 );
2486 pushes
2487 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
2488 .or_insert_with(|| (route.reason, Vec::new()))
2489 .1
2490 .push(EndpointRoute {
2491 goodbye_target: route.client.clone(),
2492 principal: Principal::Unverified,
2493 bound_at: Instant::now(),
2494 draining: false,
2495 drain_reason: None,
2496 });
2497 goodbyes.push(route.module);
2498 goodbyes.push(route.client);
2499 }
2500 for ((module_id, _), (reason, routes)) in pushes {
2501 send_route_control_pushes(
2502 &self.forwarding,
2503 routes,
2504 ClientControlPush::RouteClosed {
2505 module_id,
2506 channels: Vec::new(),
2507 reason,
2508 drained: false,
2509 abandoned: 0,
2510 excluded_subscriptions: 0,
2511 terminal: Some(false),
2512 },
2513 );
2514 }
2515 self.emit_route_goodbyes(goodbyes);
2516 }
2517
2518 fn handle_scope_describe(
2521 &self,
2522 connection_id: ConnectionId,
2523 frame: Frame,
2524 owner: Principal,
2525 scope_ref: String,
2526 ) -> Result<Vec<Frame>, RouterError> {
2527 let registered = self
2528 .registry
2529 .get_module_by_connection(connection_id)
2530 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2531 if registered.is_none() {
2532 return Ok(vec![control_error_frame(
2533 &frame,
2534 "not_registered",
2535 "scope.describe requires an active module registration owned by this connection",
2536 )?]);
2537 }
2538 let description = self
2539 .scopes
2540 .read()
2541 .unwrap_or_else(|poisoned| poisoned.into_inner())
2542 .describe(&owner, &scope_ref);
2543 let owner_configured = match &owner {
2544 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
2549 _ => false,
2550 };
2551 let response = ModuleControlResponseToModule::ScopeDescribe {
2552 status: description.status,
2553 scope_epoch: description.scope_epoch,
2554 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
2555 owner_synced: description.owner_synced,
2556 owner_configured,
2557 scope: description.stamp,
2558 };
2559 Ok(vec![control_response_body_frame(
2560 &frame,
2561 &response,
2562 "ModuleControlResponseToModule::ScopeDescribe",
2563 )?])
2564 }
2565
2566 fn handle_catalog_update(
2567 &self,
2568 connection_id: ConnectionId,
2569 frame: Frame,
2570 provides: Vec<ProviderRole>,
2571 capabilities: Option<CapabilityDeclarations>,
2572 ready: Option<bool>,
2573 ) -> Result<Vec<Frame>, RouterError> {
2574 self.refresh_capability_requirements();
2575 let Some(registration) = self
2576 .registry
2577 .get_module_by_connection(connection_id)
2578 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2579 else {
2580 return Ok(vec![control_error_frame(
2581 &frame,
2582 "not_registered",
2583 "catalog.update requires an active module registration owned by this connection",
2584 )?]);
2585 };
2586
2587 if let Some(message) =
2588 catalog_update_frozen_field_message(®istration.manifest, &provides)
2589 {
2590 return Ok(vec![control_error_frame(
2591 &frame,
2592 "catalog_update_frozen_field",
2593 message,
2594 )?]);
2595 }
2596
2597 let mut candidate = registration.manifest.clone();
2598 candidate.provides = provides.clone();
2599 candidate.capabilities = capabilities
2600 .clone()
2601 .or_else(|| registration.manifest.capabilities.clone());
2602 if let Err(err) = candidate.validate_capability_grammar() {
2603 return Ok(vec![control_error_frame(
2604 &frame,
2605 "invalid_capability_grammar",
2606 err.to_string(),
2607 )?]);
2608 }
2609
2610 let mut conflicts = self
2613 .capability_evaluator
2614 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
2615 if let Some(conflict) = conflicts.first() {
2616 let message = format!(
2617 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2618 conflict.capability, conflict.claimants[0], candidate.module_id
2619 );
2620 for conflict in &mut conflicts {
2621 conflict.source = DuplicateClaimSource::CatalogUpdate;
2622 }
2623 log_duplicate_claim_events(conflicts);
2624 return Ok(vec![control_error_frame(
2625 &frame,
2626 "reserved_capability",
2627 message,
2628 )?]);
2629 }
2630
2631 let updated = self
2632 .registry
2633 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
2634 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2635 if updated.is_none() {
2636 return Ok(vec![control_error_frame(
2637 &frame,
2638 "not_registered",
2639 "catalog.update requires an active module registration owned by this connection",
2640 )?]);
2641 }
2642 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
2643 log_duplicate_claim_events(
2644 self.capability_evaluator
2645 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
2646 );
2647 }
2648 if capability_census_trigger(
2649 registration.manifest.capabilities.as_ref(),
2650 updated
2651 .as_ref()
2652 .and_then(|entry| entry.manifest.capabilities.as_ref()),
2653 ) {
2654 self.enforce_capability_denies();
2655 }
2656 self.refresh_capability_requirements();
2657
2658 let response = ModuleControlResponseToModule::CatalogUpdate {};
2659 control_response_body_frame(
2660 &frame,
2661 &response,
2662 "ModuleControlResponseToModule::CatalogUpdate",
2663 )
2664 .map(|frame| vec![frame])
2665 }
2666
2667 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
2668 self.refresh_capability_requirements();
2669 let mut counters = self.counters.snapshot();
2675 if let (Ok((connections_with_routes, max)), Some(obj)) = (
2676 self.forwarding.client_route_concentration(),
2677 counters.as_object_mut(),
2678 ) {
2679 obj.insert(
2680 "client_connections_with_routes".into(),
2681 connections_with_routes.into(),
2682 );
2683 obj.insert("max_routes_on_one_connection".into(), max.into());
2684 }
2685 if let (Some(open_breakers), Some(obj)) = (
2691 self.route_bind_breakers.open_snapshot(),
2692 counters.as_object_mut(),
2693 ) {
2694 obj.insert("route_bind_breakers_open".into(), open_breakers);
2695 }
2696 let response = ClientControlResponse::ServerDescribe {
2697 protocol_ver: PROTOCOL_VERSION,
2698 subc_ops: subc_ops(),
2699 capabilities: self.subc_capabilities.as_ref().to_vec(),
2700 connected_clients: self.connected_clients.count(),
2701 counters: Some(counters),
2702 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
2703 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
2704 capability_requirements: self.capability_requirement_statuses(),
2705 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2706 };
2707 Ok(vec![control_response_body_frame(
2708 &frame,
2709 &response,
2710 "ClientControlResponse::ServerDescribe",
2711 )?])
2712 }
2713
2714 fn handle_catalog_list(
2715 &self,
2716 frame: Frame,
2717 module_id: Option<String>,
2718 ) -> Result<Vec<Frame>, RouterError> {
2719 let (generation, modules) = self.registry.list_modules().map_err(|err| {
2720 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
2721 })?;
2722 let entries = modules
2723 .into_iter()
2724 .filter(|registration| {
2725 module_id
2726 .as_deref()
2727 .map(|wanted| registration.manifest.module_id == wanted)
2728 .unwrap_or(true)
2729 })
2730 .map(|registration| {
2731 let not_ready = self.not_ready_reason(®istration);
2732 let roles = registration.manifest.provides;
2733 CatalogEntry {
2734 module_id: registration.manifest.module_id,
2735 ready: not_ready.is_none(),
2736 not_ready,
2737 module_version: Some(registration.manifest.module_version),
2738 roles,
2739 control_ops: registration.control_ops,
2740 capabilities: registration.manifest.capabilities,
2741 self_signals: registration.manifest.self_signals,
2742 }
2743 })
2744 .collect();
2745 let response = ClientControlResponse::CatalogList {
2746 generation,
2747 modules: entries,
2748 subc_ops: subc_ops(),
2749 };
2750 Ok(vec![control_response_body_frame(
2751 &frame,
2752 &response,
2753 "ClientControlResponse::CatalogList",
2754 )?])
2755 }
2756
2757 fn route_open_principal(
2758 &self,
2759 frame: &Frame,
2760 consumer_identity: Option<ConsumerIdentity>,
2761 ) -> Result<Result<Principal, Frame>, RouterError> {
2762 let Some(consumer_identity) = consumer_identity else {
2763 return Ok(Ok(Principal::Direct));
2764 };
2765
2766 if self.supervisor.spawned_consumer_authorized(
2767 &consumer_identity.module_id,
2768 &consumer_identity.launch_nonce,
2769 ) {
2770 return Ok(Ok(Principal::Reserved {
2771 module_id: consumer_identity.module_id,
2772 }));
2773 }
2774
2775 Ok(Err(control_error_frame(
2776 frame,
2777 "bad_consumer_identity",
2778 format!(
2779 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
2780 consumer_identity.module_id
2781 ),
2782 )?))
2783 }
2784
2785 fn route_open_refusal_frame(
2792 &self,
2793 ctx: &RouteCtx,
2794 frame: &Frame,
2795 module_id: &str,
2796 reason: &'static str,
2797 code: &'static str,
2798 message: impl Into<String>,
2799 ) -> Result<Frame, RouterError> {
2800 self.observe_route_open_refusal(ctx, module_id, reason, code);
2801 control_error_frame(frame, code, message.into())
2802 }
2803
2804 fn route_open_breaker_refusal_frame(
2822 &self,
2823 ctx: &RouteCtx,
2824 frame: &Frame,
2825 module_id: &str,
2826 consecutive_timeouts: u32,
2827 retry_in: Duration,
2828 probe_in_flight: bool,
2829 ) -> Result<Frame, RouterError> {
2830 self.counters
2831 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
2832 debug!(
2833 target: "control",
2834 code = "module_timeout",
2835 module_id = ?module_id,
2836 connection_id = ctx.connection_id.get(),
2837 consecutive_timeouts,
2838 retry_in_ms = retry_in.as_millis() as u64,
2839 probe_in_flight,
2840 "route.open refused by open bind-relay breaker"
2841 );
2842 let detail = if probe_in_flight {
2849 "one test route is already being tried; retry once it settles".to_string()
2850 } else {
2851 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
2852 };
2853 control_error_frame(
2854 frame,
2855 "module_timeout",
2856 format!(
2857 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
2858 timed out in a row); {detail}; its established routes are unaffected"
2859 ),
2860 )
2861 }
2862
2863 fn observe_route_open_refusal(
2876 &self,
2877 ctx: &RouteCtx,
2878 module_id: &str,
2879 reason: &'static str,
2880 code: &'static str,
2881 ) {
2882 self.counters.increment_route_open_refused(code);
2883 info!(
2884 target: "control",
2885 code,
2886 reason,
2887 module_id = ?module_id,
2888 connection_id = ctx.connection_id.get(),
2889 "route.open refused"
2890 );
2891 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
2892 self.route_outages.record_not_serving(module_id, reason);
2893 }
2894 }
2895
2896 fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
2941 self.route_outages.record_accepted(module_id);
2942 self.counters.increment_route_open_accepted(principal);
2943 info!(
2944 target: "control",
2945 principal,
2946 module_id,
2947 connection_id = ctx.connection_id.get(),
2948 "route.open accepted"
2949 );
2950 }
2951
2952 fn supervised_absent_route_open_refusal_frame(
2953 &self,
2954 ctx: &RouteCtx,
2955 frame: &Frame,
2956 module_id: &str,
2957 code: &'static str,
2958 status: &crate::supervise::ModuleStatus,
2959 ) -> Result<Frame, RouterError> {
2960 self.counters.increment_route_open_refused(code);
2961 info!(
2962 target: "control",
2963 code,
2964 reason = "supervised_not_registered",
2965 module_id = ?module_id,
2966 connection_id = ctx.connection_id.get(),
2967 state = %status.state,
2968 enabled = status.enabled,
2969 live = status.live,
2970 "route.open refused"
2971 );
2972 self.route_outages
2976 .record_not_serving(module_id, "supervised_not_registered");
2977 control_error_frame(
2978 frame,
2979 code,
2980 format!(
2981 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
2982 status.state, status.enabled, status.live
2983 ),
2984 )
2985 }
2986
2987 async fn handle_route_open(
2988 &self,
2989 ctx: &RouteCtx,
2990 frame: Frame,
2991 request: RouteOpenRequest,
2992 ) -> Result<Vec<Frame>, RouterError> {
2993 let RouteOpenRequest {
2994 target,
2995 mut identity,
2996 consumer_identity,
2997 consumer_capabilities,
2998 role_versions,
2999 admission_facts,
3000 scope,
3001 } = request;
3002 let target_module_id = target_module_id(&target).to_string();
3003 if self
3004 .registry
3005 .get_module_by_connection(ctx.connection_id)
3006 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3007 .is_some()
3008 {
3009 return Ok(vec![control_error_frame(
3010 &frame,
3011 "invalid_request",
3012 "module connections cannot open client routes",
3013 )?]);
3014 }
3015 debug!(
3016 connection_id = ctx.connection_id.get(),
3017 corr = frame.header.corr,
3018 module_id = %target_module_id,
3019 "handling route.open"
3020 );
3021
3022 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
3028 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
3029 self.observe_route_open_refusal(
3030 ctx,
3031 &target_module_id,
3032 "invalid_role_versions",
3033 error_codes::INVALID_REQUEST,
3034 );
3035 return Ok(vec![control_error_body_frame(
3036 &frame,
3037 ErrorBody {
3038 code: error_codes::INVALID_REQUEST.to_string(),
3039 message: error.to_string(),
3040 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
3041 },
3042 )?]);
3043 }
3044
3045 let Some(registration) = self
3064 .registry
3065 .get_module(&target_module_id)
3066 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3067 else {
3068 if let Some((status, warming)) =
3069 self.supervisor_status(&target_module_id, frame.header.corr)?
3070 {
3071 if status.protocol == ModuleProtocol::None {
3078 return Ok(vec![self.route_open_refusal_frame(
3079 ctx,
3080 &frame,
3081 &target_module_id,
3082 "protocol_none",
3083 error_codes::MODULE_NO_PROTOCOL,
3084 format!(
3085 "module_id '{target_module_id}' is declared protocol: none; \
3086 it speaks no subc wire and serves no routes"
3087 ),
3088 )?]);
3089 }
3090 let code = if warming {
3091 "module_warming"
3092 } else {
3093 "target_unavailable"
3094 };
3095 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
3096 ctx,
3097 &frame,
3098 &target_module_id,
3099 code,
3100 &status,
3101 )?]);
3102 }
3103 if let Some(removed_ago_ms) =
3104 self.supervisor.removal_tombstone_age_ms(&target_module_id)
3105 {
3106 return Ok(vec![self.route_open_refusal_frame(
3107 ctx,
3108 &frame,
3109 &target_module_id,
3110 "removed",
3111 error_codes::MODULE_REMOVED,
3112 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
3113 )?]);
3114 }
3115 return Ok(vec![self.route_open_refusal_frame(
3116 ctx,
3117 &frame,
3118 &target_module_id,
3119 "not_registered",
3120 error_codes::UNKNOWN_MODULE,
3121 format!("module_id '{target_module_id}' is not registered"),
3122 )?]);
3123 };
3124
3125 if !registration.ready {
3129 self.counters
3130 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
3131 info!(
3132 target: "control",
3133 code = error_codes::MODULE_WARMING,
3134 module_id = ?target_module_id,
3135 connection_id = ctx.connection_id.get(),
3136 reason = "declared_not_ready",
3137 "route.open refused"
3138 );
3139 self.route_outages
3142 .record_not_serving(&target_module_id, "declared_not_ready");
3143 return Ok(vec![control_error_body_frame(
3144 &frame,
3145 ErrorBody {
3146 code: error_codes::MODULE_WARMING.to_string(),
3147 message: format!(
3148 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
3149 ),
3150 detail: Some(serde_json::json!({
3151 "reason": "declared_not_ready"
3152 })),
3153 },
3154 )?]);
3155 }
3156
3157 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
3176 self.counters
3177 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
3178 info!(
3179 target: "control",
3180 code = error_codes::MODULE_WARMING,
3181 module_id = ?target_module_id,
3182 connection_id = ctx.connection_id.get(),
3183 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3184 capability = %capability,
3185 "route.open refused"
3186 );
3187 return Ok(vec![control_error_body_frame(
3188 &frame,
3189 ErrorBody {
3190 code: error_codes::MODULE_WARMING.to_string(),
3191 message: format!(
3192 "module_id '{target_module_id}' requires capability '{capability}', \
3193 which no registered module provides; retry"
3194 ),
3195 detail: Some(serde_json::json!({
3196 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3197 "capability": capability,
3198 })),
3199 },
3200 )?]);
3201 }
3202
3203 if !target_has_required_role(&target, ®istration.manifest.provides) {
3204 return Ok(vec![self.route_open_refusal_frame(
3205 ctx,
3206 &frame,
3207 &target_module_id,
3208 "role_not_provided",
3209 "target_unavailable",
3210 format!("module_id '{target_module_id}' does not provide the requested target"),
3211 )?]);
3212 }
3213
3214 if registration.state != ChannelState::Active {
3215 return Ok(vec![self.route_open_refusal_frame(
3216 ctx,
3217 &frame,
3218 &target_module_id,
3219 "registration_not_active",
3220 "target_unavailable",
3221 format!("module_id '{target_module_id}' is not active"),
3222 )?]);
3223 }
3224
3225 if self
3226 .forwarding
3227 .module_is_draining(&target_module_id)
3228 .map_err(RouterError::Forwarding)?
3229 {
3230 return Ok(vec![self.route_open_refusal_frame(
3231 ctx,
3232 &frame,
3233 &target_module_id,
3234 "reloading",
3235 "module_reloading",
3236 format!("module_id '{target_module_id}' is reloading"),
3237 )?]);
3238 }
3239
3240 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
3241 process_liveness.process_live(&target_module_id) == Some(false)
3242 }) {
3243 if process_liveness.process_replacing(&target_module_id) {
3250 return Ok(vec![self.route_open_refusal_frame(
3251 ctx,
3252 &frame,
3253 &target_module_id,
3254 "reloading",
3255 "module_reloading",
3256 format!("module_id '{target_module_id}' is reloading"),
3257 )?]);
3258 }
3259 return Ok(vec![self.route_open_refusal_frame(
3260 ctx,
3261 &frame,
3262 &target_module_id,
3263 "supervisor_not_live",
3264 "target_unavailable",
3265 format!("module_id '{target_module_id}' is not live"),
3266 )?]);
3267 }
3268
3269 if !self
3270 .forwarding
3271 .has_live_module_connection(&target_module_id)
3272 .map_err(RouterError::Forwarding)?
3273 {
3274 return Ok(vec![self.route_open_refusal_frame(
3275 ctx,
3276 &frame,
3277 &target_module_id,
3278 "no_forwarding_connection",
3279 "target_unavailable",
3280 format!("module_id '{target_module_id}' has no live forwarding connection"),
3281 )?]);
3282 }
3283
3284 if let Some(error) =
3285 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
3286 {
3287 self.observe_route_open_refusal(
3288 ctx,
3289 &target_module_id,
3290 "op_not_allowed",
3291 "op_not_allowed",
3292 );
3293 return Ok(vec![error]);
3294 }
3295
3296 let principal = match self.route_open_principal(&frame, consumer_identity)? {
3297 Ok(principal) => principal,
3298 Err(error) => {
3299 self.observe_route_open_refusal(
3300 ctx,
3301 &target_module_id,
3302 "bad_consumer_identity",
3303 "bad_consumer_identity",
3304 );
3305 return Ok(vec![error]);
3306 }
3307 };
3308
3309 if let Principal::Reserved {
3313 module_id: opening_module_id,
3314 } = &principal
3315 {
3316 if let Some(opening_registration) = self
3317 .registry
3318 .get_module(opening_module_id)
3319 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3320 {
3321 if let Some(capability) =
3322 denied_capability(&opening_registration.manifest, ®istration.manifest)
3323 {
3324 warn!(
3325 opening_module_id,
3326 target_module_id,
3327 capability,
3328 "refusing route.open because an attested capability deny edge matches"
3329 );
3330 return Ok(vec![self.route_open_refusal_frame(
3331 ctx,
3332 &frame,
3333 &target_module_id,
3334 "capability_deny_edge",
3335 "capability_forbidden",
3336 format!(
3337 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
3338 ),
3339 )?]);
3340 }
3341 }
3342 }
3343
3344 if admission_facts.is_some() {
3345 let carrier_matches = matches!(
3346 &principal,
3347 Principal::Reserved { module_id }
3348 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
3349 );
3350 if !carrier_matches {
3351 return Ok(vec![self.route_open_refusal_frame(
3352 ctx,
3353 &frame,
3354 &target_module_id,
3355 "admission_facts_carrier_not_permitted",
3356 "admission_facts_not_permitted",
3357 "admission facts may only be carried by the configured reserved module",
3358 )?]);
3359 }
3360
3361 let target_allowed = self
3362 .admission_facts_targets
3363 .as_ref()
3364 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
3365 if !target_allowed {
3366 return Ok(vec![self.route_open_refusal_frame(
3367 ctx,
3368 &frame,
3369 &target_module_id,
3370 "admission_facts_target_not_listed",
3371 "admission_facts_target_not_allowed",
3372 format!(
3373 "admission facts are not permitted for target module_id '{target_module_id}'"
3374 ),
3375 )?]);
3376 }
3377
3378 }
3382
3383 let (bound_scope, scope_stamp) = match scope {
3389 None => (None, None),
3390 Some(selector) => {
3391 let owner_configured = match &selector.owner {
3392 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3398 _ => false,
3399 };
3400 let admitted = self
3401 .scopes
3402 .read()
3403 .unwrap_or_else(|poisoned| poisoned.into_inner())
3404 .admit(&principal, &target_module_id, &selector, owner_configured);
3405 match admitted {
3406 Ok(admission) => (
3407 Some(BoundScope {
3408 owner: admission.owner,
3409 scope_ref: admission.stamp.scope_ref.clone(),
3410 tag: admission.tag,
3411 }),
3412 Some(admission.stamp),
3413 ),
3414 Err(refusal) => {
3415 return Ok(vec![self.route_open_refusal_frame(
3416 ctx,
3417 &frame,
3418 &target_module_id,
3419 refusal.code,
3420 refusal.code,
3421 refusal.message,
3422 )?]);
3423 }
3424 }
3425 }
3426 };
3427
3428 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
3450 Ok(project_root) => project_root,
3451 Err(err) => {
3452 return Ok(vec![control_error_frame(
3453 &frame,
3454 "invalid_project_root",
3455 err.to_string(),
3456 )?])
3457 }
3458 };
3459 identity.project_root = project_root.as_path().to_path_buf();
3460
3461 let _concurrency_guard = match self
3472 .route_bind_concurrency
3473 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
3474 {
3475 Ok(guard) => guard,
3476 Err(in_flight) => {
3477 return Ok(vec![self.route_open_target_capacity_refusal(
3478 ctx,
3479 &frame,
3480 &target_module_id,
3481 in_flight,
3482 )?]);
3483 }
3484 };
3485
3486 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
3489 RouteBindAdmission::Admitted { guard, probe } => {
3490 if probe {
3491 info!(
3492 module_id = %target_module_id,
3493 connection_id = ctx.connection_id.get(),
3494 "route.bind breaker half-open: admitting one probe"
3495 );
3496 }
3497 guard
3498 }
3499 RouteBindAdmission::Refused {
3500 consecutive_timeouts,
3501 retry_in,
3502 probe_in_flight,
3503 } => {
3504 return Ok(vec![self.route_open_breaker_refusal_frame(
3505 ctx,
3506 &frame,
3507 &target_module_id,
3508 consecutive_timeouts,
3509 retry_in,
3510 probe_in_flight,
3511 )?]);
3512 }
3513 };
3514
3515 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
3520 let relay_deadline = Instant::now() + route_bind_relay_timeout;
3521 let pending = match self
3522 .forwarding
3523 .begin_route_bind_relay_for(
3524 ctx.connection_id,
3525 ctx.egress.clone(),
3526 response_version(&frame),
3527 frame.header.corr,
3528 &target_module_id,
3529 principal.clone(),
3530 bound_scope,
3531 Some(project_root),
3532 relay_deadline,
3533 )
3534 .await
3535 {
3536 Ok(pending) => pending,
3537 Err(err) => {
3538 return Ok(vec![self.route_open_refusal_frame(
3539 ctx,
3540 &frame,
3541 &target_module_id,
3542 "relay_reservation_failed",
3543 forwarding_error_code(&err),
3544 err.to_string(),
3545 )?])
3546 }
3547 };
3548 let crate::forwarding::PendingRouteBindRelay {
3549 endpoint,
3550 module_sink,
3551 negotiated_ver,
3552 client_channel,
3553 client_epoch,
3554 module_channel,
3555 module_epoch,
3556 corr: relay_corr,
3557 receiver,
3558 } = pending;
3559 let mut reservation =
3560 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
3561
3562 debug!(
3563 connection_id = ctx.connection_id.get(),
3564 client_channel,
3565 client_epoch,
3566 module_channel,
3567 module_epoch,
3568 "reserved route handle pair"
3569 );
3570 let principal_label = match &principal {
3573 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
3574 Principal::Direct => "direct".to_string(),
3575 other => format!("{other:?}"),
3576 };
3577 let relay = ModuleControlRequest::RouteBind {
3578 route_channel: module_channel,
3579 epoch: module_epoch,
3580 target,
3581 identity,
3582 principal: Some(principal),
3583 consumer_capabilities,
3584 role_versions,
3585 admission_facts,
3586 scope: scope_stamp,
3587 };
3588 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
3589 RouterError::backend(
3590 0,
3591 frame.header.corr,
3592 format!("failed to encode route.bind request: {err}"),
3593 )
3594 })?;
3595 let relay_frame = Frame::build_with_version(
3596 negotiated_ver,
3597 FrameType::Request,
3598 control_flags(),
3599 0,
3600 0,
3601 relay_corr,
3602 relay_body,
3603 )
3604 .map_err(RouterError::FrameBuild)?;
3605
3606 if let Err(err) = module_sink.send(relay_frame).await {
3607 reservation.release_and_disarm();
3608 return Ok(vec![self.route_open_refusal_frame(
3609 ctx,
3610 &frame,
3611 &target_module_id,
3612 "relay_send_failed",
3613 "target_unavailable",
3614 err.to_string(),
3615 )?]);
3616 }
3617
3618 if !self
3619 .forwarding
3620 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
3621 .map_err(RouterError::Forwarding)?
3622 {
3623 self.send_abandoned_route_bind_goodbye(
3624 &module_sink,
3625 negotiated_ver,
3626 module_channel,
3627 module_epoch,
3628 );
3629 }
3630
3631 match timeout_at(relay_deadline, receiver).await {
3632 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
3633 reservation.disarm();
3634 if breaker.record_accepted() {
3635 info!(
3636 module_id = %target_module_id,
3637 "route.bind breaker closed: the probe was accepted"
3638 );
3639 }
3640 self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
3641 Ok(Vec::new())
3642 }
3643 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
3644 reservation.release_and_disarm();
3645 breaker.record_inconclusive();
3649 let scope_code = match body.code.as_str() {
3653 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
3654 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
3655 _ => None,
3656 };
3657 if let Some(code) = scope_code {
3658 self.observe_route_open_refusal(
3659 ctx,
3660 &target_module_id,
3661 "scope_changed_before_commit",
3662 code,
3663 );
3664 return Ok(vec![control_error_body_frame(&frame, body)?]);
3665 }
3666 self.counters
3667 .increment_route_open_refused("module_rejected");
3668 info!(
3669 target: "control",
3670 code = "module_rejected",
3671 module_code = ?body.code,
3672 module_id = ?target_module_id,
3673 connection_id = ctx.connection_id.get(),
3674 "route.open refused"
3675 );
3676 Ok(vec![control_error_body_frame(&frame, body)?])
3677 }
3678 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
3679 reservation.release_and_disarm();
3680 breaker.record_inconclusive();
3681 tracing::warn!(
3688 module_id = %target_module_id,
3689 "route.bind relay abandoned: {message}"
3690 );
3691 Ok(vec![self.route_open_refusal_frame(
3692 ctx,
3693 &frame,
3694 &target_module_id,
3695 "relay_abandoned",
3696 "target_unavailable",
3697 message,
3698 )?])
3699 }
3700 Ok(Err(_)) => {
3701 reservation.release_and_disarm();
3702 breaker.record_inconclusive();
3703 Ok(vec![self.route_open_refusal_frame(
3704 ctx,
3705 &frame,
3706 &target_module_id,
3707 "relay_waiter_canceled",
3708 "target_unavailable",
3709 "route.bind relay waiter was canceled before the module responded",
3710 )?])
3711 }
3712 Err(_) => {
3713 reservation.release_and_disarm();
3714 if let Some(opened) = breaker.record_timeout(
3718 self.route_bind_breaker_threshold,
3719 self.route_bind_breaker_cooldown,
3720 ) {
3721 warn!(
3722 module_id = %target_module_id,
3723 consecutive_timeouts = opened.consecutive_timeouts,
3724 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
3725 reopened_after_probe = opened.reopened_after_probe,
3726 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
3727 );
3728 }
3729 tracing::warn!(
3737 module_id = %target_module_id,
3738 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
3739 "route.bind relay timed out: module did not ack within budget"
3740 );
3741 Ok(vec![self.route_open_refusal_frame(
3742 ctx,
3743 &frame,
3744 &target_module_id,
3745 "relay_timed_out",
3746 "module_timeout",
3747 format!(
3748 "module_id '{target_module_id}' did not answer route.bind within {:?}",
3749 route_bind_relay_timeout
3750 ),
3751 )?])
3752 }
3753 }
3754 }
3755
3756 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3757 let response = ClientControlResponse::SupervisorSpawnSnapshot {
3758 snapshot: self.supervisor.spawn_snapshot(),
3759 };
3760 Ok(vec![control_response_body_frame(
3761 &frame,
3762 &response,
3763 "ClientControlResponse::SupervisorSpawnSnapshot",
3764 )?])
3765 }
3766
3767 fn handle_supervisor_spawn_subscribe(
3768 &self,
3769 ctx: &RouteCtx,
3770 frame: Frame,
3771 since: Option<SpawnCursor>,
3772 ) -> Result<Vec<Frame>, RouterError> {
3773 match self.supervisor.subscribe_spawns(
3774 ctx.connection_id,
3775 frame.header.corr,
3776 response_version(&frame),
3777 since,
3778 ctx.egress.clone(),
3779 ) {
3780 Ok(()) => Ok(Vec::new()),
3781 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
3782 Ok(vec![control_error_body_frame(
3783 &frame,
3784 ErrorBody {
3785 code: "spawn_cursor_incarnation_mismatch".to_string(),
3786 message: "spawn cursor belongs to a different daemon incarnation"
3787 .to_string(),
3788 detail: Some(serde_json::json!({
3789 "current_daemon_incarnation": current
3790 })),
3791 },
3792 )?])
3793 }
3794 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
3795 &frame,
3796 ErrorBody {
3797 code: "spawn_cursor_too_old".to_string(),
3798 message: "spawn cursor predates the retained event ring".to_string(),
3799 detail: Some(serde_json::json!({
3800 "oldest_retained_cursor": oldest
3801 })),
3802 },
3803 )?]),
3804 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
3805 0,
3806 frame.header.corr,
3807 format!("failed to open supervisor spawn subscription: {error}"),
3808 )),
3809 }
3810 }
3811
3812 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3813 let generation = self
3814 .registry
3815 .generation()
3816 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3817 let mut modules = Vec::new();
3818 for module in self.supervisor.list() {
3819 let status = module.status_for_control("list").map_err(|err| {
3820 RouterError::backend(
3821 0,
3822 frame.header.corr,
3823 format!("failed to read supervisor status: {err}"),
3824 )
3825 })?;
3826 let (configured, _) = module.configuration().map_err(|err| {
3827 RouterError::backend(
3828 0,
3829 frame.header.corr,
3830 format!("failed to read module configuration: {err}"),
3831 )
3832 })?;
3833 let image = module.running_image_agreement().await;
3835 let resources = Some(module.child_resource_usage());
3838 let pending_reload = Some(reload_verdict(
3839 &configured.program,
3840 status.spawned_from.as_deref(),
3841 image,
3842 ));
3843 modules.push(SupervisorEntry {
3844 launch_nonce_env: Some(!cfg!(unix)),
3847 module_id: status.module_id,
3848 state: status.state.to_string(),
3849 enabled: status.enabled,
3850 live: status.live,
3851 protocol: status.protocol,
3852 health: status.health.status,
3853 pending_reload,
3854 last_probe_ms: status.health.last_probe_ms,
3855 last_exit_code: status.last_exit.as_ref().and_then(|e| e.code),
3856 last_exit_signal: status.last_exit.as_ref().and_then(|e| e.signal),
3857 last_exit_ms: status.last_exit.as_ref().map(|e| e.at_ms),
3858 last_exit_kind: status.last_exit.as_ref().map(|e| e.kind.into()),
3859 restart_count: Some(status.restart_count),
3860 max_restarts: Some(status.max_restarts),
3861 lifetime_restarts: Some(status.lifetime_restarts),
3862 spawn_generation: Some(status.spawn_generation),
3863 restart_window_secs: Some(status.restart_window.as_secs()),
3864 drain_timeout_ms: Some(status.drain_timeout.as_millis() as u64),
3865 restart_backoff_ms: Some(status.restart_backoff.as_millis() as u64),
3866 restart_max_backoff_ms: Some(status.restart_max_backoff.as_millis() as u64),
3867 resources,
3868 });
3869 }
3870 let response = ClientControlResponse::SupervisorList {
3871 generation,
3872 modules,
3873 };
3874 Ok(vec![control_response_body_frame(
3875 &frame,
3876 &response,
3877 "ClientControlResponse::SupervisorList",
3878 )?])
3879 }
3880
3881 fn handle_supervisor_stderr_tail(
3882 &self,
3883 frame: Frame,
3884 module_id: String,
3885 max_lines: Option<u32>,
3886 max_bytes: Option<u32>,
3887 ) -> Result<Vec<Frame>, RouterError> {
3888 let Some(module) = self.supervisor.get(&module_id) else {
3889 return Ok(vec![control_error_frame(
3890 &frame,
3891 "unknown_module",
3892 format!("module_id '{module_id}' is not supervised"),
3893 )?]);
3894 };
3895
3896 let snapshot = module.stderr_tail(
3897 max_lines.map(|value| value as usize),
3898 max_bytes.map(|value| value as usize),
3899 );
3900
3901 let response = ClientControlResponse::SupervisorStderrTail {
3902 module_id,
3903 tail: StderrTail {
3904 capture: match snapshot.capture {
3905 CaptureState::Captured => StderrCaptureState::Captured,
3906 CaptureState::Incomplete { reason } => {
3907 StderrCaptureState::Incomplete { reason }
3908 }
3909 CaptureState::NotCaptured { reason } => {
3910 StderrCaptureState::NotCaptured { reason }
3911 }
3912 },
3913 entries: snapshot
3914 .entries
3915 .into_iter()
3916 .map(|entry| match entry {
3917 TailEntry::Line {
3918 text,
3919 truncated,
3920 at_ms,
3921 } => StderrTailEntry::Line {
3922 text,
3923 truncated,
3924 at_ms,
3925 },
3926 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
3927 })
3928 .collect(),
3929 dropped_lines: snapshot.dropped_lines,
3930 },
3931 };
3932 Ok(vec![control_response_body_frame(
3933 &frame,
3934 &response,
3935 "ClientControlResponse::SupervisorStderrTail",
3936 )?])
3937 }
3938
3939 async fn handle_supervisor_terminals(
3940 &self,
3941 frame: Frame,
3942 module_id: String,
3943 ) -> Result<Vec<Frame>, RouterError> {
3944 let Some(module) = self.supervisor.get(&module_id) else {
3945 return Ok(vec![control_error_frame(
3946 &frame,
3947 "unknown_module",
3948 format!("module_id '{module_id}' is not supervised"),
3949 )?]);
3950 };
3951
3952 let terminals = module
3955 .read_durable_terminal_history()
3956 .await
3957 .map_err(|error| {
3958 RouterError::backend(
3959 0,
3960 frame.header.corr,
3961 format!("failed to read terminal history: {error}"),
3962 )
3963 })?;
3964 let response = ClientControlResponse::SupervisorTerminals {
3965 module_id,
3966 terminals,
3967 };
3968 Ok(vec![control_response_body_frame(
3969 &frame,
3970 &response,
3971 "ClientControlResponse::SupervisorTerminals",
3972 )?])
3973 }
3974
3975 fn handle_supervisor_routes(
3976 &self,
3977 frame: Frame,
3978 module_id: Option<String>,
3979 ) -> Result<Vec<Frame>, RouterError> {
3980 let modules = self
3981 .forwarding
3982 .route_census(module_id.as_deref())
3983 .map_err(RouterError::Forwarding)?
3984 .into_iter()
3985 .map(|(module_id, routes)| SupervisorRouteModule {
3986 module_id,
3987 routes: routes
3988 .into_iter()
3989 .map(|route| SupervisorRoute {
3990 consumer: match route.principal {
3991 Principal::Reserved { module_id } => {
3992 SupervisorRouteConsumer::Reserved { module_id }
3993 }
3994 Principal::Direct | Principal::Unverified => {
3995 SupervisorRouteConsumer::Direct {
3996 connection_id: route.goodbye_target.connection_id.get(),
3997 }
3998 }
3999 },
4000 age_ms: Instant::now()
4001 .saturating_duration_since(route.bound_at)
4002 .as_millis()
4003 .try_into()
4004 .unwrap_or(u64::MAX),
4005 draining: route.draining,
4006 drain_reason: route.drain_reason,
4007 })
4008 .collect(),
4009 })
4010 .collect();
4011 let response = ClientControlResponse::SupervisorRoutes { modules };
4012 Ok(vec![control_response_body_frame(
4013 &frame,
4014 &response,
4015 "ClientControlResponse::SupervisorRoutes",
4016 )?])
4017 }
4018
4019 async fn handle_supervisor_provenance(
4020 &self,
4021 frame: Frame,
4022 module_id: Option<String>,
4023 ) -> Result<Vec<Frame>, RouterError> {
4024 let mut selected = if let Some(module_id) = module_id {
4025 let Some(module) = self.supervisor.get(&module_id) else {
4026 return Ok(vec![control_error_frame(
4027 &frame,
4028 "unknown_module",
4029 format!("module_id '{module_id}' is not supervised"),
4030 )?]);
4031 };
4032 vec![module]
4033 } else {
4034 self.supervisor.list()
4035 };
4036
4037 let mut modules = Vec::with_capacity(selected.len());
4038 for module in selected.drain(..) {
4039 let status = module.status().map_err(|err| {
4040 RouterError::backend(
4041 0,
4042 frame.header.corr,
4043 format!("failed to read supervisor status: {err}"),
4044 )
4045 })?;
4046 let module_declared = self
4047 .registry
4048 .get_module(&status.module_id)
4049 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4050 .and_then(|registration| registration.manifest.provenance)
4051 .map(|build| ModuleDeclaredProvenance::Reported { build })
4052 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
4053 #[cfg(test)]
4054 let running_image = match &self.provenance_probe_override {
4055 Some(result) => result.clone(),
4056 None => module.running_image_agreement().await,
4057 };
4058 #[cfg(not(test))]
4059 let running_image = module.running_image_agreement().await;
4060 modules.push(SupervisorModuleProvenance {
4061 module_id: status.module_id,
4062 module_declared,
4063 daemon_observed: SupervisorObservedProcess {
4064 pid: status.pid,
4065 spawned_at_ms: status.spawned_at_ms,
4066 spawned_from: status.spawned_from,
4067 running_image,
4068 },
4069 });
4070 }
4071 let daemon = SupervisorDaemonProvenance {
4072 daemon_build: self.daemon_provenance.build.clone(),
4073 daemon_observed: DaemonObservedProcess {
4074 pid: self.daemon_provenance.pid,
4075 started_at_ms: self
4076 .daemon_provenance
4077 .start_clock
4078 .map(|clock| clock.started_at_ms())
4079 .or(self.daemon_provenance.started_at_ms),
4080 running_image: self
4081 .daemon_provenance
4082 .probe
4083 .observe(
4084 self.daemon_provenance.pid,
4085 self.daemon_provenance.executable_path.as_deref(),
4086 self.daemon_provenance.executable_identity,
4087 self.daemon_provenance.process_start_time,
4088 )
4089 .await,
4090 },
4091 };
4092 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
4093 Ok(vec![control_response_body_frame(
4094 &frame,
4095 &response,
4096 "ClientControlResponse::SupervisorProvenance",
4097 )?])
4098 }
4099
4100 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4101 self.refresh_capability_requirements();
4102 let generation = self
4103 .registry
4104 .generation()
4105 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4106 let modules = self
4107 .supervisor
4108 .list()
4109 .into_iter()
4110 .map(|module| {
4111 let status = module.status_for_control("health").map_err(|err| {
4112 RouterError::backend(
4113 0,
4114 frame.header.corr,
4115 format!("failed to read supervisor health: {err}"),
4116 )
4117 })?;
4118 let module_id = status.module_id;
4119 let capability_detail = self
4120 .capability_evaluator
4121 .required_problem_detail(&module_id);
4122 Ok(SupervisorHealthEntry {
4123 module_id,
4124 status: status.health.status,
4125 detail: append_capability_problem_detail(
4126 status.health.detail,
4127 capability_detail,
4128 ),
4129 metrics: status.health.metrics,
4130 consecutive_failures: status.health.consecutive_failures,
4131 late_answer_count: status.health.late_answer_count,
4132 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
4133 last_action: status.health.last_action,
4134 last_action_ms: status.health.last_action_ms,
4135 last_probe_ms: status.health.last_probe_ms,
4136 })
4137 })
4138 .collect::<Result<Vec<_>, RouterError>>()?;
4139 let response = ClientControlResponse::SupervisorHealth {
4140 generation,
4141 modules,
4142 };
4143 Ok(vec![control_response_body_frame(
4144 &frame,
4145 &response,
4146 "ClientControlResponse::SupervisorHealth",
4147 )?])
4148 }
4149
4150 async fn handle_supervisor_restart(
4151 &self,
4152 frame: Frame,
4153 module_id: String,
4154 drain_timeout_ms: Option<u64>,
4155 ) -> Result<Vec<Frame>, RouterError> {
4156 let operation_lock = self.supervisor.operation_lock();
4157 let _operation_guard = operation_lock.lock().await;
4158 let Some(module) = self.supervisor.get(&module_id) else {
4159 return Ok(vec![control_error_frame(
4160 &frame,
4161 "unknown_module",
4162 format!("module_id '{module_id}' is not supervised"),
4163 )?]);
4164 };
4165
4166 self.route_outages.mark_operator_action(&module_id);
4167 if let Err(err) = module.restart(drain_timeout_ms).await {
4168 self.route_outages
4169 .operator_action_ended_unrefused(&module_id);
4170 let (code, message) = match err {
4171 crate::supervise::SuperviseError::Disabled { .. } => {
4172 ("module_disabled", err.to_string())
4173 }
4174 crate::supervise::SuperviseError::SwapInProgress { .. } => {
4175 ("swap_in_progress", err.to_string())
4176 }
4177 _ => (
4178 "target_unavailable",
4179 format!("failed to restart module_id '{module_id}': {err}"),
4180 ),
4181 };
4182 return Ok(vec![control_error_frame(&frame, code, message)?]);
4183 }
4184
4185 let response = ClientControlResponse::SupervisorAck {
4186 module_id,
4187 applied: true,
4188 };
4189 Ok(vec![control_response_body_frame(
4190 &frame,
4191 &response,
4192 "ClientControlResponse::SupervisorAck",
4193 )?])
4194 }
4195
4196 async fn handle_supervisor_swap(
4200 &self,
4201 frame: Frame,
4202 module_id: String,
4203 ready_timeout_ms: Option<u64>,
4204 ) -> Result<Vec<Frame>, RouterError> {
4205 let module = {
4212 let operation_lock = self.supervisor.operation_lock();
4213 let _operation_guard = operation_lock.lock().await;
4214 self.supervisor.get(&module_id)
4215 };
4216 let Some(module) = module else {
4217 return Ok(vec![control_error_frame(
4218 &frame,
4219 "unknown_module",
4220 format!("module_id '{module_id}' is not supervised"),
4221 )?]);
4222 };
4223
4224 self.route_outages.mark_operator_action(&module_id);
4225 if let Err(err) = module
4226 .swap(ready_timeout_ms.map(Duration::from_millis))
4227 .await
4228 {
4229 self.route_outages
4230 .operator_action_ended_unrefused(&module_id);
4231 use crate::supervise::SuperviseError;
4232 let message = err.to_string();
4233 let error = match err {
4234 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
4235 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
4236 code: "swap_refused".to_string(),
4237 message,
4238 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
4239 },
4240 SuperviseError::SwapFailed {
4241 arm,
4242 candidate_exit,
4243 ..
4244 } => ErrorBody {
4245 code: "swap_failed".to_string(),
4246 message,
4247 detail: Some(serde_json::json!({
4248 "arm": arm.as_str(),
4249 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
4250 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
4251 })),
4252 },
4253 _ => ErrorBody::new(
4254 "target_unavailable",
4255 format!("failed to swap module_id '{module_id}': {message}"),
4256 ),
4257 };
4258 return Ok(vec![control_error_body_frame(&frame, error)?]);
4259 }
4260 self.route_outages
4264 .operator_action_ended_unrefused(&module_id);
4265
4266 let response = ClientControlResponse::SupervisorAck {
4267 module_id,
4268 applied: true,
4269 };
4270 Ok(vec![control_response_body_frame(
4271 &frame,
4272 &response,
4273 "ClientControlResponse::SupervisorAck",
4274 )?])
4275 }
4276
4277 async fn handle_supervisor_reload(
4278 &self,
4279 frame: Frame,
4280 module_id: String,
4281 ) -> Result<Vec<Frame>, RouterError> {
4282 let operation_lock = self.supervisor.operation_lock();
4283 let _operation_guard = operation_lock.lock().await;
4284 let Some(module) = self.supervisor.get(&module_id) else {
4285 return Ok(vec![control_error_frame(
4286 &frame,
4287 "unknown_module",
4288 format!("module_id '{module_id}' is not supervised"),
4289 )?]);
4290 };
4291
4292 self.route_outages.mark_operator_action(&module_id);
4293 if let Err(err) = module.reload().await {
4294 self.route_outages
4295 .operator_action_ended_unrefused(&module_id);
4296 let (code, message) = match err {
4297 crate::supervise::SuperviseError::Disabled { .. } => {
4298 ("module_disabled", err.to_string())
4299 }
4300 crate::supervise::SuperviseError::SwapInProgress { .. } => {
4301 ("swap_in_progress", err.to_string())
4302 }
4303 _ => (
4304 "reload_failed",
4305 format!("failed to reload module_id '{module_id}': {err}"),
4306 ),
4307 };
4308 return Ok(vec![control_error_frame(&frame, code, message)?]);
4309 }
4310
4311 let response = ClientControlResponse::SupervisorAck {
4312 module_id,
4313 applied: true,
4314 };
4315 Ok(vec![control_response_body_frame(
4316 &frame,
4317 &response,
4318 "ClientControlResponse::SupervisorAck",
4319 )?])
4320 }
4321
4322 async fn handle_supervisor_rescan(
4323 &self,
4324 frame: Frame,
4325 preview: bool,
4326 ) -> Result<Vec<Frame>, RouterError> {
4327 let Some(context) = self.rescan.clone() else {
4328 return Ok(vec![control_error_frame(
4329 &frame,
4330 "rescan_unavailable",
4331 "the daemon was not started with a reloadable config path".to_string(),
4332 )?]);
4333 };
4334
4335 let operation_lock = self.supervisor.operation_lock();
4336 let _operation_guard = operation_lock.lock().await;
4337 let loaded = match crate::daemon_config::load(&context.config_path) {
4338 Ok(config) => config,
4339 Err(err) => {
4340 return Ok(vec![control_error_frame(
4341 &frame,
4342 "invalid_daemon_config",
4343 format!("supervisor rescan rejected daemon config: {err}"),
4344 )?])
4345 }
4346 };
4347 let Some(config) = loaded else {
4355 return Ok(vec![control_error_frame(
4356 &frame,
4357 "invalid_daemon_config",
4358 format!(
4359 "daemon config not found at {}; refusing to rescan (an absent config would \
4360 retire every supervised module)",
4361 context.config_path.display()
4362 ),
4363 )?]);
4364 };
4365 let (
4366 configured_port,
4367 storage_config,
4368 admission_facts_carrier_module_id,
4369 admission_facts_targets,
4370 scope_authority_owners,
4371 modules,
4372 reserved_capabilities,
4373 ) = (
4374 config.port,
4375 config.storage,
4376 config.admission_facts_carrier_module_id,
4377 config.admission_facts_targets,
4378 config.scope_authority_owners,
4379 config.modules,
4380 config.reserved_capabilities,
4381 );
4382
4383 let mut restart_required = Vec::new();
4392 for section in RestartRequiredSection::ALL {
4393 let changed = match section {
4394 RestartRequiredSection::Port => configured_port != context.configured_port,
4395 RestartRequiredSection::Storage => storage_config != context.storage_config,
4396 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
4397 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
4398 }
4399 RestartRequiredSection::AdmissionFactsTargets => {
4400 admission_facts_targets != context.admission_facts_targets
4401 }
4402 RestartRequiredSection::ScopeAuthorityOwners => {
4403 scope_authority_owners != context.scope_authority_owners
4404 }
4405 };
4406 if changed {
4407 restart_required.push(section.label().to_string());
4408 }
4409 }
4410 if !restart_required.is_empty() {
4411 warn!(
4412 config_path = %context.config_path.display(),
4413 sections = %restart_required.join(", "),
4414 "daemon config changed outside the modules section; restart the daemon to apply those changes"
4415 );
4416 }
4417
4418 for configured in &modules {
4419 if let Err(err) = validate_spec(&configured.module_spec()) {
4420 return Ok(vec![control_error_frame(
4421 &frame,
4422 "invalid_daemon_config",
4423 format!("supervisor rescan rejected daemon config: {err}"),
4424 )?]);
4425 }
4426 }
4427
4428 let configured_capabilities = modules
4429 .iter()
4430 .map(|module| (module.module_id.clone(), module.enabled))
4431 .collect::<Vec<_>>();
4432 let preview_capability_warnings = if preview {
4433 let (_, registrations) = self.runtime_capability_snapshot()?;
4434 let current_modules = self
4435 .supervisor
4436 .list()
4437 .into_iter()
4438 .map(|module| module.module_id().to_string())
4439 .collect::<BTreeSet<_>>();
4440 let resulting_modules = configured_capabilities.clone();
4441 let removed = current_modules
4442 .into_iter()
4443 .filter(|module_id| {
4444 !resulting_modules
4445 .iter()
4446 .any(|(configured_id, _)| configured_id == module_id)
4447 })
4448 .collect::<Vec<_>>();
4449 self.capability_evaluator.preview_removal_warnings(
4450 resulting_modules,
4451 &removed,
4452 ®istrations,
4453 )
4454 } else {
4455 Vec::new()
4456 };
4457 let result = match self
4458 .reconcile_supervised_modules(&context.supervisor, modules, preview)
4459 .await
4460 {
4461 Ok(result) => result,
4462 Err(message) => {
4463 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
4464 }
4465 };
4466 if !preview {
4467 self.capability_evaluator
4468 .configure(configured_capabilities, reserved_capabilities);
4469 self.capability_evaluator.wake_deadline_loop();
4470 self.refresh_capability_requirements();
4471 }
4472 let mut result = result;
4473 result.restart_required = restart_required;
4474 result.capability_warnings = preview_capability_warnings;
4475 let response = ClientControlResponse::SupervisorRescan { result };
4476 Ok(vec![control_response_body_frame(
4477 &frame,
4478 &response,
4479 "ClientControlResponse::SupervisorRescan",
4480 )?])
4481 }
4482
4483 async fn handle_supervisor_release_reserved(
4484 &self,
4485 frame: Frame,
4486 module_id: String,
4487 ) -> Result<Vec<Frame>, RouterError> {
4488 let Some(context) = self.rescan.clone() else {
4489 return Ok(vec![control_error_frame(
4490 &frame,
4491 "release_unavailable",
4492 "reserved-id release requires a daemon started with a reloadable config path",
4493 )?]);
4494 };
4495 let operation_lock = self.supervisor.operation_lock();
4496 let _operation_guard = operation_lock.lock().await;
4497 let loaded = match crate::daemon_config::load(&context.config_path) {
4498 Ok(Some(config)) => config,
4499 Ok(None) => {
4500 return Ok(vec![control_error_frame(
4501 &frame,
4502 "invalid_daemon_config",
4503 format!(
4504 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
4505 context.config_path.display()
4506 ),
4507 )?])
4508 }
4509 Err(err) => {
4510 return Ok(vec![control_error_frame(
4511 &frame,
4512 "invalid_daemon_config",
4513 format!("unable to verify reserved-id release against daemon config: {err}"),
4514 )?])
4515 }
4516 };
4517 if loaded
4518 .modules
4519 .iter()
4520 .any(|configured| configured.module_id == module_id)
4521 {
4522 return Ok(vec![control_error_frame(
4523 &frame,
4524 "reserved_module_configured",
4525 format!(
4526 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
4527 ),
4528 )?]);
4529 }
4530 if !self.supervisor.release_retained_reserved_gate(&module_id) {
4531 return Ok(vec![control_error_frame(
4532 &frame,
4533 "reserved_gate_not_retained",
4534 format!(
4535 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
4536 ),
4537 )?]);
4538 }
4539
4540 let response = ClientControlResponse::SupervisorAck {
4541 module_id,
4542 applied: true,
4543 };
4544 Ok(vec![control_response_body_frame(
4545 &frame,
4546 &response,
4547 "ClientControlResponse::SupervisorAck",
4548 )?])
4549 }
4550
4551 async fn reconcile_supervised_modules(
4560 &self,
4561 supervisor: &Supervisor,
4562 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
4563 preview: bool,
4564 ) -> Result<SupervisorRescanResult, String> {
4565 let mut current = BTreeMap::new();
4566 for module in self.supervisor.list() {
4567 let (spec, health) = module.configuration().map_err(|err| {
4568 format!(
4569 "failed to read configuration for module_id '{}': {err}",
4570 module.module_id()
4571 )
4572 })?;
4573 let enabled = module
4574 .status()
4575 .map_err(|err| {
4576 format!(
4577 "failed to read status for module_id '{}': {err}",
4578 module.module_id()
4579 )
4580 })?
4581 .enabled;
4582 current.insert(
4583 module.module_id().to_string(),
4584 (module, spec, health, enabled),
4585 );
4586 }
4587 let configured = configured_modules
4588 .into_iter()
4589 .map(|module| (module.module_id.clone(), module))
4590 .collect::<BTreeMap<_, _>>();
4591
4592 let added = configured
4593 .keys()
4594 .filter(|module_id| !current.contains_key(*module_id))
4595 .cloned()
4596 .collect::<Vec<_>>();
4597 let removed = current
4598 .keys()
4599 .filter(|module_id| !configured.contains_key(*module_id))
4600 .cloned()
4601 .collect::<Vec<_>>();
4602 let mut changed_pending_reload = Vec::new();
4603 let mut configuration_changes = BTreeSet::new();
4604 let mut enabled_changes = BTreeSet::new();
4605 let mut unchanged = 0_u32;
4606
4607 for (module_id, configured_module) in &configured {
4608 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
4609 else {
4610 continue;
4611 };
4612 let launch_changed = *current_spec != configured_module.module_spec();
4616 let configuration_changed =
4617 launch_changed || *current_health != configured_module.health;
4618 let enabled_changed = *current_enabled != configured_module.enabled;
4619 if configuration_changed {
4620 configuration_changes.insert(module_id.clone());
4621 }
4622 if launch_changed {
4623 changed_pending_reload.push(module_id.clone());
4624 }
4625 if enabled_changed {
4626 enabled_changes.insert(module_id.clone());
4627 }
4628 if !configuration_changed && !enabled_changed {
4629 unchanged = unchanged.saturating_add(1);
4630 }
4631 }
4632
4633 if preview {
4639 return Ok(SupervisorRescanResult {
4640 added,
4641 removed,
4642 changed_pending_reload,
4643 enabled_changes: enabled_changes.iter().cloned().collect(),
4644 unchanged,
4645 preview: true,
4646 restart_required: Vec::new(),
4650 capability_warnings: Vec::new(),
4651 });
4652 }
4653
4654 for module_id in &removed {
4655 let module = ¤t
4656 .get(module_id)
4657 .expect("removed module came from current supervisor state")
4658 .0;
4659 module.retire().await.map_err(|err| {
4660 format!("failed to retire module_id '{module_id}' during rescan: {err}")
4661 })?;
4662 self.supervisor.record_rescan_removal(module_id);
4696 self.supervisor.retire(module_id);
4697 self.route_outages.forget(module_id);
4698 }
4699
4700 for module_id in configured.keys() {
4701 let Some((module, _, _, _)) = current.get(module_id) else {
4702 continue;
4703 };
4704 let configured_module = configured
4705 .get(module_id)
4706 .expect("configured module id came from configured map");
4707 if configuration_changes.contains(module_id) {
4708 module
4709 .update_configuration(
4710 configured_module.module_spec(),
4711 configured_module.health.clone(),
4712 configured_module.drain_timeout_ms,
4713 )
4714 .await
4715 .map_err(|err| {
4716 format!(
4717 "failed to update module_id '{module_id}' configuration during rescan: {err}"
4718 )
4719 })?;
4720 }
4721 if enabled_changes.contains(module_id) {
4722 self.route_outages.mark_operator_action(module_id);
4725 module
4726 .set_enabled(configured_module.enabled)
4727 .await
4728 .map_err(|err| {
4729 self.route_outages.operator_action_ended_unrefused(module_id);
4730 format!(
4731 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
4732 configured_module.enabled
4733 )
4734 })?;
4735 }
4736 }
4737
4738 for module_id in &added {
4739 let configured_module = configured
4740 .get(module_id)
4741 .expect("added module id came from configured map");
4742 supervisor
4743 .supervise_configured_with_health(
4744 configured_module.module_spec(),
4745 configured_module.enabled,
4746 configured_module.health.clone(),
4747 configured_module.drain_timeout_ms,
4748 configured_module.restart,
4749 )
4750 .map_err(|err| {
4751 format!("failed to add module_id '{module_id}' during rescan: {err}")
4752 })?;
4753 }
4754
4755 Ok(SupervisorRescanResult {
4756 added,
4757 removed,
4758 changed_pending_reload,
4759 enabled_changes: enabled_changes.iter().cloned().collect(),
4760 unchanged,
4761 preview: false,
4762 restart_required: Vec::new(),
4765 capability_warnings: Vec::new(),
4766 })
4767 }
4768
4769 async fn handle_supervisor_set_enabled(
4770 &self,
4771 frame: Frame,
4772 module_id: String,
4773 enabled: bool,
4774 ) -> Result<Vec<Frame>, RouterError> {
4775 let operation_lock = self.supervisor.operation_lock();
4776 let _operation_guard = operation_lock.lock().await;
4777 let Some(module) = self.supervisor.get(&module_id) else {
4778 return Ok(vec![control_error_frame(
4779 &frame,
4780 "unknown_module",
4781 format!("module_id '{module_id}' is not supervised"),
4782 )?]);
4783 };
4784
4785 self.route_outages.mark_operator_action(&module_id);
4788 let applied = match module.set_enabled(enabled).await {
4789 Ok(applied) => applied,
4790 Err(err) => {
4791 self.route_outages
4792 .operator_action_ended_unrefused(&module_id);
4793 return Ok(vec![control_error_frame(
4794 &frame,
4795 "target_unavailable",
4796 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
4797 )?]);
4798 }
4799 };
4800 if !applied {
4801 self.route_outages
4804 .operator_action_ended_unrefused(&module_id);
4805 }
4806
4807 self.capability_evaluator.wake_deadline_loop();
4808 self.refresh_capability_requirements();
4809 let response = ClientControlResponse::SupervisorAck { module_id, applied };
4810 Ok(vec![control_response_body_frame(
4811 &frame,
4812 &response,
4813 "ClientControlResponse::SupervisorAck",
4814 )?])
4815 }
4816
4817 async fn handle_supervisor_health_probe(
4818 &self,
4819 frame: Frame,
4820 module_id: String,
4821 ) -> Result<Vec<Frame>, RouterError> {
4822 self.refresh_capability_requirements();
4823 let Some(registration) = self
4824 .registry
4825 .get_module(&module_id)
4826 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4827 else {
4828 return Ok(vec![control_error_frame(
4829 &frame,
4830 "unknown_module",
4831 format!("module_id '{module_id}' is not registered"),
4832 )?]);
4833 };
4834
4835 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
4848 {
4849 return Ok(vec![control_error_frame(
4850 &frame,
4851 "health_not_advertised",
4852 format!("module_id '{module_id}' did not advertise health.check"),
4853 )?]);
4854 }
4855
4856 let deadline = Instant::now() + self.health_probe_timeout;
4857 let pending = match self.forwarding.begin_module_control_rpc_for(
4858 &module_id,
4859 MODULE_CONTROL_OP_HEALTH_CHECK,
4860 deadline,
4861 ) {
4862 Ok(pending) => pending,
4863 Err(err) => {
4864 return Ok(vec![control_error_frame(
4865 &frame,
4866 forwarding_error_code(&err),
4867 err.to_string(),
4868 )?])
4869 }
4870 };
4871
4872 let PendingModuleControlRpc {
4873 endpoint,
4874 module_sink,
4875 negotiated_ver,
4876 corr: probe_corr,
4877 receiver,
4878 } = pending;
4879 let mut guard =
4880 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
4881 let probe_body =
4882 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
4883 RouterError::backend(
4884 0,
4885 frame.header.corr,
4886 format!("failed to encode health.check request: {err}"),
4887 )
4888 })?;
4889 let probe_frame = Frame::build_with_version(
4890 negotiated_ver,
4891 FrameType::Request,
4892 control_flags(),
4893 0,
4894 0,
4895 probe_corr,
4896 probe_body,
4897 )
4898 .map_err(RouterError::FrameBuild)?;
4899
4900 if let Err(err) = module_sink.send(probe_frame).await {
4901 return Ok(vec![control_error_frame(
4902 &frame,
4903 "target_unavailable",
4904 err.to_string(),
4905 )?]);
4906 }
4907
4908 match timeout_at(deadline, receiver).await {
4909 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
4910 guard.disarm();
4911 let Some(report) = response.health_report() else {
4912 return Ok(vec![control_error_frame(
4913 &frame,
4914 "invalid_control_body",
4915 "health.check RPC returned a non-health response",
4916 )?]);
4917 };
4918 let HealthReport {
4923 status,
4924 detail,
4925 metrics,
4926 } = report;
4927 let capability_detail = self
4928 .capability_evaluator
4929 .required_problem_detail(&module_id);
4930 let response = ClientControlResponse::SupervisorHealthProbe {
4931 module_id,
4932 status,
4933 detail: append_capability_problem_detail(detail, capability_detail),
4934 metrics,
4935 };
4936 Ok(vec![control_response_body_frame(
4937 &frame,
4938 &response,
4939 "ClientControlResponse::SupervisorHealthProbe",
4940 )?])
4941 }
4942 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
4943 guard.disarm();
4944 Ok(vec![control_error_body_frame(&frame, body)?])
4945 }
4946 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
4947 guard.disarm();
4948 Ok(vec![control_error_frame(
4949 &frame,
4950 "target_unavailable",
4951 message,
4952 )?])
4953 }
4954 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
4955 guard.disarm();
4956 Ok(vec![control_error_frame(
4957 &frame,
4958 "invalid_control_body",
4959 message,
4960 )?])
4961 }
4962 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
4963 guard.disarm();
4964 Ok(vec![control_error_frame(
4965 &frame,
4966 "invalid_control_body",
4967 format!("expected module-control op '{expected}', got '{actual}'"),
4968 )?])
4969 }
4970 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
4971 guard.disarm();
4972 Ok(vec![control_error_frame(
4973 &frame,
4974 "module_timeout",
4975 format!(
4976 "module_id '{module_id}' answered health.check after {:?}",
4977 self.health_probe_timeout
4978 ),
4979 )?])
4980 }
4981 Ok(Err(_)) => Ok(vec![control_error_frame(
4982 &frame,
4983 "target_unavailable",
4984 "health.check waiter was canceled before the module responded",
4985 )?]),
4986 Err(_) => Ok(vec![control_error_frame(
4987 &frame,
4988 "module_timeout",
4989 format!(
4990 "module_id '{module_id}' did not answer health.check within {:?}",
4991 self.health_probe_timeout
4992 ),
4993 )?]),
4994 }
4995 }
4996
4997 fn supervisor_status(
4998 &self,
4999 module_id: &str,
5000 corr: u64,
5001 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
5002 self.supervisor
5003 .get(module_id)
5004 .map(|module| {
5005 let warming = module.is_warming_for_control("status").map_err(|err| {
5006 RouterError::backend(
5007 0,
5008 corr,
5009 format!(
5010 "failed to read supervisor warming state for module_id '{module_id}': {err}"
5011 ),
5012 )
5013 })?;
5014 module.status_for_control("status").map_err(|err| {
5015 RouterError::backend(
5016 0,
5017 corr,
5018 format!(
5019 "failed to read supervisor status for module_id '{module_id}': {err}"
5020 ),
5021 )
5022 }).map(|status| (status, warming))
5023 })
5024 .transpose()
5025 }
5026
5027 fn guard_module_control_op(
5028 &self,
5029 frame: &Frame,
5030 module_id: &str,
5031 op: &str,
5032 ) -> Result<Option<Frame>, RouterError> {
5033 if self.module_grants_op(module_id, op, frame.header.corr)? {
5034 return Ok(None);
5035 }
5036
5037 Ok(Some(control_error_frame(
5038 frame,
5039 "op_not_allowed",
5040 format!("module_id '{module_id}' did not grant control op '{op}'"),
5041 )?))
5042 }
5043
5044 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
5045 let Some(registration) = self
5046 .registry
5047 .get_module(module_id)
5048 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
5049 else {
5050 return Ok(false);
5051 };
5052 Ok(module_registration_grants_op(®istration.control_ops, op))
5053 }
5054
5055 fn handle_status_update(
5056 &self,
5057 endpoint: ModuleEndpointId,
5058 frame: Frame,
5059 ) -> Result<Vec<Frame>, RouterError> {
5060 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
5061 Ok(update) => update,
5062 Err(err) => {
5063 if is_known_module_push_op(&frame.body) {
5068 return Ok(vec![control_error_frame(
5069 &frame,
5070 "invalid_control_body",
5071 format!("malformed module control push body: {err}"),
5072 )?]);
5073 }
5074 return Ok(Vec::new());
5075 }
5076 };
5077
5078 match update {
5079 ModuleControlPush::RouteStatus {
5080 route_channel,
5081 route_epoch,
5082 status,
5083 } => {
5084 self.forwarding
5085 .cache_status(endpoint, route_channel, route_epoch, status)
5086 .map_err(RouterError::Forwarding)?;
5087 }
5088 }
5089 Ok(Vec::new())
5090 }
5091
5092 fn handle_route_poll(
5093 &self,
5094 ctx: &RouteCtx,
5095 frame: Frame,
5096 route_channel: u16,
5097 route_epoch: u32,
5098 kind: PollKind,
5099 ) -> Result<Vec<Frame>, RouterError> {
5100 let snapshot = self
5101 .forwarding
5102 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
5103 .map_err(RouterError::Forwarding)?;
5104 let response = match (kind, snapshot) {
5105 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
5106 ClientControlResponse::RoutePoll {
5107 route_channel,
5108 route_epoch,
5109 status,
5110 live: None,
5111 }
5112 }
5113 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
5114 route_channel,
5115 route_epoch,
5116 status: None,
5117 live: None,
5118 },
5119 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
5120 let live = self
5146 .process_liveness
5147 .as_ref()
5148 .and_then(|source| source.process_live(&module_id))
5149 .unwrap_or(true);
5150 ClientControlResponse::RoutePoll {
5151 route_channel,
5152 route_epoch,
5153 status: None,
5154 live: Some(live),
5155 }
5156 }
5157 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
5158 route_channel,
5159 route_epoch,
5160 status: None,
5161 live: Some(false),
5162 },
5163 };
5164
5165 Ok(vec![control_response_body_frame(
5166 &frame,
5167 &response,
5168 "ClientControlResponse::RoutePoll",
5169 )?])
5170 }
5171
5172 pub(crate) fn observe_module_control_completion(
5173 &self,
5174 completion: ModuleControlRpcCompletion,
5175 ) -> bool {
5176 match completion {
5177 ModuleControlRpcCompletion::Unknown => false,
5178 ModuleControlRpcCompletion::Settled => true,
5179 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
5180 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
5181 info!(
5182 module_id = %module_id,
5183 latency_ms,
5184 "late health.check answer proves the module is alive"
5185 );
5186 match self
5187 .supervisor
5188 .record_late_health_answer(&module_id, latency_ms)
5189 {
5190 Ok(true) => {}
5191 Ok(false) => debug!(
5192 module_id = %module_id,
5193 latency_ms,
5194 "late health.check answer has no active supervisor snapshot"
5195 ),
5196 Err(err) => warn!(
5197 module_id = %module_id,
5198 latency_ms,
5199 error = %err,
5200 "failed to record late health.check answer"
5201 ),
5202 }
5203 true
5204 }
5205 }
5206 }
5207
5208 fn refuse_to_end_module_connection_for_a_client(
5229 &self,
5230 module_connection_id: ConnectionId,
5231 corr: u64,
5232 err: ForwardingError,
5233 ) -> Result<(), RouterError> {
5234 if let ForwardingError::ConnectionClosing { connection_id } = err {
5235 if connection_id != module_connection_id {
5236 warn!(
5237 module_connection_id = module_connection_id.get(),
5238 client_connection_id = connection_id.get(),
5239 corr,
5240 "dropping a route.bind response for a closing client; the module connection keeps serving"
5241 );
5242 return Ok(());
5243 }
5244 }
5245 Err(RouterError::Forwarding(err))
5246 }
5247
5248 fn handle_module_relay_response(
5249 &self,
5250 connection_id: ConnectionId,
5251 frame: Frame,
5252 ) -> Result<Vec<Frame>, RouterError> {
5253 let mut secondary_error = None;
5254 let outcome = match frame.header.ty {
5255 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
5256 Ok(probe) if probe.op == "route.bind" => {
5257 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
5258 Ok(ModuleControlResponse::RouteBindAck {}) => {
5259 RouteBindRelayOutcome::Accepted
5260 }
5261 Ok(other) => {
5262 let message =
5263 format!("route.bind response carried unexpected body: {other:?}");
5264 secondary_error = Some(control_error_frame(
5265 &frame,
5266 "invalid_control_body",
5267 message.clone(),
5268 )?);
5269 RouteBindRelayOutcome::ModuleGone(message)
5270 }
5271 Err(err) => {
5272 let message = format!("malformed route.bind response body: {err}");
5273 secondary_error = Some(control_error_frame(
5274 &frame,
5275 "invalid_control_body",
5276 message.clone(),
5277 )?);
5278 RouteBindRelayOutcome::ModuleGone(message)
5279 }
5280 }
5281 }
5282 Ok(probe) => {
5283 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
5284 {
5285 Ok(response) => ModuleControlRpcOutcome::Response(response),
5286 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
5287 "malformed {} response body: {err}",
5288 probe.op
5289 )),
5290 };
5291 let completion = self
5292 .forwarding
5293 .complete_module_control_rpc(
5294 connection_id,
5295 frame.header.corr,
5296 Some(&probe.op),
5297 outcome,
5298 )
5299 .map_err(RouterError::Forwarding)?;
5300 if !self.observe_module_control_completion(completion) {
5301 debug!(
5302 connection_id = connection_id.get(),
5303 corr = frame.header.corr,
5304 op = %probe.op,
5305 "dropping late or unknown module-control RPC response"
5306 );
5307 }
5308 return Ok(Vec::new());
5309 }
5310 Err(err) => {
5311 if let Some(expected_op) = self
5312 .forwarding
5313 .pending_module_control_op(connection_id, frame.header.corr)
5314 .map_err(RouterError::Forwarding)?
5315 {
5316 let completion = self
5317 .forwarding
5318 .complete_module_control_rpc(
5319 connection_id,
5320 frame.header.corr,
5321 None,
5322 ModuleControlRpcOutcome::MalformedResponse(format!(
5323 "malformed {expected_op} response body: {err}"
5324 )),
5325 )
5326 .map_err(RouterError::Forwarding)?;
5327 if !self.observe_module_control_completion(completion) {
5328 debug!(
5329 connection_id = connection_id.get(),
5330 corr = frame.header.corr,
5331 "dropping late malformed module-control RPC response"
5332 );
5333 }
5334 return Ok(Vec::new());
5335 }
5336 let message = format!("malformed route.bind response body: {err}");
5337 secondary_error = Some(control_error_frame(
5338 &frame,
5339 "invalid_control_body",
5340 message.clone(),
5341 )?);
5342 RouteBindRelayOutcome::ModuleGone(message)
5343 }
5344 },
5345 FrameType::Error => {
5346 if self
5347 .forwarding
5348 .pending_module_control_op(connection_id, frame.header.corr)
5349 .map_err(RouterError::Forwarding)?
5350 .is_some()
5351 {
5352 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
5353 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
5354 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
5355 "malformed module-control ERROR body: {err}"
5356 )),
5357 };
5358 let completion = self
5359 .forwarding
5360 .complete_module_control_rpc(
5361 connection_id,
5362 frame.header.corr,
5363 None,
5364 outcome,
5365 )
5366 .map_err(RouterError::Forwarding)?;
5367 if !self.observe_module_control_completion(completion) {
5368 debug!(
5369 connection_id = connection_id.get(),
5370 corr = frame.header.corr,
5371 "dropping late or unknown module-control RPC error"
5372 );
5373 }
5374 return Ok(Vec::new());
5375 }
5376 match serde_json::from_slice::<ErrorBody>(&frame.body) {
5377 Ok(body) => RouteBindRelayOutcome::Rejected(body),
5378 Err(err) => {
5379 let message = format!("malformed route.bind ERROR body: {err}");
5380 secondary_error = Some(control_error_frame(
5381 &frame,
5382 "invalid_control_body",
5383 message.clone(),
5384 )?);
5385 RouteBindRelayOutcome::ModuleGone(message)
5386 }
5387 }
5388 }
5389 ty => {
5390 return Ok(vec![control_error_frame(
5391 &frame,
5392 "unsupported_control_frame",
5393 format!("unsupported module channel-0 frame {ty:?}"),
5394 )?])
5395 }
5396 };
5397
5398 let settled =
5399 self.forwarding
5400 .complete_pending_relay(connection_id, frame.header.corr, outcome);
5401 let completion = match settled {
5402 Ok(completion) => completion,
5403 Err(err) => {
5404 self.refuse_to_end_module_connection_for_a_client(
5405 connection_id,
5406 frame.header.corr,
5407 err,
5408 )?;
5409 return Ok(secondary_error.into_iter().collect());
5410 }
5411 };
5412 if let Some(target) = completion.abandoned.as_ref() {
5413 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
5414 }
5415 if !completion.settled {
5416 debug!(
5417 connection_id = connection_id.get(),
5418 corr = frame.header.corr,
5419 frame_type = ?frame.header.ty,
5420 "dropping late or unknown route.bind relay response"
5421 );
5422 }
5423 Ok(secondary_error.into_iter().collect())
5424 }
5425
5426 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
5427 debug!(connection_id = connection_id.get(), "handling GOODBYE");
5428 self.cleanup_connection(connection_id)
5432 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
5433 Ok(Vec::new())
5434 }
5435}
5436
5437impl Default for ControlHandler {
5438 fn default() -> Self {
5439 Self::new(Arc::new(Registry::default()))
5440 }
5441}
5442
5443impl crate::supervise::SwapPromotionObserver for ControlHandler {
5444 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
5445 self.apply_registration_capabilities(registration);
5446 }
5447}
5448
5449fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
5450 CapabilityRequirementStatus {
5451 consumer: status.consumer,
5452 capability: status.capability,
5453 need: match status.need {
5454 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
5455 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
5456 },
5457 verdict: status.verdict.as_str().to_string(),
5458 episode_seq: status.episode_seq,
5459 config_satisfiable: status.config_satisfiable,
5460 runtime_available: status.runtime_available,
5461 detail: status.detail,
5462 }
5463}
5464
5465fn append_capability_problem_detail(
5466 detail: Option<String>,
5467 capability_detail: Option<String>,
5468) -> Option<String> {
5469 match (detail, capability_detail) {
5470 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
5471 (Some(detail), None) => Some(detail),
5472 (None, Some(capability_detail)) => Some(capability_detail),
5473 (None, None) => None,
5474 }
5475}
5476
5477fn subc_ops() -> Vec<String> {
5478 SUBC_CONTROL_OPS
5479 .iter()
5480 .map(|op| (*op).to_string())
5481 .collect()
5482}
5483
5484fn module_subc_ops() -> Vec<String> {
5485 SUBC_CONTROL_OPS
5486 .iter()
5487 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
5488 .map(|op| (*op).to_string())
5489 .collect()
5490}
5491
5492#[cfg(test)]
5493fn module_baseline_control_ops() -> Vec<String> {
5494 MODULE_BASELINE_CONTROL_OPS
5495 .iter()
5496 .map(|op| (*op).to_string())
5497 .collect()
5498}
5499
5500fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
5501 let mut seen = HashSet::new();
5502 let mut effective = Vec::new();
5503 for op in MODULE_BASELINE_CONTROL_OPS {
5504 if seen.insert((*op).to_string()) {
5505 effective.push((*op).to_string());
5506 }
5507 }
5508 for op in declared.unwrap_or_default() {
5509 if seen.insert(op.clone()) {
5510 effective.push(op);
5511 }
5512 }
5513 effective
5514}
5515
5516fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
5517 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
5518}
5519
5520fn target_module_id(target: &RouteTarget) -> &str {
5521 match target {
5522 RouteTarget::ToolProvider { module_id }
5523 | RouteTarget::ManagementSurface { module_id }
5524 | RouteTarget::InternalService { module_id, .. } => module_id,
5525 }
5526}
5527
5528fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
5529 roles.iter().any(|role| match (target, role) {
5530 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
5531 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
5532 (
5533 RouteTarget::InternalService { service_id, .. },
5534 ProviderRole::InternalService {
5535 service_id: provided,
5536 ..
5537 },
5538 ) => service_id == provided,
5539 _ => false,
5540 })
5541}
5542
5543fn is_routable_role(role: &ProviderRole) -> bool {
5544 matches!(
5545 role,
5546 ProviderRole::ToolProvider { .. }
5547 | ProviderRole::ManagementSurface { .. }
5548 | ProviderRole::InternalService { .. }
5549 )
5550}
5551
5552#[derive(Debug, Clone, Copy, PartialEq, Eq)]
5553enum ControlRequestBodyError {
5554 UnknownOp,
5555 InvalidBody,
5556}
5557
5558#[derive(Debug, Deserialize)]
5559struct ControlOpProbe {
5560 op: String,
5561}
5562
5563const MODULE_PUSH_OPS: &[&str] = &["route.status"];
5566
5567fn is_known_module_push_op(body: &[u8]) -> bool {
5568 serde_json::from_slice::<ControlOpProbe>(body)
5569 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
5570 .unwrap_or(false)
5571}
5572
5573fn is_known_module_request_op(body: &[u8]) -> bool {
5574 serde_json::from_slice::<ControlOpProbe>(body)
5575 .map(|probe| is_module_to_subc_op(&probe.op))
5576 .unwrap_or(false)
5577}
5578
5579fn is_module_to_subc_op(op: &str) -> bool {
5580 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
5581}
5582
5583fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
5584 debug!(
5585 op = %op,
5586 connection_id = connection_id.get(),
5587 corr,
5588 "control dispatch"
5589 );
5590}
5591
5592fn log_slow_control_dispatch(
5593 dispatch_started_at: Option<StdInstant>,
5594 op: &'static str,
5595 connection_id: ConnectionId,
5596 corr: u64,
5597) {
5598 let Some(dispatch_started_at) = dispatch_started_at else {
5599 return;
5600 };
5601 let elapsed = dispatch_started_at.elapsed();
5602 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
5603 warn!(
5604 op = %op,
5605 connection_id = connection_id.get(),
5606 corr,
5607 elapsed_ms = elapsed.as_millis() as u64,
5608 "slow control dispatch"
5609 );
5610 }
5611}
5612
5613fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
5614 match request {
5615 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
5616 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
5617 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
5618 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
5619 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
5620 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
5621 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
5622 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
5623 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
5624 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
5625 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
5626 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
5627 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
5628 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
5629 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
5630 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
5631 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
5632 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
5633 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
5634 }
5635}
5636
5637fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
5638 match request {
5639 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
5640 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
5641 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
5642 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
5643 }
5644}
5645
5646fn parse_client_control_request(
5647 body: &[u8],
5648) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
5649 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
5650 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5651 Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
5652 ControlRequestBodyError::InvalidBody
5653 }
5654 Ok(_) => ControlRequestBodyError::UnknownOp,
5655 Err(_) => ControlRequestBodyError::InvalidBody,
5656 };
5657 (err, classification)
5658 })
5659}
5660
5661fn parse_module_control_request_from_module(
5662 body: &[u8],
5663) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
5664 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
5665 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5666 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
5667 Ok(_) => ControlRequestBodyError::UnknownOp,
5668 Err(_) => ControlRequestBodyError::InvalidBody,
5669 };
5670 (err, classification)
5671 })
5672}
5673
5674#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
5675enum ProviderRoleKind {
5676 ToolProvider,
5677 PipelineStage,
5678 ManagementSurface,
5679 InternalService,
5680}
5681
5682fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
5683 match role {
5684 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
5685 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
5686 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
5687 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
5688 }
5689}
5690
5691fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
5692 roles.iter().map(provider_role_kind).collect()
5693}
5694
5695const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
5698
5699#[derive(Debug, Default, PartialEq, Eq)]
5701struct ScopeOutcomeCounts {
5702 created: usize,
5703 replaced: usize,
5704 updated: usize,
5705 unchanged: usize,
5706 refused: usize,
5707}
5708
5709impl ScopeOutcomeCounts {
5710 fn of(results: &[ScopeRecordResult]) -> Self {
5711 let mut counts = Self::default();
5712 for result in results {
5713 let slot = match result.outcome {
5714 ScopeRecordOutcome::Created => &mut counts.created,
5715 ScopeRecordOutcome::Replaced => &mut counts.replaced,
5716 ScopeRecordOutcome::Updated => &mut counts.updated,
5717 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
5718 ScopeRecordOutcome::Refused => &mut counts.refused,
5719 };
5720 *slot += 1;
5721 }
5722 counts
5723 }
5724}
5725
5726#[cfg(test)]
5727mod scope_outcome_count_tests {
5728 use super::*;
5729
5730 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
5731 ScopeRecordResult {
5732 scope_ref: "r".to_string(),
5733 scope_epoch: 1,
5734 outcome,
5735 code: None,
5736 message: None,
5737 version: None,
5738 parent_state: None,
5739 }
5740 }
5741
5742 #[test]
5745 fn every_outcome_is_counted_in_its_own_field() {
5746 let results = [
5747 result(ScopeRecordOutcome::Created),
5748 result(ScopeRecordOutcome::Created),
5749 result(ScopeRecordOutcome::Replaced),
5750 result(ScopeRecordOutcome::Updated),
5751 result(ScopeRecordOutcome::Unchanged),
5752 result(ScopeRecordOutcome::Refused),
5753 result(ScopeRecordOutcome::Refused),
5754 result(ScopeRecordOutcome::Refused),
5755 ];
5756 assert_eq!(
5757 ScopeOutcomeCounts::of(&results),
5758 ScopeOutcomeCounts {
5759 created: 2,
5760 replaced: 1,
5761 updated: 1,
5762 unchanged: 1,
5763 refused: 3,
5764 }
5765 );
5766 }
5767}
5768
5769fn capability_census_trigger(
5773 old: Option<&CapabilityDeclarations>,
5774 new: Option<&CapabilityDeclarations>,
5775) -> bool {
5776 let old_provides = old
5777 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
5778 .unwrap_or_default();
5779 let old_denies = old
5780 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
5781 .unwrap_or_default();
5782 let new = new.cloned().unwrap_or(CapabilityDeclarations {
5783 provides: Vec::new(),
5784 requires: Vec::new(),
5785 must_never_reach: Vec::new(),
5786 });
5787
5788 new.provides
5789 .iter()
5790 .any(|capability| !old_provides.contains(capability))
5791 || new
5792 .must_never_reach
5793 .iter()
5794 .any(|capability| !old_denies.contains(capability))
5795}
5796
5797fn denied_capability<'a>(
5800 opening_manifest: &'a ModuleManifest,
5801 target_manifest: &ModuleManifest,
5802) -> Option<&'a str> {
5803 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
5804 let target_capabilities = target_manifest.capabilities.as_ref()?;
5805 opening_capabilities
5806 .must_never_reach
5807 .iter()
5808 .find(|denied| {
5809 target_capabilities
5810 .provides
5811 .iter()
5812 .any(|provided| provided == *denied)
5813 })
5814 .map(String::as_str)
5815}
5816
5817fn catalog_update_frozen_field_message(
5818 registered: &ModuleManifest,
5819 provides: &[ProviderRole],
5820) -> Option<String> {
5821 let old_has_provides = !registered.provides.is_empty();
5822 let new_has_provides = !provides.is_empty();
5823 if old_has_provides != new_has_provides {
5824 return Some(format!(
5825 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
5826 registered.module_id
5827 ));
5828 }
5829
5830 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
5831 return Some(format!(
5832 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
5833 registered.module_id
5834 ));
5835 }
5836
5837 let registered_concurrency = manifest_concurrency(registered);
5838 let mut candidate = registered.clone();
5839 candidate.provides = provides.to_vec();
5840 let candidate_concurrency = manifest_concurrency(&candidate);
5841 if candidate_concurrency != registered_concurrency {
5842 return Some(format!(
5843 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
5844 registered.module_id, registered_concurrency, candidate_concurrency
5845 ));
5846 }
5847
5848 None
5851}
5852
5853fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
5854 manifest.provides.iter().any(is_routable_role)
5855}
5856
5857fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
5863 manifest
5864 .provides
5865 .iter()
5866 .find_map(|provider| match provider {
5867 ProviderRole::ToolProvider { concurrency, .. }
5868 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
5869 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
5870 })
5871 .unwrap_or(Concurrency::ModuleManaged)
5872}
5873
5874fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
5881 let has_management_surface = manifest
5882 .provides
5883 .iter()
5884 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
5885 if !has_management_surface {
5886 return false;
5887 }
5888 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
5889 return false;
5890 };
5891 let Some(provides) = raw
5892 .get("manifest")
5893 .and_then(|manifest| manifest.get("provides"))
5894 .and_then(serde_json::Value::as_array)
5895 else {
5896 return false;
5897 };
5898 provides.iter().any(|role| {
5903 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
5904 && role.get("concurrency").is_none()
5905 })
5906}
5907
5908fn negotiate_version(peer_version: u8) -> Result<u8, String> {
5909 if peer_version != PROTOCOL_VERSION {
5910 return Err(format!(
5911 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
5912 ));
5913 }
5914 Ok(PROTOCOL_VERSION)
5915}
5916
5917fn pong(frame: &Frame) -> Result<Frame, RouterError> {
5918 Frame::build_with_version(
5919 response_version(frame),
5920 FrameType::Pong,
5921 frame.header.flags,
5922 0,
5923 0,
5924 frame.header.corr,
5925 Vec::new(),
5926 )
5927 .map_err(RouterError::FrameBuild)
5928}
5929
5930fn control_error_frame(
5931 frame: &Frame,
5932 code: &'static str,
5933 message: impl Into<String>,
5934) -> Result<Frame, RouterError> {
5935 control_error_body_frame(
5936 frame,
5937 ErrorBody {
5938 code: code.to_string(),
5939 message: message.into(),
5940 detail: None,
5941 },
5942 )
5943}
5944
5945fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
5946 let body = serde_json::to_vec(&error).map_err(|err| {
5947 RouterError::backend(
5948 0,
5949 frame.header.corr,
5950 format!("failed to encode control ERROR: {err}"),
5951 )
5952 })?;
5953
5954 Frame::build_with_version(
5955 response_version(frame),
5956 FrameType::Error,
5957 control_flags(),
5958 0,
5959 0,
5960 frame.header.corr,
5961 body,
5962 )
5963 .map_err(RouterError::FrameBuild)
5964}
5965
5966fn control_response_body_frame<T: Serialize>(
5967 frame: &Frame,
5968 reply: &T,
5969 label: &'static str,
5970) -> Result<Frame, RouterError> {
5971 let body = serde_json::to_vec(reply).map_err(|err| {
5972 RouterError::backend(
5973 0,
5974 frame.header.corr,
5975 format!("failed to encode {label}: {err}"),
5976 )
5977 })?;
5978
5979 Frame::build_with_version(
5980 response_version(frame),
5981 FrameType::Response,
5982 control_flags(),
5983 0,
5984 0,
5985 frame.header.corr,
5986 body,
5987 )
5988 .map_err(RouterError::FrameBuild)
5989}
5990
5991fn forwarding_error_code(err: &ForwardingError) -> &'static str {
6021 match err {
6022 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
6023 ForwardingError::NoModuleConnection => "target_unavailable",
6024 ForwardingError::ModuleReloading { .. } => "module_reloading",
6025 ForwardingError::ClientRouteChannelExhausted { .. }
6026 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
6027 ForwardingError::StaleModuleEndpoint
6028 | ForwardingError::UnknownReservation { .. }
6029 | ForwardingError::ConnectionClosing { .. }
6030 | ForwardingError::ClientEgressClosed { .. }
6031 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
6032 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
6035 ForwardingError::RelayCorrelationExhausted
6036 | ForwardingError::RouteOpenBuild(_)
6037 | ForwardingError::Poisoned => "forwarding_error",
6038 }
6039}
6040
6041fn response_version(frame: &Frame) -> u8 {
6042 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
6043 frame.header.ver
6044 } else {
6045 PROTOCOL_VERSION
6046 }
6047}
6048
6049fn control_flags() -> Flags {
6050 Flags::new(false, Priority::Passive, false)
6051}
6052
6053fn send_goodbye_target_best_effort(
6058 counters: &DaemonCounters,
6059 target: &GoodbyeTarget,
6060 context: &'static str,
6061) {
6062 let Ok(frame) = Frame::build_with_version(
6063 target.negotiated_ver,
6064 FrameType::Goodbye,
6065 control_flags(),
6066 target.channel,
6067 target.epoch,
6068 0,
6069 Vec::new(),
6070 ) else {
6071 return;
6072 };
6073 crate::forwarding::send_module_route_goodbye(
6074 counters,
6075 &target.sink,
6076 frame,
6077 target.module_id.as_deref(),
6078 context,
6079 );
6080}
6081
6082pub(crate) fn send_route_control_pushes(
6083 forwarding: &ForwardingTable,
6084 routes: Vec<EndpointRoute>,
6085 push: ClientControlPush,
6086) {
6087 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
6088 for route in routes {
6089 let target = route.goodbye_target;
6090 if let Some((existing, channels)) = targets
6091 .iter_mut()
6092 .find(|(existing, _)| existing.connection_id == target.connection_id)
6093 {
6094 debug_assert_eq!(
6095 existing.negotiated_ver, target.negotiated_ver,
6096 "one connection cannot negotiate multiple frame versions"
6097 );
6098 if !channels.contains(&target.channel) {
6099 channels.push(target.channel);
6100 }
6101 continue;
6102 }
6103 let channel = target.channel;
6104 targets.push((target, vec![channel]));
6105 }
6106 for (target, mut channels) in targets {
6107 channels.sort_unstable();
6108 let mut push = push.clone();
6109 match &mut push {
6110 ClientControlPush::RouteClosing {
6111 channels: covered, ..
6112 }
6113 | ClientControlPush::RouteClosed {
6114 channels: covered, ..
6115 } => *covered = channels,
6116 }
6117 let body = match serde_json::to_vec(&push) {
6118 Ok(body) => body,
6119 Err(err) => {
6120 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
6121 continue;
6122 }
6123 };
6124 let frame = match Frame::build_with_version(
6125 target.negotiated_ver,
6126 FrameType::Push,
6127 control_flags(),
6128 0,
6129 0,
6130 0,
6131 body.clone(),
6132 ) {
6133 Ok(frame) => frame,
6134 Err(err) => {
6135 warn!(
6136 route_channel = target.channel,
6137 error = %err,
6138 "failed to build route lifecycle control PUSH frame"
6139 );
6140 continue;
6141 }
6142 };
6143 if let Err(err) = target.sink.try_send(frame) {
6144 if target.close_on_delivery_failure() {
6145 warn!(
6146 target_connection_id = target.connection_id.get(),
6147 route_channel = target.channel,
6148 error = %err,
6149 "route lifecycle control PUSH was not delivered to client; closing target connection"
6150 );
6151 let _ = forwarding.escalate_client_delivery_failure(
6152 target.connection_id,
6153 target.channel,
6154 target.epoch,
6155 CloseReason::new(
6156 "route_lifecycle_push_delivery_failed",
6157 format!(
6158 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
6159 target.channel
6160 ),
6161 ),
6162 crate::forwarding::UndeliveredFrame {
6163 module_id: target.module_id.as_deref(),
6164 sink: &target.sink,
6165 },
6166 );
6167 }
6168 }
6169 }
6170}
6171
6172#[cfg(test)]
6173mod tests {
6174 #[cfg(unix)]
6175 #[tokio::test]
6176 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
6177 let dir = subc_test_support::TestTempDir::new("rescan-live-health");
6178 let path = dir.join("subc.jsonc");
6179 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
6180 "program":"/bin/sleep","args":["60"],"protocol":"none",
6181 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
6182 }}}).to_string()).unwrap();
6183 let mut configured = crate::daemon_config::load(&path)
6184 .unwrap()
6185 .unwrap()
6186 .modules
6187 .pop()
6188 .unwrap();
6189 let registry = std::sync::Arc::new(crate::Registry::default());
6190 let handle = crate::SupervisorHandle::new();
6191 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
6192 .with_handle(handle.clone());
6193 let module = supervisor
6194 .supervise_configured_with_health(
6195 configured.module_spec(),
6196 true,
6197 configured.health.clone(),
6198 None,
6199 configured.restart,
6200 )
6201 .unwrap();
6202 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
6203 let before = module.status().unwrap().pid;
6204 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
6205 configured.health.cadence = std::time::Duration::from_secs(3600);
6206 let health_only = handler
6207 .reconcile_supervised_modules(&supervisor, vec![configured.clone()], false)
6208 .await
6209 .unwrap();
6210 assert!(
6211 health_only.changed_pending_reload.is_empty(),
6212 "health policy is already applied live"
6213 );
6214 assert_eq!(module.status().unwrap().pid, before);
6215 assert_eq!(
6216 module.configuration().unwrap().1.http,
6217 configured.health.http
6218 );
6219 configured.args = vec!["61".into()];
6220 let launch = handler
6221 .reconcile_supervised_modules(&supervisor, vec![configured], false)
6222 .await
6223 .unwrap();
6224 assert_eq!(launch.changed_pending_reload, ["stock"]);
6225 assert_eq!(
6226 module.status().unwrap().pid,
6227 before,
6228 "a launch edit is stored until reload"
6229 );
6230 module.drain().await.unwrap();
6231 }
6232 use std::{
6233 collections::BTreeMap,
6234 fmt,
6235 path::PathBuf,
6236 sync::{Arc, Mutex},
6237 time::Duration,
6238 };
6239 use subc_test_support::TestTempDir;
6240
6241 use serde_json::{json, Value};
6242 use subc_protocol::{
6243 manifest::{
6244 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
6245 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
6246 },
6247 session::HealthStatus,
6248 FrameType,
6249 };
6250
6251 use super::*;
6252 use crate::{
6253 forwarding::{DataRoute, DataRouteState},
6254 registry::ChannelState,
6255 router::FrameSink,
6256 stderr_tail::DEFAULT_MAX_LINE_BYTES,
6257 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
6258 RouteCtx, Router,
6259 };
6260 use tokio::{
6261 sync::mpsc,
6262 time::{sleep, Instant},
6263 };
6264 use tracing::{
6265 field::{Field, Visit},
6266 Event, Subscriber,
6267 };
6268 use tracing_subscriber::{layer::Context, prelude::*, Layer};
6269
6270 fn fake_aft_stub_path() -> PathBuf {
6289 let mut path = std::env::current_exe().expect("current_exe available in tests");
6290 path.pop(); path.pop(); path.push(if cfg!(windows) {
6293 "fake-aft-stub.exe"
6294 } else {
6295 "fake-aft-stub"
6296 });
6297 assert!(
6298 path.exists(),
6299 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
6300 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
6301 path.display()
6302 );
6303 path
6304 }
6305
6306 fn client_retries(code: &str) -> bool {
6311 subc_protocol::error_codes::is_retryable_route_open(code)
6312 }
6313
6314 #[test]
6325 fn retryability_of_forwarding_codes_matches_the_failure() {
6326 let transient = [
6329 ForwardingError::NoModuleConnection,
6330 ForwardingError::ModuleReloading {
6331 module_id: "m".into(),
6332 },
6333 ForwardingError::StaleModuleEndpoint,
6334 ForwardingError::UnknownReservation {
6335 client_channel: 1,
6336 module_channel: 1,
6337 },
6338 ForwardingError::ConnectionClosing {
6339 connection_id: ConnectionId::new(1),
6340 },
6341 ForwardingError::ClientEgressClosed {
6342 connection_id: ConnectionId::new(1),
6343 },
6344 ForwardingError::ModuleEgressUnavailable {
6345 connection_id: ConnectionId::new(1),
6346 },
6347 ];
6348 for err in transient {
6349 let code = forwarding_error_code(&err);
6350 assert!(
6351 client_retries(code),
6352 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
6353 );
6354 }
6355
6356 let permanent = [
6361 ForwardingError::ConnectionRoleConflict {
6362 connection_id: ConnectionId::new(1),
6363 },
6364 ForwardingError::ClientRouteChannelExhausted {
6365 connection_id: ConnectionId::new(1),
6366 },
6367 ForwardingError::ModuleRouteChannelExhausted {
6368 endpoint: ModuleEndpointId {
6369 connection_id: ConnectionId::new(1),
6370 generation: 1,
6371 },
6372 },
6373 ForwardingError::RelayCorrelationExhausted,
6374 ForwardingError::RouteOpenBuild("x".into()),
6375 ForwardingError::Poisoned,
6376 ];
6377 for err in permanent {
6378 let code = forwarding_error_code(&err);
6379 assert!(
6380 !client_retries(code),
6381 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
6382 );
6383 }
6384 }
6385
6386 #[tokio::test]
6414 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
6415 let handler = ControlHandler::default();
6416 let frame =
6417 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
6418
6419 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
6423 assert_eq!(
6424 stamped,
6425 Principal::Direct,
6426 "a caller that proved nothing must not be stamped as a supervised module"
6427 );
6428
6429 let forged = handler
6434 .route_open_principal(
6435 &frame,
6436 Some(ConsumerIdentity {
6437 module_id: "aft".to_string(),
6438 launch_nonce: "not-a-real-nonce".to_string(),
6439 }),
6440 )
6441 .unwrap();
6442 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
6443 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
6444 }
6445
6446 #[test]
6463 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
6464 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
6465 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
6466 let ClientControlRequest::RouteOpen {
6467 consumer_identity, ..
6468 } = parsed
6469 else {
6470 panic!("route.open body must parse as RouteOpen");
6471 };
6472 assert_eq!(
6473 consumer_identity,
6474 Some(ConsumerIdentity {
6475 module_id: "aft".to_string(),
6476 launch_nonce: "n".to_string(),
6477 }),
6478 "the wire field name must reach the value route_open_principal reads"
6479 );
6480 }
6481
6482 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
6483 ModuleManifest::builder(module_id, "0.1.0")
6484 .protocol_ver(protocol_ver)
6485 .provides(vec![ProviderRole::ToolProvider {
6486 tools: vec![Tool {
6487 name: "read".to_string(),
6488 description: None,
6489 execution_mode: ExecutionMode::Pure,
6490 schema: json!({"type": "object"}),
6491 }],
6492 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
6493 concurrency: Concurrency::ModuleManaged,
6494 emits_push: true,
6495 sub_supervises: true,
6496 }])
6497 .build()
6498 }
6499
6500 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
6501 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
6502 }
6503
6504 fn hello_frame_with_control_ops(
6505 module_id: &str,
6506 protocol_ver: u8,
6507 corr: u64,
6508 control_ops: Option<Vec<String>>,
6509 ) -> Frame {
6510 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
6511 }
6512
6513 fn hello_frame_with_nonce(
6514 module_id: &str,
6515 protocol_ver: u8,
6516 corr: u64,
6517 launch_nonce: Option<&str>,
6518 ) -> Frame {
6519 hello_frame_full(
6520 module_id,
6521 protocol_ver,
6522 corr,
6523 None,
6524 launch_nonce.map(ToOwned::to_owned),
6525 )
6526 }
6527
6528 fn hello_frame_full(
6529 module_id: &str,
6530 protocol_ver: u8,
6531 corr: u64,
6532 control_ops: Option<Vec<String>>,
6533 launch_nonce: Option<String>,
6534 ) -> Frame {
6535 let body = serde_json::to_vec(&ModuleHelloBody {
6536 manifest: manifest(module_id, protocol_ver),
6537 protocol_ver,
6538 control_ops,
6539 launch_nonce,
6540 })
6541 .unwrap();
6542 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
6543 }
6544
6545 fn non_routable_hello_frame_with_control_ops(
6546 module_id: &str,
6547 corr: u64,
6548 control_ops: Option<Vec<String>>,
6549 ) -> Frame {
6550 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
6551 manifest.provides.clear();
6552 let body = serde_json::to_vec(&ModuleHelloBody {
6553 manifest,
6554 protocol_ver: PROTOCOL_VERSION,
6555 control_ops,
6556 launch_nonce: None,
6557 })
6558 .unwrap();
6559 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
6560 }
6561
6562 fn capability_grammar_hello_frame(
6563 capabilities: Value,
6564 runtime_computed: Option<Value>,
6565 corr: u64,
6566 ) -> Frame {
6567 let mut body = serde_json::to_value(ModuleHelloBody {
6568 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
6569 protocol_ver: PROTOCOL_VERSION,
6570 control_ops: None,
6571 launch_nonce: None,
6572 })
6573 .expect("HELLO body serializes");
6574 body["manifest"]["capabilities"] = capabilities;
6575 if let Some(runtime_computed) = runtime_computed {
6576 body["runtime_computed"] = runtime_computed;
6577 }
6578 Frame::build(
6579 FrameType::Hello,
6580 control_flags(),
6581 0,
6582 0,
6583 corr,
6584 serde_json::to_vec(&body).expect("HELLO body reserializes"),
6585 )
6586 .expect("HELLO frame builds")
6587 }
6588
6589 fn channel_request(channel: u16, corr: u64) -> Frame {
6590 Frame::build(
6591 FrameType::Request,
6592 Flags::new(true, Priority::Interactive, false),
6593 channel,
6594 0,
6595 corr,
6596 b"opaque".to_vec(),
6597 )
6598 .unwrap()
6599 }
6600
6601 fn route_ctx(
6602 connection_id: ConnectionId,
6603 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
6604 let (tx, rx) = mpsc::channel(8);
6605 (
6606 RouteCtx {
6607 connection_id,
6608 egress: FrameSink::new(tx),
6609 },
6610 rx,
6611 )
6612 }
6613
6614 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
6615 serde_json::from_slice(&frame.body).unwrap()
6616 }
6617
6618 async fn hello_via_sink(
6623 handler: &ControlHandler,
6624 ctx: &RouteCtx,
6625 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
6626 hello: Frame,
6627 ) -> Frame {
6628 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
6629 assert!(
6630 replies.is_empty(),
6631 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
6632 );
6633 let ack = rx
6634 .try_recv()
6635 .expect("HELLO_ACK is queued on the module sink")
6636 .frame;
6637 assert_eq!(ack.header.ty, FrameType::HelloAck);
6638 ack
6639 }
6640
6641 fn parse_error(frame: &Frame) -> Value {
6642 serde_json::from_slice(&frame.body).unwrap()
6643 }
6644
6645 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
6646 serde_json::from_slice(&frame.body).unwrap()
6647 }
6648
6649 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
6650 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
6651 route_channel,
6652 route_epoch: 0,
6653 kind,
6654 })
6655 .unwrap();
6656 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6657 }
6658
6659 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
6660 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
6661 module_id: module_id.to_string(),
6662 })
6663 .unwrap();
6664 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6665 }
6666
6667 fn route_open_frame(corr: u64, module_id: &str, project_root: TestTempDir) -> Frame {
6668 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
6669 }
6670
6671 fn route_open_frame_with_consumer_capabilities(
6672 corr: u64,
6673 module_id: &str,
6674 project_root: TestTempDir,
6675 consumer_capabilities: Option<Vec<String>>,
6676 ) -> Frame {
6677 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6678 target: RouteTarget::ToolProvider {
6679 module_id: module_id.to_string(),
6680 },
6681 identity: BindIdentity::new(
6682 project_root.path().to_path_buf(),
6683 "unit".to_string(),
6684 "session".to_string(),
6685 ),
6686 consumer_identity: None,
6687 consumer_capabilities,
6688 role_versions: None,
6689 admission_facts: None,
6690 scope: None,
6691 })
6692 .unwrap();
6693 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6694 }
6695
6696 fn route_open_frame_with_role_versions(
6697 corr: u64,
6698 module_id: &str,
6699 project_root: TestTempDir,
6700 role_versions: Option<BTreeMap<String, String>>,
6701 ) -> Frame {
6702 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6703 target: RouteTarget::ToolProvider {
6704 module_id: module_id.to_string(),
6705 },
6706 identity: BindIdentity::new(
6707 project_root.path().to_path_buf(),
6708 "unit".to_string(),
6709 format!("session-{corr}"),
6710 ),
6711 consumer_identity: None,
6712 consumer_capabilities: None,
6713 role_versions,
6714 admission_facts: None,
6715 scope: None,
6716 })
6717 .unwrap();
6718 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6719 }
6720
6721 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
6722 entries
6723 .iter()
6724 .map(|(role, version)| (role.to_string(), version.to_string()))
6725 .collect()
6726 }
6727
6728 fn route_open_frame_with_admission_facts(
6729 corr: u64,
6730 module_id: &str,
6731 project_root: TestTempDir,
6732 consumer_identity: Option<subc_control::ConsumerIdentity>,
6733 facts: Option<Value>,
6734 ) -> Frame {
6735 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6736 target: RouteTarget::ToolProvider {
6737 module_id: module_id.to_string(),
6738 },
6739 identity: BindIdentity::new(
6740 project_root.path().to_path_buf(),
6741 "unit".to_string(),
6742 format!("session-{corr}"),
6743 ),
6744 consumer_identity,
6745 consumer_capabilities: None,
6746 role_versions: None,
6747 admission_facts: facts,
6748 scope: None,
6749 })
6750 .unwrap();
6751 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6752 }
6753
6754 #[derive(Clone, Default)]
6755 struct EventCapture {
6756 events: Arc<Mutex<Vec<CapturedEvent>>>,
6757 }
6758
6759 #[derive(Clone, Debug)]
6760 struct CapturedEvent {
6761 target: String,
6762 level: tracing::Level,
6763 fields: BTreeMap<String, String>,
6764 }
6765
6766 impl EventCapture {
6767 fn events(&self) -> Vec<CapturedEvent> {
6768 self.events.lock().unwrap().clone()
6769 }
6770 }
6771
6772 impl<S> Layer<S> for EventCapture
6773 where
6774 S: Subscriber,
6775 {
6776 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
6777 let mut visitor = EventFieldVisitor::default();
6778 event.record(&mut visitor);
6779 self.events.lock().unwrap().push(CapturedEvent {
6780 target: event.metadata().target().to_string(),
6781 level: *event.metadata().level(),
6782 fields: visitor.fields,
6783 });
6784 }
6785 }
6786
6787 #[derive(Default)]
6788 struct EventFieldVisitor {
6789 fields: BTreeMap<String, String>,
6790 }
6791
6792 impl Visit for EventFieldVisitor {
6793 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
6794 self.fields
6795 .insert(field.name().to_string(), format!("{value:?}"));
6796 }
6797 }
6798
6799 fn health_response(corr: u64, status: HealthStatus) -> Frame {
6800 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
6801 status,
6802 detail: Some("warming".to_string()),
6803 metrics: Some(json!({"queue_depth": 3})),
6804 })
6805 .unwrap();
6806 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
6807 }
6808
6809 fn route_bind_ack(corr: u64) -> Frame {
6810 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
6811 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
6812 }
6813
6814 fn unique_project_root(label: &str) -> TestTempDir {
6815 TestTempDir::new(label)
6816 }
6817
6818 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
6819 match parse_route_poll(frame) {
6820 ClientControlResponse::RoutePoll {
6821 status: None,
6822 live: Some(live),
6823 ..
6824 } => assert_eq!(live, expected_live),
6825 other => panic!("unexpected route.poll response: {other:?}"),
6826 }
6827 }
6828
6829 fn bind_liveness_route(
6830 registry: &Registry,
6831 forwarding: &ForwardingTable,
6832 module_id: &str,
6833 ) -> (RouteCtx, u16, u32) {
6834 let module_connection = ConnectionId::new(101);
6835 let client_connection = ConnectionId::new(202);
6836 let registration = registry
6837 .register_with_control_ops(
6838 manifest(module_id, PROTOCOL_VERSION),
6839 PROTOCOL_VERSION,
6840 module_connection,
6841 module_baseline_control_ops(),
6842 )
6843 .unwrap();
6844 let (module_tx, _module_rx) = mpsc::channel(8);
6845 let endpoint = forwarding
6846 .register_module_connection(
6847 module_connection,
6848 module_id.to_string(),
6849 PROTOCOL_VERSION,
6850 manifest_concurrency(®istration.manifest),
6851 FrameSink::new(module_tx),
6852 )
6853 .unwrap();
6854 let (client_ctx, _client_rx) = route_ctx(client_connection);
6855 let pending = forwarding
6856 .begin_route_bind_relay_for_test(
6857 client_connection,
6858 client_ctx.egress.clone(),
6859 1,
6860 module_id,
6861 )
6862 .unwrap();
6863 assert_eq!(pending.endpoint, endpoint);
6864 let route_channel = pending.client_channel;
6865 let route_epoch = pending.client_epoch;
6866 forwarding
6867 .complete_pending_relay(
6868 module_connection,
6869 pending.corr,
6870 RouteBindRelayOutcome::Accepted,
6871 )
6872 .unwrap();
6873 (client_ctx, route_channel, route_epoch)
6874 }
6875
6876 struct FakeProcessLiveness {
6877 live: Option<bool>,
6878 }
6879
6880 impl ModuleProcessLiveness for FakeProcessLiveness {
6881 fn process_live(&self, _module_id: &str) -> Option<bool> {
6882 self.live
6883 }
6884 }
6885
6886 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6887 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
6888 {
6889 let registry = Arc::new(Registry::default());
6890 let supervisor_handle = SupervisorHandle::new();
6891 let supervisor = Supervisor::new(
6892 Arc::clone(®istry),
6893 RestartPolicy::new(1, Duration::from_millis(10)),
6894 )
6895 .with_handle(supervisor_handle.clone());
6896 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
6897 let module = supervisor
6898 .spawn(ModuleSpec {
6899 module_id: "stderr-tail-wire".to_string(),
6900 program: fake_aft_stub_path(),
6901 args: Vec::new(),
6902 env: vec![
6903 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
6904 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
6905 ],
6906 reserved: false,
6907 reserved_prefixes: Vec::new(),
6908 protocol: ModuleProtocol::Subc,
6909 overlap: Default::default(),
6910 })
6911 .unwrap();
6912
6913 let deadline = Instant::now() + Duration::from_secs(5);
6914 loop {
6915 let tail = module.stderr_tail(None, None);
6916 if tail
6917 .entries
6918 .iter()
6919 .any(|entry| matches!(entry, TailEntry::ProcessStart))
6920 && tail.entries.iter().any(|entry| {
6921 matches!(
6922 entry,
6923 TailEntry::Line {
6924 truncated: true,
6925 ..
6926 }
6927 )
6928 })
6929 {
6930 break;
6931 }
6932 assert!(
6933 Instant::now() < deadline,
6934 "module did not produce a truncated line and restart boundary: {tail:?}"
6935 );
6936 sleep(Duration::from_millis(10)).await;
6937 }
6938
6939 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
6940 let request = ClientControlRequest::SupervisorStderrTail {
6941 module_id: "stderr-tail-wire".to_string(),
6942 max_lines: None,
6943 max_bytes: None,
6944 };
6945 let frame = Frame::build(
6946 FrameType::Request,
6947 control_flags(),
6948 0,
6949 0,
6950 1,
6951 serde_json::to_vec(&request).unwrap(),
6952 )
6953 .unwrap();
6954 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6955 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
6956 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
6957 serde_json::from_slice(&responses[0].body).unwrap()
6958 else {
6959 panic!("expected supervisor.stderr_tail response");
6960 };
6961
6962 assert!(
6963 tail.entries
6964 .iter()
6965 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
6966 "the control response lost the restart boundary"
6967 );
6968 let Some(StderrTailEntry::Line {
6969 text,
6970 truncated,
6971 at_ms,
6972 }) = tail.entries.iter().find(|entry| {
6973 matches!(
6974 entry,
6975 StderrTailEntry::Line {
6976 truncated: true,
6977 ..
6978 }
6979 )
6980 })
6981 else {
6982 panic!("the control response lost the truncated line");
6983 };
6984 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
6985 assert!(*truncated);
6986 assert!(
6987 at_ms.is_some(),
6988 "the control response lost the line's capture time"
6989 );
6990 }
6991
6992 #[tokio::test(flavor = "current_thread")]
6997 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
6998 let dir = TestTempDir::new("terminals-off-worker");
6999 let journal_path = dir.join("terminals.jsonl");
7000 let registry = Arc::new(Registry::default());
7001 let supervisor_handle = SupervisorHandle::new();
7002 let supervisor =
7003 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
7004 .with_handle(supervisor_handle.clone())
7005 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
7006 let module = supervisor
7007 .spawn(ModuleSpec {
7008 module_id: "terminal-off-worker".to_string(),
7009 program: fake_aft_stub_path(),
7010 args: Vec::new(),
7011 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
7012 reserved: false,
7013 reserved_prefixes: Vec::new(),
7014 protocol: ModuleProtocol::Subc,
7015 overlap: Default::default(),
7016 })
7017 .unwrap();
7018 let deadline = Instant::now() + Duration::from_secs(5);
7019 while module.terminal_history().entries.len() != 2 {
7020 assert!(Instant::now() < deadline, "module did not record two exits");
7021 sleep(Duration::from_millis(10)).await;
7022 }
7023
7024 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
7025 let handler =
7026 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
7027 let frame = Frame::build(
7028 FrameType::Request,
7029 control_flags(),
7030 0,
7031 0,
7032 1,
7033 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
7034 module_id: "terminal-off-worker".to_string(),
7035 })
7036 .unwrap(),
7037 )
7038 .unwrap();
7039 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7040 let spawned_at = std::time::Instant::now();
7041 let read = tokio::spawn({
7042 let handler = Arc::clone(&handler);
7043 async move { handler.handle_control_frame(&ctx, frame).await }
7044 });
7045 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
7048 .await
7049 .unwrap()
7050 .expect("the history read reached its pause");
7051 let elapsed = spawned_at.elapsed();
7052 assert!(
7053 elapsed < Duration::from_secs(2) && !read.is_finished(),
7054 "this task could not run while the history read was paused \
7055 (resumed after {elapsed:?}, read finished: {})",
7056 read.is_finished()
7057 );
7058
7059 drop(release);
7060 let responses = read.await.unwrap().unwrap();
7061 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
7062 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
7063 panic!("expected supervisor.terminals response");
7064 };
7065 assert_eq!(terminals.entries.len(), 2);
7066 assert_eq!(terminals.journal_skipped_lines, 0);
7067 assert_eq!(terminals.journal_read_errors, 0);
7068 }
7069
7070 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7071 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
7072 let registry = Arc::new(Registry::default());
7073 let supervisor_handle = SupervisorHandle::new();
7074 let supervisor =
7075 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
7076 .with_handle(supervisor_handle.clone());
7077 let module = supervisor
7078 .spawn(ModuleSpec {
7079 module_id: "terminal-golden".to_string(),
7080 program: fake_aft_stub_path(),
7081 args: Vec::new(),
7082 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
7083 reserved: false,
7084 reserved_prefixes: Vec::new(),
7085 protocol: ModuleProtocol::Subc,
7086 overlap: Default::default(),
7087 })
7088 .unwrap();
7089
7090 let deadline = Instant::now() + Duration::from_secs(5);
7091 while module.terminal_history().entries.len() != 2 {
7092 assert!(
7093 Instant::now() < deadline,
7094 "module did not retain two terminal exits: {:?}",
7095 module.terminal_history()
7096 );
7097 sleep(Duration::from_millis(10)).await;
7098 }
7099
7100 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
7101 let request = ClientControlRequest::SupervisorTerminals {
7102 module_id: "terminal-golden".to_string(),
7103 };
7104 let frame = Frame::build(
7105 FrameType::Request,
7106 control_flags(),
7107 0,
7108 0,
7109 1,
7110 serde_json::to_vec(&request).unwrap(),
7111 )
7112 .unwrap();
7113 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7114 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
7115 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
7116 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
7117 panic!("expected supervisor.terminals response");
7118 };
7119 assert_eq!(terminals.entries.len(), 2);
7120 assert_eq!(terminals.dropped, 0);
7121
7122 let mut rendered = serde_json::to_value(response).unwrap();
7123 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
7126 for (index, entry) in rendered["entries"]
7127 .as_array_mut()
7128 .expect("terminal response entries array")
7129 .iter_mut()
7130 .enumerate()
7131 {
7132 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
7133 }
7134
7135 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
7136 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
7137 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
7138 if std::env::var_os("UPDATE_GOLDEN").is_some() {
7139 std::fs::write(&golden_path, &serialized).unwrap();
7140 }
7141 let expected: Value =
7142 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
7143 assert_eq!(rendered, expected);
7144 }
7145
7146 #[test]
7147 fn hello_registers_manifest_and_returns_ack() {
7148 let registry = Arc::new(Registry::default());
7149 let handler = ControlHandler::new(Arc::clone(®istry));
7150 let conn = ConnectionId::new(1);
7151
7152 let responses = handler
7153 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
7154 .unwrap();
7155
7156 assert_eq!(responses.len(), 1);
7157 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
7158 assert_eq!(responses[0].header.channel, 0);
7159 assert_eq!(responses[0].header.corr, 7);
7160 let ack = parse_ack(&responses[0]);
7161 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
7162 assert!(ack
7163 .subc_capabilities
7164 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
7165 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
7166 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
7167 assert!(ack
7168 .subc_ops
7169 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
7170 assert!(ack
7171 .subc_ops
7172 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
7173
7174 let registration = registry.get_module("aft").unwrap().unwrap();
7175 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
7176 assert_eq!(registration.state, ChannelState::Active);
7177 assert_eq!(registration.connection_id, conn);
7178 assert_eq!(registration.control_ops, module_baseline_control_ops());
7179 }
7180
7181 #[test]
7182 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
7183 let invalid_identifiers = [
7184 ("case_change", "credentials-Provider/v1"),
7185 ("leading_zero", "credentials-provider/v01"),
7186 ("trailing_hyphen", "credentials-provider-/v1"),
7187 ("consecutive_hyphens", "credentials--provider/v1"),
7188 ("uppercase", "Credentials-provider/v1"),
7189 ("missing_v", "credentials-provider/1"),
7190 ("whitespace", "credentials provider/v1"),
7191 ("zero_version", "credentials-provider/v0"),
7192 ("out_of_range_version", "credentials-provider/v4294967296"),
7193 (
7194 "overlength_name",
7195 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
7196 ),
7197 ];
7198 let mut cases = invalid_identifiers
7199 .into_iter()
7200 .map(|(name, identifier)| {
7201 (
7202 format!("identifier_{name}"),
7203 "capabilities.provides[0]".to_string(),
7204 identifier.to_string(),
7205 json!({ "provides": [identifier] }),
7206 None,
7207 )
7208 })
7209 .collect::<Vec<_>>();
7210 cases.extend([
7211 (
7212 "unknown_need".to_string(),
7213 "capabilities.requires[0].need".to_string(),
7214 "deferred".to_string(),
7215 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
7216 None,
7217 ),
7218 (
7219 "duplicate_provides".to_string(),
7220 "capabilities.provides[1]".to_string(),
7221 "credentials-provider/v1".to_string(),
7222 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
7223 None,
7224 ),
7225 (
7226 "duplicate_must_never_reach".to_string(),
7227 "capabilities.must_never_reach[1]".to_string(),
7228 "credentials-provider/v1".to_string(),
7229 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
7230 None,
7231 ),
7232 (
7233 "duplicate_requires_same_need".to_string(),
7234 "capabilities.requires[1]".to_string(),
7235 "credentials-provider/v1".to_string(),
7236 json!({ "requires": [
7237 { "capability": "credentials-provider/v1", "need": "required" },
7238 { "capability": "credentials-provider/v1", "need": "required" }
7239 ] }),
7240 None,
7241 ),
7242 (
7243 "duplicate_requires_conflicting_need".to_string(),
7244 "capabilities.requires[1]".to_string(),
7245 "credentials-provider/v1".to_string(),
7246 json!({ "requires": [
7247 { "capability": "credentials-provider/v1", "need": "required" },
7248 { "capability": "credentials-provider/v1", "need": "optional" }
7249 ] }),
7250 None,
7251 ),
7252 (
7253 "capabilities_root_pointer".to_string(),
7254 "runtime_computed[0]".to_string(),
7255 "/capabilities".to_string(),
7256 json!({}),
7257 Some(json!(["/capabilities"])),
7258 ),
7259 (
7260 "capabilities_descendant_pointer".to_string(),
7261 "runtime_computed[0]".to_string(),
7262 "/capabilities/provides".to_string(),
7263 json!({}),
7264 Some(json!(["/capabilities/provides"])),
7265 ),
7266 (
7267 "malformed_pointer_without_leading_slash".to_string(),
7268 "runtime_computed[0]".to_string(),
7269 "capabilities".to_string(),
7270 json!({}),
7271 Some(json!(["capabilities"])),
7272 ),
7273 (
7274 "malformed_pointer_escape".to_string(),
7275 "runtime_computed[0]".to_string(),
7276 "/roles/~2/tools".to_string(),
7277 json!({}),
7278 Some(json!(["/roles/~2/tools"])),
7279 ),
7280 (
7281 "unknown_capabilities_field".to_string(),
7282 "capabilities.future".to_string(),
7283 "<array>".to_string(),
7284 json!({ "future": [] }),
7285 None,
7286 ),
7287 ]);
7288
7289 for (index, (name, field, value, capabilities, runtime_computed)) in
7290 cases.into_iter().enumerate()
7291 {
7292 let registry = Arc::new(Registry::default());
7293 let handler = ControlHandler::new(Arc::clone(®istry));
7294 let response = handler
7295 .handle_control(
7296 ConnectionId::new((index + 1) as u64),
7297 capability_grammar_hello_frame(
7298 capabilities,
7299 runtime_computed,
7300 index as u64 + 1,
7301 ),
7302 )
7303 .expect("invalid HELLO returns a refusal");
7304
7305 assert_eq!(response.len(), 1, "{name} must emit one refusal");
7306 let error = parse_error(&response[0]);
7307 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
7308 let message = error["message"]
7309 .as_str()
7310 .expect("error message is a string");
7311 assert!(
7312 message.contains(&field),
7313 "{name}: field missing from {message}"
7314 );
7315 assert!(
7316 message.contains(&value),
7317 "{name}: value missing from {message}"
7318 );
7319 assert_eq!(
7320 registry
7321 .active_registration_count()
7322 .expect("registry reads"),
7323 0,
7324 "{name}: refused HELLO must not create a catalog entry"
7325 );
7326 }
7327 }
7328
7329 #[test]
7330 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
7331 let registry = Arc::new(Registry::default());
7332 let handler = ControlHandler::new(Arc::clone(®istry));
7333 let capabilities = json!({
7334 "provides": ["credentials-provider/v1"],
7335 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
7336 "must_never_reach": ["federation-transport/v1"]
7337 });
7338 let response = handler
7339 .handle_control(
7340 ConnectionId::new(99),
7341 capability_grammar_hello_frame(
7342 capabilities.clone(),
7343 Some(json!(["/roles/0/tools"])),
7344 99,
7345 ),
7346 )
7347 .expect("valid HELLO registers");
7348 assert_eq!(response[0].header.ty, FrameType::HelloAck);
7349
7350 let request = Frame::build(
7351 FrameType::Request,
7352 control_flags(),
7353 0,
7354 0,
7355 100,
7356 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7357 .expect("catalog request serializes"),
7358 )
7359 .expect("catalog request frame builds");
7360 let response = handler
7361 .handle_catalog_list(request, None)
7362 .expect("catalog list succeeds");
7363 let ClientControlResponse::CatalogList { modules, .. } =
7364 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
7365 else {
7366 panic!("catalog request must return catalog.list");
7367 };
7368 assert_eq!(modules.len(), 1);
7369 assert_eq!(
7370 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
7371 capabilities
7372 );
7373 }
7374
7375 #[test]
7376 fn catalog_list_mirrors_management_operation_description() {
7377 let registry = Arc::new(Registry::default());
7378 let handler = ControlHandler::new(Arc::clone(®istry));
7379 let description = "List managed records and return their identifiers and metadata.";
7380 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
7381 manifest.provides = vec![ProviderRole::ManagementSurface {
7382 operations: vec![ManagementOperation {
7383 name: "records.list".to_string(),
7384 kind: ManagementOperationKind::Query,
7385 description: Some(description.to_string()),
7386 }],
7387 config_schema: json!({"type": "object"}),
7388 observability: vec![ObservabilitySurface {
7389 name: "records.stats".to_string(),
7390 kind: ObservabilityKind::Snapshot,
7391 }],
7392 identity_scope: vec![IdentityScope::Project],
7393 concurrency: Concurrency::ModuleManaged,
7394 }];
7395 registry
7396 .register_with_control_ops(
7397 manifest,
7398 PROTOCOL_VERSION,
7399 ConnectionId::new(99),
7400 Vec::new(),
7401 )
7402 .expect("described management manifest registers");
7403
7404 let request = Frame::build(
7405 FrameType::Request,
7406 control_flags(),
7407 0,
7408 0,
7409 100,
7410 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7411 .expect("catalog request serializes"),
7412 )
7413 .expect("catalog request frame builds");
7414 let response = handler
7415 .handle_catalog_list(request, None)
7416 .expect("catalog list succeeds");
7417 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
7418 assert_eq!(
7419 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
7420 "catalog.list must preserve the declared operation description verbatim"
7421 );
7422 }
7423
7424 #[test]
7425 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
7426 let registry = Arc::new(Registry::default());
7427 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
7428 [("vault".to_string(), true), ("squatter".to_string(), true)],
7429 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
7430 );
7431 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
7432 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
7433 provides: vec!["credentials-provider/v1".to_string()],
7434 requires: Vec::new(),
7435 must_never_reach: Vec::new(),
7436 });
7437 let frame = Frame::build(
7438 FrameType::Hello,
7439 control_flags(),
7440 0,
7441 0,
7442 77,
7443 serde_json::to_vec(&ModuleHelloBody {
7444 manifest: squatter,
7445 protocol_ver: PROTOCOL_VERSION,
7446 control_ops: None,
7447 launch_nonce: None,
7448 })
7449 .expect("HELLO serializes"),
7450 )
7451 .expect("HELLO frame builds");
7452 let response = handler
7453 .handle_control(ConnectionId::new(77), frame)
7454 .expect("reserved claim receives a typed refusal");
7455 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
7456 assert_eq!(
7457 registry
7458 .active_registration_count()
7459 .expect("registry reads"),
7460 0,
7461 "a reserved capability refusal must not leave a catalog entry"
7462 );
7463 }
7464
7465 #[test]
7466 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
7467 for settlement in ["timeout", "inconclusive", "drop"] {
7468 let breakers = RouteBindBreakers::default();
7469 let RouteBindAdmission::Admitted {
7470 guard: mut old,
7471 probe: false,
7472 } = breakers.admit("prov")
7473 else {
7474 panic!("ordinary relay admitted")
7475 };
7476 let RouteBindAdmission::Admitted {
7477 guard: mut opener, ..
7478 } = breakers.admit("prov")
7479 else {
7480 panic!("second relay admitted")
7481 };
7482 assert!(
7483 !opener
7484 .record_timeout(1, Duration::ZERO)
7485 .unwrap()
7486 .reopened_after_probe
7487 );
7488 let RouteBindAdmission::Admitted {
7489 guard: mut probe,
7490 probe: true,
7491 } = breakers.admit("prov")
7492 else {
7493 panic!("one cooldown probe admitted")
7494 };
7495 match settlement {
7496 "timeout" => assert!(
7497 !old.record_timeout(1, Duration::ZERO)
7498 .unwrap()
7499 .reopened_after_probe
7500 ),
7501 "inconclusive" => old.record_inconclusive(),
7502 "drop" => drop(old),
7503 _ => unreachable!(),
7504 }
7505 assert!(
7506 matches!(
7507 breakers.admit("prov"),
7508 RouteBindAdmission::Refused {
7509 probe_in_flight: true,
7510 ..
7511 }
7512 ),
7513 "{settlement} of a pre-open relay cannot release the real probe"
7514 );
7515 assert!(
7516 probe
7517 .record_timeout(1, Duration::ZERO)
7518 .unwrap()
7519 .reopened_after_probe
7520 );
7521 assert!(matches!(
7522 breakers.admit("prov"),
7523 RouteBindAdmission::Admitted { probe: true, .. }
7524 ));
7525 }
7526 let breakers = RouteBindBreakers::default();
7527 let admit = || match breakers.admit("prov") {
7528 RouteBindAdmission::Admitted { guard, .. } => guard,
7529 _ => panic!("relay admitted"),
7530 };
7531 admit().record_timeout(1, Duration::ZERO);
7532 let mut old_probe = admit();
7533 breakers.reset_for_new_module_connection("prov");
7534 admit().record_timeout(1, Duration::ZERO);
7535 let _new_probe = admit();
7536 old_probe.record_inconclusive();
7537 assert!(matches!(
7538 breakers.admit("prov"),
7539 RouteBindAdmission::Refused {
7540 probe_in_flight: true,
7541 ..
7542 }
7543 ));
7544 }
7545
7546 #[tokio::test]
7547 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
7548 for candidate in [false, true] {
7549 let registry = Arc::new(Registry::default());
7550 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
7551 [("vault".to_string(), true), ("squatter".to_string(), true)],
7552 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
7553 );
7554 let conn = ConnectionId::new(77);
7555 let (ctx, mut rx) = route_ctx(conn);
7556 let initial = capability_manifest("squatter", &[], &[]);
7557 if candidate {
7558 registry
7559 .register_candidate_with_control_ops(
7560 initial.clone(),
7561 PROTOCOL_VERSION,
7562 conn,
7563 module_baseline_control_ops(),
7564 )
7565 .unwrap();
7566 handler
7567 .forwarding
7568 .register_candidate_module_connection(
7569 conn,
7570 "squatter".to_string(),
7571 PROTOCOL_VERSION,
7572 manifest_concurrency(&initial),
7573 ctx.egress.clone(),
7574 )
7575 .unwrap();
7576 } else {
7577 hello_via_sink(
7578 &handler,
7579 &ctx,
7580 &mut rx,
7581 hello_frame_with_manifest(initial.clone(), 1),
7582 )
7583 .await;
7584 }
7585 let response = handler
7586 .handle_control_frame(
7587 &ctx,
7588 catalog_update_with_capabilities_frame(
7589 2,
7590 capability_manifest("squatter", &["credentials-provider/v1"], &[])
7591 .capabilities
7592 .unwrap(),
7593 ),
7594 )
7595 .await
7596 .unwrap();
7597 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
7598 assert_eq!(
7599 registry
7600 .get_module_by_connection(conn)
7601 .unwrap()
7602 .unwrap()
7603 .manifest,
7604 initial
7605 );
7606 }
7607 }
7608
7609 #[test]
7610 fn server_describe_surfaces_required_capability_verdict_fields() {
7611 let registry = Arc::new(Registry::default());
7612 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
7613 [
7614 ("consumer".to_string(), true),
7615 ("provider".to_string(), false),
7616 ],
7617 BTreeMap::new(),
7618 );
7619 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
7620 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
7621 provides: Vec::new(),
7622 requires: vec![subc_protocol::manifest::CapabilityRequirement {
7623 capability: "credentials-provider/v1".to_string(),
7624 need: subc_protocol::manifest::CapabilityNeed::Required,
7625 }],
7626 must_never_reach: Vec::new(),
7627 });
7628 let hello = Frame::build(
7629 FrameType::Hello,
7630 control_flags(),
7631 0,
7632 0,
7633 78,
7634 serde_json::to_vec(&ModuleHelloBody {
7635 manifest: consumer,
7636 protocol_ver: PROTOCOL_VERSION,
7637 control_ops: None,
7638 launch_nonce: None,
7639 })
7640 .expect("HELLO serializes"),
7641 )
7642 .expect("HELLO frame builds");
7643 handler
7644 .handle_control(ConnectionId::new(78), hello)
7645 .expect("consumer registers");
7646 let describe = Frame::build(
7647 FrameType::Request,
7648 control_flags(),
7649 0,
7650 0,
7651 79,
7652 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
7653 .expect("request serializes"),
7654 )
7655 .expect("describe frame builds");
7656 let response = handler
7657 .handle_server_describe(describe)
7658 .expect("server.describe succeeds");
7659 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
7660 let requirement = &rendered["capability_requirements"][0];
7661 assert_eq!(requirement["consumer"], "consumer");
7662 assert_eq!(requirement["verdict"], "never_provided");
7663 assert_eq!(requirement["episode_seq"], 1);
7664 assert_eq!(requirement["config_satisfiable"], false);
7665 assert_eq!(requirement["runtime_available"], false);
7666 assert!(requirement["detail"]
7667 .as_str()
7668 .expect("detail string")
7669 .contains("credentials-provider/v1"));
7670 }
7671
7672 #[test]
7673 fn catalog_list_omits_capabilities_for_legacy_manifest() {
7674 let registry = Arc::new(Registry::default());
7675 let handler = ControlHandler::new(Arc::clone(®istry));
7676 let hello = handler
7677 .handle_control(
7678 ConnectionId::new(101),
7679 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
7680 )
7681 .expect("legacy HELLO registers");
7682 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
7683
7684 let request = Frame::build(
7685 FrameType::Request,
7686 control_flags(),
7687 0,
7688 0,
7689 102,
7690 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7691 .expect("catalog request serializes"),
7692 )
7693 .expect("catalog request frame builds");
7694 let response = handler
7695 .handle_catalog_list(request, None)
7696 .expect("catalog list succeeds");
7697 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
7698 assert!(
7699 body["modules"][0].get("capabilities").is_none(),
7700 "legacy manifest must retain an absent capabilities field on catalog.list"
7701 );
7702 }
7703
7704 #[test]
7705 fn hello_ack_omits_storage_when_no_storage_config() {
7706 let registry = Arc::new(Registry::default());
7707 let handler = ControlHandler::new(Arc::clone(®istry));
7708 let responses = handler
7709 .handle_control(
7710 ConnectionId::new(1),
7711 hello_frame("aft", PROTOCOL_VERSION, 7),
7712 )
7713 .unwrap();
7714 let ack = parse_ack(&responses[0]);
7715 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
7716 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
7717 }
7718
7719 #[tokio::test]
7720 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
7721 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
7722 let registry = Arc::new(Registry::default());
7723 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
7724 let responses = handler
7725 .handle_control(
7726 ConnectionId::new(1),
7727 hello_frame("aft", PROTOCOL_VERSION, 7),
7728 )
7729 .unwrap();
7730 let ack = parse_ack(&responses[0]);
7731 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
7732
7733 let described = handler
7734 .handle_control_frame(
7735 &route_ctx(ConnectionId::new(2)).0,
7736 Frame::build(
7737 FrameType::Request,
7738 control_flags(),
7739 0,
7740 0,
7741 9,
7742 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
7743 )
7744 .unwrap(),
7745 )
7746 .await
7747 .unwrap();
7748 let ClientControlResponse::ServerDescribe { machine_id, .. } =
7749 serde_json::from_slice(&described[0].body).unwrap()
7750 else {
7751 panic!("server.describe answered with another shape");
7752 };
7753 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
7754 }
7755
7756 #[test]
7757 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
7758 let registry = Arc::new(Registry::default());
7761 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
7762 crate::daemon_config::StorageConfig::Sqlite {
7763 data_home: std::path::PathBuf::from("/data"),
7764 },
7765 ));
7766
7767 let responses = handler
7768 .handle_control(
7769 ConnectionId::new(1),
7770 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
7771 )
7772 .unwrap();
7773 let ack = parse_ack(&responses[0]);
7774 assert_eq!(
7775 ack.storage,
7776 Some(serde_json::json!({
7777 "module_id": "alfonso-routing",
7778 "storage_namespace": "default",
7779 "isolation": { "kind": "module" },
7780 "backend": {
7781 "backend": "sqlite",
7782 "path": "/data/cortexkit/alfonso-routing/store.db"
7783 }
7784 })),
7785 "the delivered descriptor is the module's own sqlite store path"
7786 );
7787 }
7788
7789 #[test]
7790 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
7791 let registry = Arc::new(Registry::default());
7792 let handler = ControlHandler::new(Arc::clone(®istry));
7793 let conn = ConnectionId::new(1);
7794 let responses = handler
7795 .handle_control(
7796 conn,
7797 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
7798 )
7799 .unwrap();
7800 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
7801 let registration = registry.get_module("aft").unwrap().unwrap();
7802 assert_eq!(registration.control_ops, module_baseline_control_ops());
7803
7804 let frame =
7805 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
7806 assert!(handler
7807 .guard_module_control_op(&frame, "aft", "route.bind")
7808 .unwrap()
7809 .is_none());
7810 let error = handler
7811 .guard_module_control_op(&frame, "aft", "test.synthetic")
7812 .unwrap()
7813 .expect("synthetic ungranted op should be rejected");
7814 assert_eq!(error.header.ty, FrameType::Error);
7815 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
7816 }
7817
7818 #[test]
7819 fn hello_control_ops_some_adds_optional_grants() {
7820 let registry = Arc::new(Registry::default());
7821 let handler = ControlHandler::new(Arc::clone(®istry));
7822 handler
7823 .handle_control(
7824 ConnectionId::new(1),
7825 hello_frame_with_control_ops(
7826 "aft",
7827 PROTOCOL_VERSION,
7828 7,
7829 Some(vec![
7830 "future.synthetic".to_string(),
7831 "route.bind".to_string(),
7832 ]),
7833 ),
7834 )
7835 .unwrap();
7836 let registration = registry.get_module("aft").unwrap().unwrap();
7837 assert_eq!(
7838 registration.control_ops,
7839 vec![
7840 "route.bind".to_string(),
7841 "route.status".to_string(),
7842 "future.synthetic".to_string(),
7843 ]
7844 );
7845 let frame =
7846 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
7847 assert!(handler
7848 .guard_module_control_op(&frame, "aft", "future.synthetic")
7849 .unwrap()
7850 .is_none());
7851 }
7852
7853 #[tokio::test]
7854 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
7855 let registry = Arc::new(Registry::default());
7856 let forwarding = Arc::new(ForwardingTable::default());
7857 let handler =
7858 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
7859 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
7860 hello_via_sink(
7861 &handler,
7862 &module_ctx,
7863 &mut module_rx,
7864 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
7865 )
7866 .await;
7867
7868 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
7869 let responses = handler
7870 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
7871 .await
7872 .unwrap();
7873 assert_eq!(responses.len(), 1);
7874 assert_eq!(responses[0].header.ty, FrameType::Error);
7875 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
7876 assert!(module_rx.try_recv().is_err());
7877 }
7878
7879 #[tokio::test]
7880 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
7881 let registry = Arc::new(Registry::default());
7882 let forwarding = Arc::new(ForwardingTable::default());
7883 let handler =
7884 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
7885 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
7886 hello_via_sink(
7887 &handler,
7888 &module_ctx,
7889 &mut module_rx,
7890 hello_frame_with_control_ops(
7891 "aft",
7892 PROTOCOL_VERSION,
7893 7,
7894 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
7895 ),
7896 )
7897 .await;
7898
7899 let project_root = unique_project_root("demux");
7900 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
7901 let route_handler = handler.clone();
7902 let route_task = tokio::spawn(async move {
7903 route_handler
7904 .handle_control_frame(
7905 &route_client_ctx,
7906 route_open_frame(100, "aft", project_root),
7907 )
7908 .await
7909 .unwrap()
7910 });
7911 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7912 .await
7913 .unwrap()
7914 .unwrap();
7915 assert!(matches!(
7916 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
7917 ModuleControlRequest::RouteBind { .. }
7918 ));
7919
7920 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
7921 let health_handler = handler.clone();
7922 let health_task = tokio::spawn(async move {
7923 health_handler
7924 .handle_control_frame(
7925 &health_client_ctx,
7926 supervisor_health_probe_frame(101, "aft"),
7927 )
7928 .await
7929 .unwrap()
7930 });
7931 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7932 .await
7933 .unwrap()
7934 .unwrap();
7935 assert_eq!(
7936 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
7937 ModuleControlRequest::HealthCheck {}
7938 );
7939
7940 handler
7941 .handle_control_frame(
7942 &module_ctx,
7943 health_response(health_frame.header.corr, HealthStatus::Degraded),
7944 )
7945 .await
7946 .unwrap();
7947 let health_response = health_task.await.unwrap();
7948 assert_eq!(health_response.len(), 1);
7949 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
7950 ClientControlResponse::SupervisorHealthProbe {
7951 module_id,
7952 status,
7953 detail,
7954 metrics,
7955 } => {
7956 assert_eq!(module_id, "aft");
7957 assert_eq!(status, HealthStatus::Degraded);
7958 assert_eq!(detail.as_deref(), Some("warming"));
7959 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
7960 }
7961 other => panic!("unexpected health response: {other:?}"),
7962 }
7963
7964 handler
7965 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
7966 .await
7967 .unwrap();
7968 let route_response = route_task.await.unwrap();
7969 assert!(route_response.is_empty());
7970 let published = route_client_rx.recv().await.unwrap();
7971 assert!(matches!(
7972 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
7973 ClientControlResponse::RouteOpen { .. }
7974 ));
7975 }
7976
7977 async fn relay_route_open(
7982 handler: &ControlHandler,
7983 client_connection: ConnectionId,
7984 client_egress: &FrameSink,
7985 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
7986 corr: u64,
7987 module_id: &str,
7988 project_root_label: &str,
7989 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
7990 let ctx = RouteCtx {
7991 connection_id: client_connection,
7992 egress: client_egress.clone(),
7993 };
7994 let handler = handler.clone();
7995 let project_root = unique_project_root(project_root_label);
7996 let module_id = module_id.to_string();
7997 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
7998 let task = tokio::spawn(async move {
7999 let _guard = tracing::dispatcher::set_default(&dispatch);
8000 handler
8001 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
8002 .await
8003 .unwrap()
8004 });
8005 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
8006 .await
8007 .expect("module receives the relayed route.bind")
8008 .expect("module egress is open");
8009 (task, bind.frame)
8010 }
8011
8012 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
8013 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
8014 ModuleControlRequest::RouteBind {
8015 route_channel,
8016 epoch,
8017 ..
8018 } => (route_channel, epoch),
8019 other => panic!("expected a route.bind request, got {other:?}"),
8020 }
8021 }
8022
8023 fn published_route(frame: &Frame) -> (u16, u32) {
8024 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
8025 ClientControlResponse::RouteOpen {
8026 route_channel,
8027 route_epoch,
8028 } => (route_channel, route_epoch),
8029 other => panic!("expected a route.open response, got {other:?}"),
8030 }
8031 }
8032
8033 #[tokio::test]
8053 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
8054 let registry = Arc::new(Registry::default());
8055 let forwarding = Arc::new(ForwardingTable::default());
8056 let handler =
8057 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8058
8059 let module_connection = ConnectionId::new(30);
8060 let (module_ctx, mut module_rx) = route_ctx(module_connection);
8061 hello_via_sink(
8062 &handler,
8063 &module_ctx,
8064 &mut module_rx,
8065 hello_frame("aft", PROTOCOL_VERSION, 7),
8066 )
8067 .await;
8068
8069 let dying_client = ConnectionId::new(31);
8070 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
8071
8072 let (first_task, first_bind) = relay_route_open(
8075 &handler,
8076 dying_client,
8077 &dying_ctx.egress,
8078 &mut module_rx,
8079 100,
8080 "aft",
8081 "closing-first",
8082 )
8083 .await;
8084 handler
8085 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
8086 .await
8087 .unwrap();
8088 assert!(first_task.await.unwrap().is_empty());
8089 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
8090
8091 let (second_task, second_bind) = relay_route_open(
8093 &handler,
8094 dying_client,
8095 &dying_ctx.egress,
8096 &mut module_rx,
8097 101,
8098 "aft",
8099 "closing-second",
8100 )
8101 .await;
8102 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
8103
8104 assert!(forwarding
8107 .escalate_client_delivery_failure(
8108 dying_client,
8109 first_channel,
8110 first_epoch,
8111 CloseReason::new(
8112 "module_to_client_delivery_failed",
8113 "client egress refused a module frame",
8114 ),
8115 crate::forwarding::UndeliveredFrame {
8116 module_id: None,
8117 sink: &dying_ctx.egress,
8118 },
8119 )
8120 .unwrap());
8121 assert!(!dying_ctx.egress.is_closed());
8122
8123 let ack = handler
8125 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
8126 .await;
8127 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
8128 if module_loop_error.is_some() {
8129 handler.cleanup_connection(module_connection).unwrap();
8133 }
8134 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8138 .await
8139 .ok()
8140 .flatten();
8141
8142 assert!(
8144 registry
8145 .get_module_by_connection(module_connection)
8146 .unwrap()
8147 .is_some(),
8148 "one client's closing connection ended the shared module connection: \
8149 {module_loop_error:?}"
8150 );
8151 let cotenant = ConnectionId::new(32);
8153 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
8154 let (cotenant_task, cotenant_bind) = relay_route_open(
8155 &handler,
8156 cotenant,
8157 &cotenant_ctx.egress,
8158 &mut module_rx,
8159 102,
8160 "aft",
8161 "closing-cotenant",
8162 )
8163 .await;
8164 handler
8165 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
8166 .await
8167 .unwrap();
8168 assert!(cotenant_task.await.unwrap().is_empty());
8169 let (cotenant_channel, cotenant_epoch) =
8170 published_route(&cotenant_rx.recv().await.unwrap());
8171 assert!(matches!(
8172 forwarding
8173 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
8174 .unwrap(),
8175 DataRoute::Client(DataRouteState::Bound(_))
8176 ));
8177
8178 let goodbye = post_ack_module_frame
8181 .expect("module receives a GOODBYE for the abandoned route channel");
8182 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
8183 assert_eq!(goodbye.header.channel, abandoned_channel);
8184 assert_eq!(goodbye.header.epoch, abandoned_epoch);
8185
8186 assert!(dying_rx.try_recv().is_err());
8190 let second_response = second_task.await.unwrap();
8191 assert_eq!(second_response.len(), 1);
8192 assert_eq!(
8193 parse_error(&second_response[0])["code"],
8194 "target_unavailable"
8195 );
8196 }
8197
8198 #[test]
8205 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
8206 let handler = ControlHandler::default();
8207 let module_connection = ConnectionId::new(30);
8208 let client_connection = ConnectionId::new(31);
8209
8210 handler
8211 .refuse_to_end_module_connection_for_a_client(
8212 module_connection,
8213 77,
8214 ForwardingError::ConnectionClosing {
8215 connection_id: client_connection,
8216 },
8217 )
8218 .expect("a closing client must never end the module connection");
8219
8220 assert!(matches!(
8221 handler.refuse_to_end_module_connection_for_a_client(
8222 module_connection,
8223 78,
8224 ForwardingError::ConnectionClosing {
8225 connection_id: module_connection,
8226 },
8227 ),
8228 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
8229 connection_id
8230 })) if connection_id == module_connection
8231 ));
8232 assert!(matches!(
8233 handler.refuse_to_end_module_connection_for_a_client(
8234 module_connection,
8235 79,
8236 ForwardingError::Poisoned,
8237 ),
8238 Err(RouterError::Forwarding(ForwardingError::Poisoned))
8239 ));
8240 assert!(matches!(
8241 handler.refuse_to_end_module_connection_for_a_client(
8242 module_connection,
8243 80,
8244 ForwardingError::StaleModuleEndpoint,
8245 ),
8246 Err(RouterError::Forwarding(
8247 ForwardingError::StaleModuleEndpoint
8248 ))
8249 ));
8250 }
8251
8252 #[tokio::test]
8265 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
8266 let registry = Arc::new(Registry::default());
8267 let forwarding = Arc::new(ForwardingTable::default());
8268 let supervisor = SupervisorHandle::new();
8269 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8270 let handler =
8271 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8272 .with_supervisor(supervisor);
8273
8274 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
8275 hello_via_sink(
8276 &handler,
8277 &target_ctx,
8278 &mut target_rx,
8279 hello_frame("target", PROTOCOL_VERSION, 1),
8280 )
8281 .await;
8282
8283 let wrong_nonce = handler
8287 .handle_control_frame(
8288 &route_ctx(ConnectionId::new(91)).0,
8289 route_open_frame_with_admission_facts(
8290 20,
8291 "target",
8292 unique_project_root("admission-facts"),
8293 Some(subc_control::ConsumerIdentity {
8294 module_id: "fed".to_string(),
8295 launch_nonce: "not-the-real-nonce".to_string(),
8296 }),
8297 None,
8298 ),
8299 )
8300 .await
8301 .unwrap();
8302 assert_eq!(
8303 parse_error(&wrong_nonce[0])["code"],
8304 "bad_consumer_identity",
8305 "a mismatched launch nonce must be refused, not stamped Reserved"
8306 );
8307
8308 let never_spawned = handler
8312 .handle_control_frame(
8313 &route_ctx(ConnectionId::new(92)).0,
8314 route_open_frame_with_admission_facts(
8315 21,
8316 "target",
8317 unique_project_root("admission-facts"),
8318 Some(subc_control::ConsumerIdentity {
8319 module_id: "never-spawned".to_string(),
8320 launch_nonce: "any-nonce".to_string(),
8321 }),
8322 None,
8323 ),
8324 )
8325 .await
8326 .unwrap();
8327 assert_eq!(
8328 parse_error(&never_spawned[0])["code"],
8329 "bad_consumer_identity",
8330 "an unspawned module_id must be refused rather than accepted for lack of a record"
8331 );
8332 }
8333
8334 #[tokio::test]
8357 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
8358 let registry = Arc::new(Registry::default());
8359 let forwarding = Arc::new(ForwardingTable::default());
8360 let supervisor = SupervisorHandle::new();
8361 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8362 let handler =
8363 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8364 .with_supervisor(supervisor);
8365
8366 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
8367 hello_via_sink(
8368 &handler,
8369 &target_ctx,
8370 &mut target_rx,
8371 hello_frame("target", PROTOCOL_VERSION, 1),
8372 )
8373 .await;
8374
8375 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
8376 let route_handler = handler.clone();
8377 let route_task = tokio::spawn(async move {
8378 route_handler
8379 .handle_control_frame(
8380 &client_ctx,
8381 route_open_frame_with_admission_facts(
8382 30,
8383 "target",
8384 unique_project_root("admission-facts"),
8385 Some(subc_control::ConsumerIdentity {
8386 module_id: "fed".to_string(),
8387 launch_nonce: "fed-nonce".to_string(),
8388 }),
8389 None,
8390 ),
8391 )
8392 .await
8393 .unwrap()
8394 });
8395
8396 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8403 .await
8404 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
8405 .expect("module control channel closed before route.bind");
8406 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8407 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
8408 panic!("expected route.bind")
8409 };
8410 assert_eq!(
8411 principal,
8412 Some(Principal::Reserved {
8413 module_id: "fed".to_string()
8414 }),
8415 "a correctly attested consumer must be stamped Reserved for its own id"
8416 );
8417
8418 handler
8419 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8420 .await
8421 .unwrap();
8422 assert!(route_task.await.unwrap().is_empty());
8423 assert!(
8424 matches!(
8425 serde_json::from_slice::<ClientControlResponse>(
8426 &client_rx.recv().await.unwrap().body
8427 )
8428 .unwrap(),
8429 ClientControlResponse::RouteOpen { .. }
8430 ),
8431 "the route must actually open, not merely avoid an error"
8432 );
8433 }
8434
8435 #[tokio::test(start_paused = true)]
8436 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
8437 let registry = Arc::new(Registry::default());
8438 let forwarding = Arc::new(ForwardingTable::default());
8439 let supervisor = SupervisorHandle::new();
8440 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8441 let handler =
8442 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8443 .with_supervisor(supervisor);
8444
8445 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
8446 hello_via_sink(
8447 &handler,
8448 &target_ctx,
8449 &mut target_rx,
8450 hello_frame("target", PROTOCOL_VERSION, 1),
8451 )
8452 .await;
8453
8454 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
8455 let direct_handler = handler.clone();
8456 let direct_open = tokio::spawn(async move {
8457 direct_handler
8458 .handle_control_frame(
8459 &direct_ctx,
8460 route_open_frame(2, "target", unique_project_root("route-census-direct")),
8461 )
8462 .await
8463 .unwrap()
8464 });
8465 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8466 .await
8467 .expect("no direct route.bind within 5s")
8468 .expect("target control channel closed before direct route.bind");
8469 handler
8470 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
8471 .await
8472 .unwrap();
8473 assert!(direct_open.await.unwrap().is_empty());
8474 let _ = direct_rx.recv().await.unwrap();
8475
8476 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
8477 let reserved_handler = handler.clone();
8478 let reserved_open = tokio::spawn(async move {
8479 reserved_handler
8480 .handle_control_frame(
8481 &reserved_ctx,
8482 route_open_frame_with_admission_facts(
8483 3,
8484 "target",
8485 unique_project_root("admission-facts"),
8486 Some(ConsumerIdentity {
8487 module_id: "fed".to_string(),
8488 launch_nonce: "fed-nonce".to_string(),
8489 }),
8490 None,
8491 ),
8492 )
8493 .await
8494 .unwrap()
8495 });
8496 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8497 .await
8498 .expect("no reserved route.bind within 5s")
8499 .expect("target control channel closed before reserved route.bind");
8500 handler
8501 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
8502 .await
8503 .unwrap();
8504 assert!(reserved_open.await.unwrap().is_empty());
8505 let _ = reserved_rx.recv().await.unwrap();
8506
8507 forwarding
8508 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
8509 .unwrap();
8510 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
8511 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
8512 module_id: Some("target".to_string()),
8513 })
8514 .unwrap();
8515 let census_frame =
8516 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
8517 let response = handler
8518 .handle_control_frame(&census_ctx, census_frame)
8519 .await
8520 .unwrap()
8521 .pop()
8522 .unwrap();
8523 let actual: Value = serde_json::from_slice(&response.body).unwrap();
8524 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
8525 assert!(matches!(
8526 decoded,
8527 ClientControlResponse::SupervisorRoutes { .. }
8528 ));
8529 let routes = actual["modules"][0]["routes"].as_array().unwrap();
8530 assert_eq!(routes.len(), 2);
8531 assert!(routes.iter().all(|route| route["draining"] == true));
8532 assert!(
8535 routes.iter().all(|route| route["drain_reason"] == "reload"),
8536 "draining routes must name the drain's reason: {routes:?}"
8537 );
8538 assert!(routes.iter().any(|route| {
8539 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
8540 }));
8541 assert!(routes.iter().any(|route| {
8542 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
8543 }));
8544
8545 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8546 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
8547 if std::env::var_os("UPDATE_GOLDEN").is_some() {
8548 std::fs::write(
8549 &golden_path,
8550 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
8551 )
8552 .unwrap();
8553 }
8554 let expected: Value =
8555 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
8556 assert_eq!(actual, expected);
8557 }
8558
8559 async fn query_live_roots(
8560 handler: &ControlHandler,
8561 module_ctx: &RouteCtx,
8562 ) -> ModuleControlResponseToModule {
8563 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
8564 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
8565 let response = handler
8566 .handle_control_frame(module_ctx, frame)
8567 .await
8568 .unwrap()
8569 .pop()
8570 .unwrap();
8571 serde_json::from_slice(&response.body).unwrap()
8572 }
8573
8574 #[tokio::test(start_paused = true)]
8575 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
8576 let registry = Arc::new(Registry::default());
8577 let forwarding = Arc::new(ForwardingTable::default());
8578 let handler = ControlHandler::with_forwarding(registry, forwarding);
8579 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
8580 hello_via_sink(
8581 &handler,
8582 &target_ctx,
8583 &mut target_rx,
8584 hello_frame("target", PROTOCOL_VERSION, 1),
8585 )
8586 .await;
8587 let root = unique_project_root("live-roots-known");
8588 let path = ProjectRootId::from_path_allowing_missing(root.path())
8589 .unwrap()
8590 .as_path()
8591 .to_path_buf();
8592 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
8593 let open_handler = handler.clone();
8594 let opened = tokio::spawn(async move {
8595 open_handler
8596 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
8597 .await
8598 .unwrap()
8599 });
8600 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8601 .await
8602 .unwrap()
8603 .unwrap();
8604 handler
8605 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
8606 .await
8607 .unwrap();
8608 assert!(opened.await.unwrap().is_empty());
8609 let _ = client_rx.recv().await.unwrap();
8610
8611 let root = unique_project_root("live-roots-pending");
8612 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
8613 .unwrap()
8614 .as_path()
8615 .to_path_buf();
8616 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
8617 let open_handler = handler.clone();
8618 let pending = tokio::spawn(async move {
8619 open_handler
8620 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
8621 .await
8622 .unwrap()
8623 });
8624 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8625 .await
8626 .unwrap()
8627 .unwrap();
8628 let actual = query_live_roots(&handler, &target_ctx).await;
8629 let ModuleControlResponseToModule::LiveRoots {
8630 roots,
8631 unknown_root_bindings,
8632 total_bindings,
8633 } = actual
8634 else {
8635 panic!("expected live roots")
8636 };
8637 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
8638 assert_eq!(unknown_root_bindings, 0);
8639 assert_eq!(
8640 roots.len(),
8641 2,
8642 "root-known arm must retain each canonical root"
8643 );
8644 assert_eq!(
8645 total_bindings,
8646 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
8647 );
8648 let counts = roots
8649 .iter()
8650 .map(|root| (root.project_root.clone(), root.bound, root.pending))
8651 .collect::<Vec<_>>();
8652 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
8653 expected.sort_by(|a, b| a.0.cmp(&b.0));
8654 assert_eq!(
8655 counts, expected,
8656 "roots must sort by path and count pending separately"
8657 );
8658 handler
8659 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
8660 .await
8661 .unwrap();
8662 assert!(pending.await.unwrap().is_empty());
8663 }
8664
8665 #[tokio::test(start_paused = true)]
8666 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
8667 let forwarding = Arc::new(ForwardingTable::default());
8668 let handler =
8669 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
8670 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
8671 hello_via_sink(
8672 &handler,
8673 &target_ctx,
8674 &mut target_rx,
8675 hello_frame("target", PROTOCOL_VERSION, 1),
8676 )
8677 .await;
8678 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
8679 let pending = forwarding
8680 .begin_route_bind_relay_for_test(
8681 client_ctx.connection_id,
8682 client_ctx.egress.clone(),
8683 2,
8684 "target",
8685 )
8686 .unwrap();
8687 forwarding
8688 .complete_pending_relay(
8689 target_ctx.connection_id,
8690 pending.corr,
8691 RouteBindRelayOutcome::Accepted,
8692 )
8693 .unwrap();
8694 let actual = query_live_roots(&handler, &target_ctx).await;
8695 let ModuleControlResponseToModule::LiveRoots {
8696 roots,
8697 unknown_root_bindings,
8698 total_bindings,
8699 } = actual
8700 else {
8701 panic!("expected live roots")
8702 };
8703 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
8704 assert_eq!(
8705 unknown_root_bindings, 1,
8706 "unknown-root arm must not read as no bindings"
8707 );
8708 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
8709 assert_eq!(
8710 total_bindings,
8711 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
8712 );
8713 }
8714
8715 #[tokio::test(start_paused = true)]
8723 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
8724 let forwarding = Arc::new(ForwardingTable::default());
8725 let handler =
8726 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
8727 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
8728 let replies = handler
8729 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
8730 .await
8731 .unwrap();
8732 let queued_by_hello = module_rx.len();
8733
8734 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
8735 let open_handler = handler.clone();
8736 let open = tokio::spawn(async move {
8737 open_handler
8738 .handle_control_frame(
8739 &client_ctx,
8740 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
8741 )
8742 .await
8743 .unwrap()
8744 });
8745 let mut spins = 0;
8747 while module_rx.len() == queued_by_hello {
8748 spins += 1;
8749 assert!(spins < 10_000, "route.open never queued a route.bind");
8750 tokio::task::yield_now().await;
8751 }
8752
8753 for reply in replies {
8755 module_ctx.egress.send(reply).await.unwrap();
8756 }
8757
8758 let first = module_rx.recv().await.unwrap().frame;
8759 assert_eq!(
8760 first.header.ty,
8761 FrameType::HelloAck,
8762 "the first frame a registering module reads must be its HELLO_ACK"
8763 );
8764 assert_eq!(first.header.corr, 7);
8765 let second = module_rx.recv().await.unwrap().frame;
8766 assert_eq!(second.header.ty, FrameType::Request);
8767 assert!(
8768 matches!(
8769 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
8770 ModuleControlRequest::RouteBind { .. }
8771 ),
8772 "the route.bind follows the ack"
8773 );
8774 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
8775
8776 handler
8777 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
8778 .await
8779 .unwrap();
8780 assert!(open.await.unwrap().is_empty());
8781 let _ = client_rx.recv().await.unwrap();
8782 }
8783
8784 #[tokio::test(start_paused = true)]
8785 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
8786 let handler = ControlHandler::with_forwarding(
8787 Arc::new(Registry::default()),
8788 Arc::new(ForwardingTable::default()),
8789 );
8790 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
8791 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
8792 hello_via_sink(
8793 &handler,
8794 &first_ctx,
8795 &mut first_rx,
8796 hello_frame("first", PROTOCOL_VERSION, 1),
8797 )
8798 .await;
8799 hello_via_sink(
8800 &handler,
8801 &second_ctx,
8802 &mut second_rx,
8803 hello_frame("second", PROTOCOL_VERSION, 2),
8804 )
8805 .await;
8806 let root = unique_project_root("second-only");
8807 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
8808 let cloned = handler.clone();
8809 let open = tokio::spawn(async move {
8810 cloned
8811 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
8812 .await
8813 .unwrap()
8814 });
8815 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
8816 .await
8817 .unwrap()
8818 .unwrap();
8819 let first = query_live_roots(&handler, &first_ctx).await;
8820 let second = query_live_roots(&handler, &second_ctx).await;
8821 assert!(
8822 matches!(
8823 first,
8824 ModuleControlResponseToModule::LiveRoots {
8825 total_bindings: 0,
8826 ..
8827 }
8828 ),
8829 "cross-module scope must not expose another module's roots"
8830 );
8831 assert!(
8832 matches!(
8833 second,
8834 ModuleControlResponseToModule::LiveRoots {
8835 total_bindings: 1,
8836 ..
8837 }
8838 ),
8839 "second module must see its pending route"
8840 );
8841 handler
8842 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
8843 .await
8844 .unwrap();
8845 assert!(open.await.unwrap().is_empty());
8846 }
8847
8848 #[tokio::test(start_paused = true)]
8849 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
8850 let handler = ControlHandler::with_forwarding(
8851 Arc::new(Registry::default()),
8852 Arc::new(ForwardingTable::default()),
8853 );
8854 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
8855 hello_via_sink(
8856 &handler,
8857 &target_ctx,
8858 &mut target_rx,
8859 hello_frame("target", PROTOCOL_VERSION, 1),
8860 )
8861 .await;
8862 let actual = query_live_roots(&handler, &target_ctx).await;
8863 let ModuleControlResponseToModule::LiveRoots {
8864 roots,
8865 unknown_root_bindings,
8866 total_bindings,
8867 } = actual
8868 else {
8869 panic!("expected live roots")
8870 };
8871 assert!(roots.is_empty());
8872 assert_eq!(unknown_root_bindings, 0);
8873 assert_eq!(total_bindings, 0);
8874 assert_eq!(
8875 total_bindings,
8876 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
8877 );
8878 }
8879
8880 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
8887 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8888 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
8889 let text = std::fs::read_to_string(&path)
8890 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
8891 let vectors: Vec<(String, Value)> = text
8892 .lines()
8893 .filter(|line| !line.trim().is_empty())
8894 .map(|line| {
8895 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
8896 let id = entry["corpus_id"]
8897 .as_str()
8898 .expect("every vector carries a corpus_id")
8899 .to_string();
8900 (id, entry["package"].clone())
8901 })
8902 .collect();
8903 assert_eq!(
8907 vectors.len(),
8908 3,
8909 "vendored fed corpus changed size; re-sync from subc-federation"
8910 );
8911
8912 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
8927 let richest = vectors
8928 .iter()
8929 .filter_map(|(_, package)| package.as_object())
8930 .map(|object| {
8931 object
8932 .keys()
8933 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
8934 .count()
8935 })
8936 .max()
8937 .unwrap_or(0);
8938 assert!(
8939 richest >= 2,
8940 "vendored corpus no longer carries a package with unmodelled fields, \
8941 so the relay test can no longer distinguish a verbatim relay from a lossy one"
8942 );
8943
8944 vectors
8945 }
8946
8947 #[tokio::test]
8955 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
8956 for (corpus_id, package) in fed_admission_facts_vectors() {
8957 let registry = Arc::new(Registry::default());
8958 let forwarding = Arc::new(ForwardingTable::default());
8959 let supervisor = SupervisorHandle::new();
8960 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8961 let handler =
8962 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8963 .with_supervisor(supervisor)
8964 .with_admission_facts_config(
8965 Some("fed".to_string()),
8966 Some(vec!["target".to_string()]),
8967 );
8968
8969 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
8970 hello_via_sink(
8971 &handler,
8972 &target_ctx,
8973 &mut target_rx,
8974 hello_frame("target", PROTOCOL_VERSION, 1),
8975 )
8976 .await;
8977
8978 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
8979 let route_handler = handler.clone();
8980 let expected = package.clone();
8981 let route_task = tokio::spawn(async move {
8982 route_handler
8983 .handle_control_frame(
8984 &client_ctx,
8985 route_open_frame_with_admission_facts(
8986 20,
8987 "target",
8988 unique_project_root("admission-facts"),
8989 Some(subc_control::ConsumerIdentity {
8990 module_id: "fed".to_string(),
8991 launch_nonce: "fed-nonce".to_string(),
8992 }),
8993 Some(package),
8994 ),
8995 )
8996 .await
8997 .unwrap()
8998 });
8999
9000 let bind_frame = target_rx.recv().await.unwrap();
9001 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9002 let ModuleControlRequest::RouteBind {
9003 admission_facts, ..
9004 } = bind
9005 else {
9006 panic!("{corpus_id}: expected route.bind")
9007 };
9008 assert_eq!(
9009 admission_facts,
9010 Some(expected),
9011 "{corpus_id}: relay must not add, drop or reshape any field"
9012 );
9013
9014 handler
9015 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
9016 .await
9017 .unwrap();
9018 route_task.await.unwrap();
9019 }
9020 }
9021
9022 #[tokio::test]
9023 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
9024 let registry = Arc::new(Registry::default());
9025 let forwarding = Arc::new(ForwardingTable::default());
9026 let supervisor = SupervisorHandle::new();
9027 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9028 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
9029 let handler =
9030 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9031 .with_supervisor(supervisor)
9032 .with_admission_facts_config(
9033 Some("fed".to_string()),
9034 Some(vec!["target".to_string()]),
9035 );
9036
9037 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
9038 hello_via_sink(
9039 &handler,
9040 &target_ctx,
9041 &mut target_rx,
9042 hello_frame("target", PROTOCOL_VERSION, 1),
9043 )
9044 .await;
9045 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
9046 hello_via_sink(
9047 &handler,
9048 &other_ctx,
9049 &mut other_rx,
9050 hello_frame("other", PROTOCOL_VERSION, 2),
9051 )
9052 .await;
9053
9054 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
9055 let expected_facts = facts.clone();
9056 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
9057 let route_handler = handler.clone();
9058 let route_task = tokio::spawn(async move {
9059 route_handler
9060 .handle_control_frame(
9061 &client_ctx,
9062 route_open_frame_with_admission_facts(
9063 10,
9064 "target",
9065 unique_project_root("admission-facts"),
9066 Some(subc_control::ConsumerIdentity {
9067 module_id: "fed".to_string(),
9068 launch_nonce: "fed-nonce".to_string(),
9069 }),
9070 Some(facts.clone()),
9071 ),
9072 )
9073 .await
9074 .unwrap()
9075 });
9076 let bind_frame = target_rx.recv().await.unwrap();
9077 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9078 let ModuleControlRequest::RouteBind {
9079 admission_facts, ..
9080 } = bind
9081 else {
9082 panic!("expected route.bind")
9083 };
9084 assert_eq!(admission_facts, Some(expected_facts));
9085 handler
9086 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
9087 .await
9088 .unwrap();
9089 assert!(route_task.await.unwrap().is_empty());
9090 assert!(matches!(
9091 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
9092 .unwrap(),
9093 ClientControlResponse::RouteOpen { .. }
9094 ));
9095
9096 let direct = handler
9097 .handle_control_frame(
9098 &route_ctx(ConnectionId::new(73)).0,
9099 route_open_frame_with_admission_facts(
9100 11,
9101 "target",
9102 unique_project_root("admission-facts"),
9103 None,
9104 Some(json!({"x": 1})),
9105 ),
9106 )
9107 .await
9108 .unwrap();
9109 assert_eq!(
9110 parse_error(&direct[0])["code"],
9111 "admission_facts_not_permitted"
9112 );
9113
9114 let different_reserved = handler
9115 .handle_control_frame(
9116 &route_ctx(ConnectionId::new(77)).0,
9117 route_open_frame_with_admission_facts(
9118 15,
9119 "target",
9120 unique_project_root("admission-facts"),
9121 Some(subc_control::ConsumerIdentity {
9122 module_id: "other".to_string(),
9123 launch_nonce: "other-nonce".to_string(),
9124 }),
9125 Some(json!({"x": 1})),
9126 ),
9127 )
9128 .await
9129 .unwrap();
9130 assert_eq!(
9131 parse_error(&different_reserved[0])["code"],
9132 "admission_facts_not_permitted"
9133 );
9134
9135 let other_target = handler
9136 .handle_control_frame(
9137 &route_ctx(ConnectionId::new(74)).0,
9138 route_open_frame_with_admission_facts(
9139 12,
9140 "other",
9141 unique_project_root("admission-facts"),
9142 Some(subc_control::ConsumerIdentity {
9143 module_id: "fed".to_string(),
9144 launch_nonce: "fed-nonce".to_string(),
9145 }),
9146 Some(json!({"x": 1})),
9147 ),
9148 )
9149 .await
9150 .unwrap();
9151 assert_eq!(
9152 parse_error(&other_target[0])["code"],
9153 "admission_facts_target_not_allowed"
9154 );
9155
9156 let nonexistent = handler
9157 .handle_control_frame(
9158 &route_ctx(ConnectionId::new(75)).0,
9159 route_open_frame_with_admission_facts(
9160 13,
9161 "missing",
9162 unique_project_root("admission-facts"),
9163 None,
9164 Some(json!({"x": 1})),
9165 ),
9166 )
9167 .await
9168 .unwrap();
9169 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
9170
9171 let described = handler
9172 .handle_control_frame(
9173 &route_ctx(ConnectionId::new(76)).0,
9174 Frame::build(
9175 FrameType::Request,
9176 control_flags(),
9177 0,
9178 0,
9179 14,
9180 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
9181 )
9182 .unwrap(),
9183 )
9184 .await
9185 .unwrap();
9186 let ClientControlResponse::ServerDescribe { capabilities, .. } =
9187 serde_json::from_slice(&described[0].body).unwrap()
9188 else {
9189 panic!("expected server.describe response")
9190 };
9191 assert!(capabilities
9192 .iter()
9193 .any(|cap| cap == "admission_facts_relay_v1"));
9194 }
9195
9196 #[tokio::test]
9197 async fn admission_facts_without_configured_carrier_are_rejected() {
9198 let registry = Arc::new(Registry::default());
9199 let forwarding = Arc::new(ForwardingTable::default());
9200 let handler = ControlHandler::with_forwarding(registry, forwarding);
9201 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
9202 hello_via_sink(
9203 &handler,
9204 &target_ctx,
9205 &mut target_rx,
9206 hello_frame("target", PROTOCOL_VERSION, 1),
9207 )
9208 .await;
9209
9210 let responses = handler
9211 .handle_control_frame(
9212 &route_ctx(ConnectionId::new(79)).0,
9213 route_open_frame_with_admission_facts(
9214 16,
9215 "target",
9216 unique_project_root("admission-facts"),
9217 None,
9218 Some(json!({"x": 1})),
9219 ),
9220 )
9221 .await
9222 .unwrap();
9223 assert_eq!(
9224 parse_error(&responses[0])["code"],
9225 "admission_facts_not_permitted"
9226 );
9227 }
9228
9229 #[tokio::test]
9230 async fn route_open_relays_consumer_capabilities_verbatim() {
9231 let registry = Arc::new(Registry::default());
9232 let forwarding = Arc::new(ForwardingTable::default());
9233 let handler =
9234 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9235 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
9236 hello_via_sink(
9237 &handler,
9238 &module_ctx,
9239 &mut module_rx,
9240 hello_frame("aft", PROTOCOL_VERSION, 7),
9241 )
9242 .await;
9243
9244 let expected = vec!["elicitation".to_string(), "roots".to_string()];
9245 let expected_for_request = expected.clone();
9246 let project_root = unique_project_root("consumer-capabilities-present");
9247 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
9248 let route_handler = handler.clone();
9249 let route_task = tokio::spawn(async move {
9250 route_handler
9251 .handle_control_frame(
9252 &client_ctx,
9253 route_open_frame_with_consumer_capabilities(
9254 401,
9255 "aft",
9256 project_root,
9257 Some(expected_for_request),
9258 ),
9259 )
9260 .await
9261 .unwrap()
9262 });
9263 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9264 .await
9265 .unwrap()
9266 .unwrap();
9267 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9268 let ModuleControlRequest::RouteBind {
9269 consumer_capabilities,
9270 ..
9271 } = bind
9272 else {
9273 panic!("expected route.bind request, got {bind:?}");
9274 };
9275 assert_eq!(consumer_capabilities, Some(expected.clone()));
9276
9277 handler
9278 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9279 .await
9280 .unwrap();
9281 let route_response = route_task.await.unwrap();
9282 assert!(route_response.is_empty());
9283 let published = client_rx.recv().await.unwrap();
9284 assert!(matches!(
9285 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9286 ClientControlResponse::RouteOpen { .. }
9287 ));
9288 }
9289
9290 #[tokio::test]
9291 async fn route_open_without_consumer_capabilities_relays_none() {
9292 let registry = Arc::new(Registry::default());
9293 let forwarding = Arc::new(ForwardingTable::default());
9294 let handler =
9295 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9296 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
9297 hello_via_sink(
9298 &handler,
9299 &module_ctx,
9300 &mut module_rx,
9301 hello_frame("aft", PROTOCOL_VERSION, 7),
9302 )
9303 .await;
9304
9305 let project_root = unique_project_root("consumer-capabilities-absent");
9306 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
9307 let route_handler = handler.clone();
9308 let route_task = tokio::spawn(async move {
9309 route_handler
9310 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
9311 .await
9312 .unwrap()
9313 });
9314 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9315 .await
9316 .unwrap()
9317 .unwrap();
9318 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9319 let ModuleControlRequest::RouteBind {
9320 consumer_capabilities,
9321 ..
9322 } = bind
9323 else {
9324 panic!("expected route.bind request, got {bind:?}");
9325 };
9326 assert_eq!(consumer_capabilities, None);
9327
9328 handler
9329 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9330 .await
9331 .unwrap();
9332 let route_response = route_task.await.unwrap();
9333 assert!(route_response.is_empty());
9334 let published = client_rx.recv().await.unwrap();
9335 assert!(matches!(
9336 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9337 ClientControlResponse::RouteOpen { .. }
9338 ));
9339 }
9340
9341 async fn bind_role_versions_for(
9345 sent: Option<BTreeMap<String, String>>,
9346 connection: u64,
9347 ) -> Option<BTreeMap<String, String>> {
9348 let registry = Arc::new(Registry::default());
9349 let forwarding = Arc::new(ForwardingTable::default());
9350 let handler =
9351 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9352 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
9353 hello_via_sink(
9354 &handler,
9355 &module_ctx,
9356 &mut module_rx,
9357 hello_frame("aft", PROTOCOL_VERSION, 7),
9358 )
9359 .await;
9360 let project_root = unique_project_root("role-versions");
9361 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
9362 let route_handler = handler.clone();
9363 let route_task = tokio::spawn(async move {
9364 route_handler
9365 .handle_control_frame(
9366 &client_ctx,
9367 route_open_frame_with_role_versions(403, "aft", project_root, sent),
9368 )
9369 .await
9370 .unwrap()
9371 });
9372 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9373 .await
9374 .expect("a well-formed route.open reaches the module as a bind")
9375 .unwrap();
9376 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9377 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
9378 panic!("expected route.bind request, got {bind:?}");
9379 };
9380 handler
9381 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9382 .await
9383 .unwrap();
9384 assert!(route_task.await.unwrap().is_empty());
9385 let published = client_rx.recv().await.unwrap();
9386 assert!(matches!(
9387 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9388 ClientControlResponse::RouteOpen { .. }
9389 ));
9390 role_versions
9391 }
9392
9393 #[tokio::test]
9394 async fn route_open_relays_role_versions_verbatim() {
9395 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
9396 assert_eq!(
9397 bind_role_versions_for(Some(sent.clone()), 141).await,
9398 Some(sent)
9399 );
9400 }
9401
9402 #[tokio::test]
9405 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
9406 assert_eq!(bind_role_versions_for(None, 143).await, None);
9407 assert_eq!(
9408 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
9409 None
9410 );
9411 }
9412
9413 #[tokio::test]
9414 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
9415 let registry = Arc::new(Registry::default());
9416 let forwarding = Arc::new(ForwardingTable::default());
9417 let handler =
9418 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9419 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
9420 hello_via_sink(
9421 &handler,
9422 &module_ctx,
9423 &mut module_rx,
9424 hello_frame("aft", PROTOCOL_VERSION, 7),
9425 )
9426 .await;
9427
9428 let nine: BTreeMap<String, String> = (0..9)
9429 .map(|index| (format!("role-{index}"), "v1".to_string()))
9430 .collect();
9431 for (label, malformed) in [
9432 (
9433 "invalid role name",
9434 role_versions(&[("Tool_Provider", "v1")]),
9435 ),
9436 ("invalid version", role_versions(&[("tool-provider", "v0")])),
9437 ("nine entries", nine),
9438 ] {
9439 let responses = tokio::time::timeout(
9442 Duration::from_secs(1),
9443 handler.handle_control_frame(
9444 &route_ctx(ConnectionId::new(148)).0,
9445 route_open_frame_with_role_versions(
9446 404,
9447 "aft",
9448 unique_project_root("role-versions-malformed"),
9449 Some(malformed),
9450 ),
9451 ),
9452 )
9453 .await
9454 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
9455 .unwrap();
9456 assert_eq!(responses.len(), 1, "{label}");
9457 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
9458 let error = parse_error(&responses[0]);
9459 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
9460 assert_eq!(
9461 error["detail"]["field"], "role_versions",
9462 "{label}: {error}"
9463 );
9464 assert!(
9465 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
9466 "{label}: a malformed declaration is terminal"
9467 );
9468 assert!(
9469 module_rx.try_recv().is_err(),
9470 "{label}: the module must never see a bind"
9471 );
9472 }
9473 }
9474
9475 #[tokio::test]
9479 async fn route_role_versions_capability_is_advertised() {
9480 let handler = ControlHandler::new(Arc::new(Registry::default()));
9481 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
9482 let ack = hello_via_sink(
9483 &handler,
9484 &ctx,
9485 &mut rx,
9486 hello_frame("m", PROTOCOL_VERSION, 1),
9487 )
9488 .await;
9489 let ack = parse_ack(&ack);
9490 assert!(
9491 ack.subc_capabilities
9492 .iter()
9493 .any(|c| c == "route-role-versions/v1"),
9494 "{:?}",
9495 ack.subc_capabilities
9496 );
9497
9498 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
9499 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
9500 let reply = handler
9501 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
9502 .await
9503 .unwrap()
9504 .pop()
9505 .unwrap();
9506 let ClientControlResponse::ServerDescribe { capabilities, .. } =
9507 serde_json::from_slice(&reply.body).unwrap()
9508 else {
9509 panic!("not a server.describe reply");
9510 };
9511 assert!(
9512 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
9513 "{capabilities:?}"
9514 );
9515 }
9516
9517 #[tokio::test]
9518 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
9519 let registry = Arc::new(Registry::default());
9520 let forwarding = Arc::new(ForwardingTable::default());
9521 let handler =
9522 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9523 .with_health_probe_timeout(Duration::from_secs(5));
9524 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
9525 hello_via_sink(
9526 &handler,
9527 &module_ctx,
9528 &mut module_rx,
9529 non_routable_hello_frame_with_control_ops(
9530 "mcp",
9531 300,
9532 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9533 ),
9534 )
9535 .await;
9536 assert!(registry
9537 .get_module("mcp")
9538 .unwrap()
9539 .unwrap()
9540 .manifest
9541 .provides
9542 .is_empty());
9543
9544 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
9545 let route_response = handler
9546 .handle_control_frame(
9547 &route_client_ctx,
9548 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
9549 )
9550 .await
9551 .unwrap();
9552 assert_eq!(route_response[0].header.ty, FrameType::Error);
9553 assert_eq!(
9554 parse_error(&route_response[0])["code"],
9555 "target_unavailable"
9556 );
9557 assert!(parse_error(&route_response[0])["message"]
9558 .as_str()
9559 .unwrap()
9560 .contains("does not provide the requested target"));
9561 assert!(module_rx.try_recv().is_err());
9562
9563 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
9564 let health_handler = handler.clone();
9565 let health_task = tokio::spawn(async move {
9566 health_handler
9567 .handle_control_frame(
9568 &health_client_ctx,
9569 supervisor_health_probe_frame(302, "mcp"),
9570 )
9571 .await
9572 .unwrap()
9573 });
9574 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9575 .await
9576 .unwrap()
9577 .unwrap();
9578 assert_eq!(
9579 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
9580 ModuleControlRequest::HealthCheck {}
9581 );
9582 handler
9583 .handle_control_frame(
9584 &module_ctx,
9585 health_response(health_frame.header.corr, HealthStatus::Ok),
9586 )
9587 .await
9588 .unwrap();
9589 let health_response = health_task.await.unwrap();
9590 assert_eq!(health_response[0].header.ty, FrameType::Response);
9591 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
9592 ClientControlResponse::SupervisorHealthProbe {
9593 module_id, status, ..
9594 } => {
9595 assert_eq!(module_id, "mcp");
9596 assert_eq!(status, HealthStatus::Ok);
9597 }
9598 other => panic!("unexpected health response: {other:?}"),
9599 }
9600
9601 forwarding
9606 .cleanup_connection(module_ctx.connection_id)
9607 .unwrap();
9608 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
9609 let cleanup_response = tokio::time::timeout(
9610 Duration::from_millis(200),
9611 handler.handle_control_frame(
9612 &cleanup_probe_ctx,
9613 supervisor_health_probe_frame(303, "mcp"),
9614 ),
9615 )
9616 .await
9617 .expect("probe should fail immediately when the control lane is gone")
9618 .unwrap();
9619 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
9620 assert_eq!(
9621 parse_error(&cleanup_response[0])["code"],
9622 "target_unavailable"
9623 );
9624 assert!(parse_error(&cleanup_response[0])["message"]
9625 .as_str()
9626 .unwrap()
9627 .contains("no module connection"));
9628
9629 handler
9630 .cleanup_connection(module_ctx.connection_id)
9631 .unwrap();
9632 }
9633
9634 #[tokio::test]
9635 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
9636 let registry = Arc::new(Registry::default());
9637 let supervisor_handle = SupervisorHandle::new();
9638 let supervisor =
9639 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
9640 .with_handle(supervisor_handle.clone())
9641 .with_connection_file_path(
9642 std::env::temp_dir()
9643 .join(format!("subc-route-open-warming-{}", std::process::id())),
9644 );
9645 let module = supervisor
9646 .supervise_configured(
9647 ModuleSpec {
9648 module_id: "warming".to_string(),
9649 program: fake_aft_stub_path(),
9650 args: Vec::new(),
9651 env: Vec::new(),
9652 reserved: false,
9653 reserved_prefixes: Vec::new(),
9654 protocol: ModuleProtocol::Subc,
9655 overlap: Default::default(),
9656 },
9657 true,
9658 )
9659 .unwrap();
9660 assert_eq!(module.state().unwrap(), ModuleState::Running);
9661
9662 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9663 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
9664 let response = handler
9665 .handle_control_frame(
9666 &ctx,
9667 route_open_frame(304, "warming", unique_project_root("warming")),
9668 )
9669 .await
9670 .unwrap();
9671 module.stop().await.unwrap();
9672
9673 assert_eq!(response[0].header.ty, FrameType::Error);
9674 let error = parse_error(&response[0]);
9675 assert_eq!(error["code"], "module_warming");
9676 assert!(error["message"]
9677 .as_str()
9678 .unwrap()
9679 .contains("state=running, enabled=true, live=false"));
9680 }
9681
9682 #[test]
9683 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
9684 let handler = ControlHandler::new(Arc::new(Registry::default()));
9685 let capture = EventCapture::default();
9686 let _subscriber =
9687 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9688 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
9689 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
9690 let pending = (0..limit).collect::<Vec<_>>();
9691 let response = handler
9692 .route_open_capacity_refusal(
9693 &ctx,
9694 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
9695 "busy",
9696 pending.len(),
9697 limit,
9698 )
9699 .unwrap();
9700 assert_eq!(parse_error(&response)["code"], "target_unavailable");
9701 let event = capture
9702 .events()
9703 .into_iter()
9704 .find(|event| {
9705 event.target == "control"
9706 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
9707 })
9708 .expect("connection admission refusal event");
9709 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
9710 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
9711 }
9712
9713 #[test]
9714 fn route_open_target_cap_logs_admission_reason_and_capacity() {
9715 let handler = ControlHandler::new(Arc::new(Registry::default()));
9716 let capture = EventCapture::default();
9717 let _subscriber =
9718 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9719 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
9720 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
9721 let guards = (0..limit)
9722 .map(|_| {
9723 handler
9724 .route_bind_concurrency
9725 .try_admit("busy", limit)
9726 .unwrap()
9727 })
9728 .collect::<Vec<_>>();
9729 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
9730 Err(in_flight) => in_flight,
9731 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
9732 };
9733 let response = handler
9734 .route_open_target_capacity_refusal(
9735 &ctx,
9736 &route_open_frame(397, "busy", unique_project_root("target-cap")),
9737 "busy",
9738 in_flight,
9739 )
9740 .unwrap();
9741 assert_eq!(parse_error(&response)["code"], "target_unavailable");
9742 let event = capture
9743 .events()
9744 .into_iter()
9745 .find(|event| {
9746 event.target == "control"
9747 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
9748 })
9749 .expect("target admission refusal event");
9750 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
9751 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
9752 drop(guards);
9753 }
9754
9755 #[tokio::test]
9760 async fn route_open_refusal_names_the_check_that_refused() {
9761 let handler = ControlHandler::new(Arc::new(Registry::default()));
9762 let capture = EventCapture::default();
9763 let _subscriber =
9764 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9765 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
9766 let response = handler
9767 .handle_control_frame(
9768 &ctx,
9769 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
9770 )
9771 .await
9772 .unwrap();
9773
9774 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
9775 let event = capture
9776 .events()
9777 .into_iter()
9778 .find(|event| {
9779 event.target == "control"
9780 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
9781 })
9782 .expect("route.open refusal event");
9783 assert_eq!(
9784 event.fields.get("reason"),
9785 Some(&"\"not_registered\"".to_string())
9786 );
9787 }
9788
9789 #[tokio::test]
9790 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
9791 let registry = Arc::new(Registry::default());
9792 let supervisor_handle = SupervisorHandle::new();
9793 let supervisor =
9794 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
9795 .with_handle(supervisor_handle.clone())
9796 .with_connection_file_path(std::env::temp_dir().join(format!(
9797 "subc-route-open-refusal-info-{}",
9798 std::process::id()
9799 )));
9800 let module = supervisor
9801 .supervise_configured(
9802 ModuleSpec {
9803 module_id: "warming".to_string(),
9804 program: fake_aft_stub_path(),
9805 args: Vec::new(),
9806 env: Vec::new(),
9807 reserved: false,
9808 reserved_prefixes: Vec::new(),
9809 protocol: ModuleProtocol::Subc,
9810 overlap: Default::default(),
9811 },
9812 true,
9813 )
9814 .unwrap();
9815 assert_eq!(module.state().unwrap(), ModuleState::Running);
9816
9817 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9818 assert!(handler
9819 .counters()
9820 .snapshot()
9821 .get("route_open_refused_by_code")
9822 .is_none());
9823 let capture = EventCapture::default();
9824 let _subscriber =
9825 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9826 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
9827 let response = handler
9828 .handle_control_frame(
9829 &ctx,
9830 route_open_frame(394, "warming", unique_project_root("refusal-info")),
9831 )
9832 .await
9833 .unwrap();
9834 module.stop().await.unwrap();
9835
9836 assert_eq!(parse_error(&response[0])["code"], "module_warming");
9837 let event = capture
9838 .events()
9839 .into_iter()
9840 .find(|event| {
9841 event.target == "control"
9842 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
9843 })
9844 .expect("route.open refusal event");
9845 assert_eq!(
9846 event.fields.get("module_id"),
9847 Some(&"\"warming\"".to_string())
9848 );
9849 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
9850 assert_eq!(
9851 event.fields.get("reason"),
9852 Some(&"\"supervised_not_registered\"".to_string())
9853 );
9854 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
9855 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
9856 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
9857 assert_eq!(
9858 handler.counters().snapshot()["route_open_refused_by_code"],
9859 json!({ "module_warming": 1 })
9860 );
9861 }
9862
9863 const OUTAGE_START: &str = "route.open refusing module: not serving";
9864 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
9865
9866 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
9867 capture
9868 .events()
9869 .into_iter()
9870 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
9871 .collect()
9872 }
9873
9874 fn supervise_stub(
9875 registry: &Arc<Registry>,
9876 module_id: &str,
9877 enabled: bool,
9878 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
9879 let supervisor_handle = SupervisorHandle::new();
9880 let supervisor =
9881 Supervisor::new(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
9882 .with_handle(supervisor_handle.clone())
9883 .with_connection_file_path(std::env::temp_dir().join(format!(
9884 "subc-route-outage-{module_id}-{}",
9885 std::process::id()
9886 )));
9887 let module = supervisor
9888 .supervise_configured(
9889 ModuleSpec {
9890 module_id: module_id.to_string(),
9891 program: fake_aft_stub_path(),
9892 args: Vec::new(),
9893 env: Vec::new(),
9894 reserved: false,
9895 reserved_prefixes: Vec::new(),
9896 protocol: ModuleProtocol::Subc,
9897 overlap: Default::default(),
9898 },
9899 enabled,
9900 )
9901 .unwrap();
9902 (supervisor_handle, module)
9903 }
9904
9905 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
9906 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
9907 module_id: module_id.to_string(),
9908 drain_timeout_ms: Some(50),
9909 })
9910 .unwrap();
9911 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9912 }
9913
9914 #[test]
9918 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
9919 let registry = Arc::new(Registry::default());
9920 let forwarding = Arc::new(ForwardingTable::default());
9921 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9922 let second = ControlHandler::with_forwarding(registry, forwarding);
9923 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
9924 }
9925
9926 #[tokio::test(flavor = "current_thread")]
9930 async fn route_open_unknown_module_refusals_add_no_outage_state() {
9931 let handler = ControlHandler::new(Arc::new(Registry::default()));
9932 let capture = EventCapture::default();
9933 let _subscriber =
9934 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9935 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
9936 for corr in 0..8 {
9937 let response = handler
9938 .handle_control_frame(
9939 &ctx,
9940 route_open_frame(
9941 380 + corr,
9942 &format!("nobody-{corr}"),
9943 unique_project_root("outage-unknown"),
9944 ),
9945 )
9946 .await
9947 .unwrap();
9948 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
9949 }
9950
9951 assert_eq!(handler.route_outages.tracked_module_count(), 0);
9952 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
9953 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
9954 }
9955
9956 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9961 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
9962 let registry = Arc::new(Registry::default());
9963 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
9964 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9965 let capture = EventCapture::default();
9966 let _subscriber =
9967 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9968 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
9969 handler.route_outages.record_accepted("outage-restart");
9972
9973 let response = handler
9974 .handle_control_frame(
9975 &ctx,
9976 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
9977 )
9978 .await
9979 .unwrap();
9980 assert_eq!(response[0].header.ty, FrameType::Error);
9981 let starts = outage_lines(&capture, OUTAGE_START);
9982 assert_eq!(starts.len(), 1, "{starts:?}");
9983 assert_eq!(starts[0].level, tracing::Level::WARN);
9984 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
9985 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
9986 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
9987 handler.route_outages.record_accepted("outage-restart");
9988 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
9989
9990 let restart = handler
9991 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
9992 .await
9993 .unwrap();
9994 assert_eq!(
9995 restart[0].header.ty,
9996 FrameType::Response,
9997 "{:?}",
9998 parse_error(&restart[0])
9999 );
10000 handler
10001 .handle_control_frame(
10002 &ctx,
10003 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
10004 )
10005 .await
10006 .unwrap();
10007 module.stop().await.unwrap();
10008
10009 let starts = outage_lines(&capture, OUTAGE_START);
10010 assert_eq!(starts.len(), 2, "{starts:?}");
10011 assert_eq!(starts[1].level, tracing::Level::INFO);
10012 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
10013 }
10014
10015 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
10019 async fn failed_operator_restart_leaves_no_operator_mark() {
10020 let registry = Arc::new(Registry::default());
10021 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
10022 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10023 let capture = EventCapture::default();
10024 let _subscriber =
10025 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10026 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
10027 handler.route_outages.record_accepted("outage-disabled");
10028
10029 let restart = handler
10030 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
10031 .await
10032 .unwrap();
10033 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
10034 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
10035
10036 handler
10037 .handle_control_frame(
10038 &ctx,
10039 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
10040 )
10041 .await
10042 .unwrap();
10043 let starts = outage_lines(&capture, OUTAGE_START);
10044 assert_eq!(starts.len(), 1, "{starts:?}");
10045 assert_eq!(starts[0].level, tracing::Level::WARN);
10046 }
10047
10048 #[tokio::test(flavor = "current_thread")]
10049 async fn route_open_unknown_module_escapes_target_module_id() {
10050 let handler = ControlHandler::new(Arc::new(Registry::default()));
10051 let capture = EventCapture::default();
10052 let _subscriber =
10053 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10054 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
10055 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
10056 let response = handler
10057 .handle_control_frame(
10058 &ctx,
10059 route_open_frame(
10060 395,
10061 hostile_module_id,
10062 unique_project_root("hostile-target-module-id"),
10063 ),
10064 )
10065 .await
10066 .unwrap();
10067
10068 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
10069 let event = capture
10070 .events()
10071 .into_iter()
10072 .find(|event| {
10073 event.target == "control"
10074 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
10075 })
10076 .expect("route.open unknown-module refusal event");
10077 let logged = event.fields.get("module_id").expect("module_id field");
10078 assert!(!logged.bytes().any(|byte| byte < 0x20));
10079 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
10080 }
10081
10082 #[tokio::test(flavor = "current_thread")]
10083 async fn route_open_module_rejection_uses_daemon_counter_key() {
10084 let registry = Arc::new(Registry::default());
10085 let forwarding = Arc::new(ForwardingTable::default());
10086 let handler =
10087 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10088 let module_connection = ConnectionId::new(95);
10089 let (module_ctx, mut module_rx) = route_ctx(module_connection);
10090 hello_via_sink(
10091 &handler,
10092 &module_ctx,
10093 &mut module_rx,
10094 hello_frame("aft", PROTOCOL_VERSION, 395),
10095 )
10096 .await;
10097
10098 let client_connection = ConnectionId::new(96);
10099 let (client_ctx, _client_rx) = route_ctx(client_connection);
10100 let capture = EventCapture::default();
10101 let _subscriber =
10102 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10103 let (route_task, bind) = relay_route_open(
10104 &handler,
10105 client_connection,
10106 &client_ctx.egress,
10107 &mut module_rx,
10108 396,
10109 "aft",
10110 "hostile-module-code",
10111 )
10112 .await;
10113 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
10114 let rejection = Frame::build(
10115 FrameType::Error,
10116 control_flags(),
10117 0,
10118 0,
10119 bind.header.corr,
10120 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
10121 )
10122 .unwrap();
10123 handler
10124 .handle_control_frame(&module_ctx, rejection)
10125 .await
10126 .unwrap();
10127
10128 let response = route_task.await.unwrap();
10129 assert_eq!(parse_error(&response[0])["code"], hostile_code);
10130 let counters = handler.counters().snapshot();
10131 assert_eq!(
10132 counters["route_open_refused_by_code"],
10133 json!({ "module_rejected": 1 })
10134 );
10135 assert!(counters["route_open_refused_by_code"]
10136 .get(hostile_code)
10137 .is_none());
10138
10139 let event = capture
10140 .events()
10141 .into_iter()
10142 .find(|event| {
10143 event.target == "control"
10144 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
10145 })
10146 .expect("route.open module-rejection refusal event");
10147 let logged = event.fields.get("module_code").expect("module_code field");
10148 assert!(!logged.bytes().any(|byte| byte < 0x20));
10149 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
10150 }
10151
10152 #[tokio::test]
10153 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
10154 let registry = Arc::new(Registry::default());
10155 let supervisor_handle = SupervisorHandle::new();
10156 let missing_program = std::env::temp_dir().join(format!(
10157 "subc-route-open-missing-program-{}",
10158 std::process::id()
10159 ));
10160 let supervisor =
10161 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
10162 .with_handle(supervisor_handle.clone());
10163 let module = supervisor
10164 .supervise_configured(
10165 ModuleSpec {
10166 module_id: "failed".to_string(),
10167 program: missing_program,
10168 args: Vec::new(),
10169 env: Vec::new(),
10170 reserved: false,
10171 reserved_prefixes: Vec::new(),
10172 protocol: ModuleProtocol::Subc,
10173 overlap: Default::default(),
10174 },
10175 true,
10176 )
10177 .unwrap();
10178 assert_eq!(module.state().unwrap(), ModuleState::Failed);
10179
10180 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10181 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
10182 let response = handler
10183 .handle_control_frame(
10184 &ctx,
10185 route_open_frame(305, "failed", unique_project_root("failed")),
10186 )
10187 .await
10188 .unwrap();
10189
10190 assert_eq!(response[0].header.ty, FrameType::Error);
10191 let error = parse_error(&response[0]);
10192 assert_eq!(error["code"], "target_unavailable");
10193 assert!(error["message"]
10194 .as_str()
10195 .unwrap()
10196 .contains("state=failed, enabled=true, live=false"));
10197 }
10198
10199 #[tokio::test]
10200 async fn route_open_role_mismatch_remains_target_unavailable() {
10201 let registry = Arc::new(Registry::default());
10202 let handler = ControlHandler::new(Arc::clone(®istry));
10203 handler
10204 .handle_control(
10205 ConnectionId::new(41),
10206 non_routable_hello_frame_with_control_ops("health-only", 306, None),
10207 )
10208 .unwrap();
10209
10210 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
10211 let response = handler
10212 .handle_control_frame(
10213 &ctx,
10214 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
10215 )
10216 .await
10217 .unwrap();
10218
10219 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10220 assert!(parse_error(&response[0])["message"]
10221 .as_str()
10222 .unwrap()
10223 .contains("does not provide the requested target"));
10224 }
10225
10226 #[tokio::test]
10227 async fn route_open_inactive_registration_remains_target_unavailable() {
10228 let registry = Arc::new(Registry::default());
10229 let handler = ControlHandler::new(Arc::clone(®istry));
10230 handler
10231 .handle_control(
10232 ConnectionId::new(43),
10233 hello_frame("inactive", PROTOCOL_VERSION, 308),
10234 )
10235 .unwrap();
10236 assert!(registry
10237 .set_module_state_for_test("inactive", ChannelState::Closed)
10238 .unwrap());
10239
10240 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
10241 let response = handler
10242 .handle_control_frame(
10243 &ctx,
10244 route_open_frame(309, "inactive", unique_project_root("inactive")),
10245 )
10246 .await
10247 .unwrap();
10248
10249 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10250 assert!(parse_error(&response[0])["message"]
10251 .as_str()
10252 .unwrap()
10253 .contains("is not active"));
10254 }
10255
10256 #[tokio::test]
10257 async fn late_health_reply_is_recorded_through_the_module_response_path() {
10258 let registry = Arc::new(Registry::default());
10259 let forwarding = Arc::new(ForwardingTable::default());
10260 let supervisor_handle = SupervisorHandle::new();
10261 let supervisor = Supervisor::new(Arc::clone(®istry), crate::RestartPolicy::default())
10262 .with_forwarding(Arc::clone(&forwarding))
10263 .with_handle(supervisor_handle.clone());
10264 let module = supervisor
10265 .supervise_configured(
10266 crate::ModuleSpec {
10267 module_id: "late-health-response".to_string(),
10268 program: PathBuf::from("disabled-module"),
10269 args: Vec::new(),
10270 env: Vec::new(),
10271 reserved: false,
10272 reserved_prefixes: Vec::new(),
10273 protocol: ModuleProtocol::Subc,
10274 overlap: Default::default(),
10275 },
10276 false,
10277 )
10278 .unwrap();
10279 let handler =
10280 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10281 .with_supervisor(supervisor_handle);
10282 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
10283 handler
10284 .handle_control_frame(
10285 &module_ctx,
10286 hello_frame_with_control_ops(
10287 "late-health-response",
10288 PROTOCOL_VERSION,
10289 7,
10290 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10291 ),
10292 )
10293 .await
10294 .unwrap();
10295 let probe_started_at = Instant::now() - Duration::from_millis(80);
10296 let pending = forwarding
10297 .begin_health_probe_rpc_for(
10298 "late-health-response",
10299 MODULE_CONTROL_OP_HEALTH_CHECK,
10300 probe_started_at,
10301 Instant::now() - Duration::from_millis(1),
10302 )
10303 .unwrap();
10304 assert!(forwarding
10305 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
10306 .unwrap());
10307
10308 let responses = handler
10309 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
10310 .await
10311 .unwrap();
10312
10313 assert!(responses.is_empty());
10314 let health = module.status().unwrap().health;
10315 assert_eq!(health.late_answer_count, 1);
10316 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
10317 }
10318
10319 #[tokio::test]
10320 async fn health_probe_timeout_and_module_death_are_typed() {
10321 let registry = Arc::new(Registry::default());
10322 let forwarding = Arc::new(ForwardingTable::default());
10323 let handler =
10324 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10325 .with_health_probe_timeout(Duration::from_millis(50));
10326 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
10327 hello_via_sink(
10328 &handler,
10329 &module_ctx,
10330 &mut module_rx,
10331 hello_frame_with_control_ops(
10332 "aft",
10333 PROTOCOL_VERSION,
10334 7,
10335 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10336 ),
10337 )
10338 .await;
10339
10340 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10341 let responses = handler
10342 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
10343 .await
10344 .unwrap();
10345 assert_eq!(responses[0].header.ty, FrameType::Error);
10346 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
10347 let _ = module_rx.try_recv();
10348
10349 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
10350 let health_handler = handler.clone();
10351 let death_task = tokio::spawn(async move {
10352 health_handler
10353 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
10354 .await
10355 .unwrap()
10356 });
10357 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10358 .await
10359 .unwrap()
10360 .unwrap();
10361 handler
10362 .cleanup_connection(module_ctx.connection_id)
10363 .unwrap();
10364 let responses = death_task.await.unwrap();
10365 assert_eq!(responses[0].header.ty, FrameType::Error);
10366 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
10367 }
10368
10369 #[test]
10370 fn hello_requires_exact_protocol_version() {
10371 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
10372 let registry = Arc::new(Registry::default());
10373 let handler = ControlHandler::new(Arc::clone(®istry));
10374 let responses = handler
10375 .handle_control(
10376 ConnectionId::new(connection),
10377 hello_frame("aft", offered, 9),
10378 )
10379 .unwrap();
10380
10381 assert_eq!(responses.len(), 1);
10382 assert_eq!(responses[0].header.ty, FrameType::Error);
10383 let error = parse_error(&responses[0]);
10384 assert_eq!(error["code"], "version_unsupported");
10385 assert!(registry.get_module("aft").unwrap().is_none());
10386 assert_eq!(registry.active_registration_count().unwrap(), 0);
10387 }
10388 }
10389
10390 #[test]
10391 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
10392 let registry = Arc::new(Registry::default());
10393 let forwarding = Arc::new(ForwardingTable::default());
10394 let handler =
10395 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10396 let module_connection = ConnectionId::new(301);
10397 let registration = registry
10398 .register_with_control_ops(
10399 manifest("aft-push", PROTOCOL_VERSION),
10400 PROTOCOL_VERSION,
10401 module_connection,
10402 module_baseline_control_ops(),
10403 )
10404 .unwrap();
10405 let (module_tx, _module_rx) = mpsc::channel(8);
10406 let endpoint = forwarding
10407 .register_module_connection(
10408 module_connection,
10409 "aft-push".to_string(),
10410 PROTOCOL_VERSION,
10411 manifest_concurrency(®istration.manifest),
10412 FrameSink::new(module_tx),
10413 )
10414 .unwrap();
10415
10416 let unknown = Frame::build(
10418 FrameType::Push,
10419 control_flags(),
10420 0,
10421 0,
10422 5,
10423 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
10424 )
10425 .unwrap();
10426 let out = handler.handle_status_update(endpoint, unknown).unwrap();
10427 assert!(
10428 out.is_empty(),
10429 "unknown push op must be ignored, got {out:?}"
10430 );
10431
10432 let malformed = Frame::build(
10434 FrameType::Push,
10435 control_flags(),
10436 0,
10437 0,
10438 6,
10439 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
10440 )
10441 .unwrap();
10442 let out = handler.handle_status_update(endpoint, malformed).unwrap();
10443 assert_eq!(out.len(), 1);
10444 assert_eq!(out[0].header.ty, FrameType::Error);
10445 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
10446 }
10447
10448 #[test]
10449 fn hello_rejected_when_connection_already_owns_client_routes() {
10450 let registry = Arc::new(Registry::default());
10451 let forwarding = Arc::new(ForwardingTable::default());
10452 let handler =
10453 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10454 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
10456 let client_connection = ConnectionId::new(202);
10457
10458 let responses = handler
10461 .handle_control(
10462 client_connection,
10463 hello_frame("aft-second", PROTOCOL_VERSION, 9),
10464 )
10465 .unwrap();
10466 assert_eq!(responses[0].header.ty, FrameType::Error);
10467 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
10468 assert!(registry.get_module("aft-second").unwrap().is_none());
10469 }
10470
10471 #[tokio::test]
10472 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
10473 let registry = Arc::new(Registry::default());
10474 let forwarding = Arc::new(ForwardingTable::default());
10475 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
10476 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
10477 hello_via_sink(
10478 &handler,
10479 &module_ctx,
10480 &mut module_rx,
10481 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
10482 )
10483 .await;
10484 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
10485 let pending = forwarding
10486 .begin_route_bind_relay_for_test(
10487 client_ctx.connection_id,
10488 client_ctx.egress.clone(),
10489 2,
10490 "alpha",
10491 )
10492 .unwrap();
10493 forwarding
10494 .complete_pending_relay(
10495 module_ctx.connection_id,
10496 pending.corr,
10497 RouteBindRelayOutcome::Accepted,
10498 )
10499 .unwrap();
10500 client_rx.try_recv().unwrap();
10501 for module_id in ["beta", "alpha"] {
10502 let replies = handler
10503 .handle_control_frame(
10504 &module_ctx,
10505 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
10506 )
10507 .await
10508 .unwrap();
10509 assert_eq!(replies.len(), 1, "second HELLO must be refused");
10510 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
10511 }
10512 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
10513 assert!(registry.get_module("beta").unwrap().is_none());
10514 assert!(matches!(
10515 forwarding
10516 .lookup_data_route(
10517 client_ctx.connection_id,
10518 pending.client_channel,
10519 pending.client_epoch,
10520 )
10521 .unwrap(),
10522 DataRoute::Client(DataRouteState::Bound(_))
10523 ));
10524 assert!(handler
10525 .hello_launch_nonces
10526 .lock()
10527 .unwrap()
10528 .presented(module_ctx.connection_id, Some("alpha-nonce")));
10529 assert!(module_rx.try_recv().is_err());
10530 }
10531
10532 #[test]
10533 fn reserved_module_hello_requires_matching_launch_nonce() {
10534 let registry = Arc::new(Registry::default());
10535 let supervisor = SupervisorHandle::new();
10536 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
10539 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
10540
10541 let no_nonce = handler
10543 .handle_control(
10544 ConnectionId::new(1),
10545 hello_frame("vault", PROTOCOL_VERSION, 1),
10546 )
10547 .unwrap();
10548 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
10549 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
10550 assert!(registry.get_module("vault").unwrap().is_none());
10551
10552 let wrong = handler
10554 .handle_control(
10555 ConnectionId::new(2),
10556 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
10557 )
10558 .unwrap();
10559 assert_eq!(wrong[0].header.ty, FrameType::Error);
10560 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
10561 assert!(registry.get_module("vault").unwrap().is_none());
10562
10563 let ok = handler
10565 .handle_control(
10566 ConnectionId::new(3),
10567 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
10568 )
10569 .unwrap();
10570 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
10571 assert!(registry.get_module("vault").unwrap().is_some());
10572 }
10573
10574 #[test]
10575 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
10576 let registry = Arc::new(Registry::default());
10577 let supervisor = SupervisorHandle::new();
10578 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
10579 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
10580 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
10581
10582 let squat = handler
10583 .handle_control(
10584 ConnectionId::new(1),
10585 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
10586 )
10587 .unwrap();
10588 assert_eq!(squat[0].header.ty, FrameType::Error);
10589 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
10590 assert!(parse_error(&squat[0])["message"]
10591 .as_str()
10592 .unwrap()
10593 .contains("fed:"));
10594
10595 let accepted_peer = handler
10596 .handle_control(
10597 ConnectionId::new(2),
10598 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
10599 )
10600 .unwrap();
10601 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
10602
10603 let accepted_short = handler
10604 .handle_control(
10605 ConnectionId::new(3),
10606 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
10607 )
10608 .unwrap();
10609 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
10610
10611 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
10612 let response = handler
10613 .handle_control(
10614 ConnectionId::new(conn),
10615 hello_frame(module_id, PROTOCOL_VERSION, conn),
10616 )
10617 .unwrap();
10618 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
10619 }
10620 }
10621
10622 #[test]
10623 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
10624 let registry = Arc::new(Registry::default());
10625 let supervisor = SupervisorHandle::new();
10626 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
10627 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
10628 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
10629 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
10630
10631 let owner_nonce = handler
10632 .handle_control(
10633 ConnectionId::new(1),
10634 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
10635 )
10636 .unwrap();
10637 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
10638 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
10639 assert!(registry.get_module("fed:special").unwrap().is_none());
10640
10641 let exact_nonce = handler
10642 .handle_control(
10643 ConnectionId::new(2),
10644 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
10645 )
10646 .unwrap();
10647 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
10648 assert!(registry.get_module("fed:special").unwrap().is_some());
10649 }
10650
10651 #[test]
10652 fn non_reserved_module_ignores_launch_nonce() {
10653 let registry = Arc::new(Registry::default());
10654 let handler = ControlHandler::new(Arc::clone(®istry));
10657 let no_nonce = handler
10658 .handle_control(
10659 ConnectionId::new(1),
10660 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
10661 )
10662 .unwrap();
10663 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
10664 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
10665
10666 let echoed_nonce = handler
10667 .handle_control(
10668 ConnectionId::new(2),
10669 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
10670 )
10671 .unwrap();
10672 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
10673 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
10674 }
10675
10676 #[test]
10677 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
10678 let handler = ControlHandler::default();
10679 let conn = ConnectionId::new(1);
10680 let malformed = Frame::build(
10681 FrameType::Hello,
10682 control_flags(),
10683 0,
10684 0,
10685 3,
10686 b"{not json".to_vec(),
10687 )
10688 .unwrap();
10689
10690 let error = handler.handle_control(conn, malformed).unwrap();
10691 assert_eq!(error[0].header.ty, FrameType::Error);
10692 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
10693
10694 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
10695 let pong = handler.handle_control(conn, ping).unwrap();
10696 assert_eq!(pong[0].header.ty, FrameType::Pong);
10697 assert_eq!(pong[0].header.corr, 4);
10698 }
10699
10700 #[test]
10701 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
10702 let registry = Arc::new(Registry::default());
10703 let handler = ControlHandler::new(Arc::clone(®istry));
10704
10705 handler
10706 .handle_control(
10707 ConnectionId::new(1),
10708 hello_frame("aft", PROTOCOL_VERSION, 1),
10709 )
10710 .unwrap();
10711 let duplicate = handler
10712 .handle_control(
10713 ConnectionId::new(2),
10714 hello_frame("aft", PROTOCOL_VERSION, 2),
10715 )
10716 .unwrap();
10717
10718 assert_eq!(duplicate[0].header.ty, FrameType::Error);
10719 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
10720 let registration = registry.get_module("aft").unwrap().unwrap();
10721 assert_eq!(registration.connection_id, ConnectionId::new(1));
10722 }
10723
10724 #[test]
10725 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
10726 let registry = Arc::new(Registry::default());
10727 let forwarding = Arc::new(ForwardingTable::default());
10728 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
10729 let handler =
10730 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10731 .with_process_liveness(process_liveness);
10732 let (ctx, route_channel, route_epoch) =
10733 bind_liveness_route(®istry, &forwarding, "aft-dead");
10734 let responses = handler
10735 .handle_route_poll(
10736 &ctx,
10737 route_poll_frame(41, PollKind::Liveness, route_channel),
10738 route_channel,
10739 route_epoch,
10740 PollKind::Liveness,
10741 )
10742 .unwrap();
10743
10744 assert_eq!(responses.len(), 1);
10745 assert_eq!(responses[0].header.ty, FrameType::Response);
10746 assert_route_poll_liveness(&responses[0], false);
10747 }
10748
10749 #[test]
10750 fn liveness_poll_without_process_source_uses_bound_route() {
10751 let registry = Arc::new(Registry::default());
10752 let forwarding = Arc::new(ForwardingTable::default());
10753 let handler =
10754 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10755 let (ctx, route_channel, route_epoch) =
10756 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
10757 let responses = handler
10758 .handle_route_poll(
10759 &ctx,
10760 route_poll_frame(42, PollKind::Liveness, route_channel),
10761 route_channel,
10762 route_epoch,
10763 PollKind::Liveness,
10764 )
10765 .unwrap();
10766
10767 assert_route_poll_liveness(&responses[0], true);
10768 }
10769
10770 #[test]
10771 fn liveness_poll_untracked_process_source_uses_bound_route() {
10772 let registry = Arc::new(Registry::default());
10773 let forwarding = Arc::new(ForwardingTable::default());
10774 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
10775 let handler =
10776 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10777 .with_process_liveness(process_liveness);
10778 let (ctx, route_channel, route_epoch) =
10779 bind_liveness_route(®istry, &forwarding, "aft-untracked");
10780 let responses = handler
10781 .handle_route_poll(
10782 &ctx,
10783 route_poll_frame(43, PollKind::Liveness, route_channel),
10784 route_channel,
10785 route_epoch,
10786 PollKind::Liveness,
10787 )
10788 .unwrap();
10789
10790 assert_route_poll_liveness(&responses[0], true);
10791 }
10792
10793 #[tokio::test]
10794 async fn unknown_op_returns_unknown_control_op() {
10795 let handler = ControlHandler::default();
10796 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
10797 let request = Frame::build(
10798 FrameType::Request,
10799 control_flags(),
10800 0,
10801 0,
10802 55,
10803 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
10804 )
10805 .unwrap();
10806
10807 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
10808
10809 assert_eq!(response.len(), 1);
10810 assert_eq!(response[0].header.ty, FrameType::Error);
10811 assert_eq!(response[0].header.corr, 55);
10812 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
10813 }
10814
10815 #[tokio::test]
10816 async fn supervisor_provenance_rejects_unknown_exact_module() {
10817 let handler = ControlHandler::default();
10818 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
10819 let request = Frame::build(
10820 FrameType::Request,
10821 control_flags(),
10822 0,
10823 0,
10824 57,
10825 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
10826 )
10827 .unwrap();
10828
10829 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
10830
10831 assert_eq!(response.len(), 1);
10832 assert_eq!(response[0].header.ty, FrameType::Error);
10833 assert_eq!(response[0].header.corr, 57);
10834 let error = parse_error(&response[0]);
10835 assert_eq!(error["code"], "unknown_module");
10836 assert_eq!(error["message"], "module_id 'missing' is not supervised");
10837 }
10838
10839 #[test]
10840 fn provenance_probe_override_keeps_handler_tests_deterministic() {
10841 let expected = subc_control::RunningImageAgreement::Unavailable {
10842 reason: subc_control::RunningImageUnavailableReason::HashFailed,
10843 };
10844 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
10845 assert_eq!(handler.provenance_probe_override, Some(expected));
10846 }
10847
10848 #[test]
10849 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
10850 let verdict = reload_verdict(
10851 std::path::Path::new("/bin/new"),
10852 Some(std::path::Path::new("/bin/old")),
10853 subc_control::RunningImageAgreement::Unavailable {
10854 reason: subc_control::RunningImageUnavailableReason::HashFailed,
10855 },
10856 );
10857 assert!(matches!(
10858 verdict.path,
10859 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
10860 if configured == std::path::Path::new("/bin/new")
10861 && spawned_from == std::path::Path::new("/bin/old")
10862 ));
10863 }
10864
10865 #[test]
10866 fn reload_verdict_detects_replaced_image_at_same_path() {
10867 let image = subc_control::RunningImageAgreement::Mismatch {
10868 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
10869 digest: "old".into(),
10870 },
10871 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
10872 digest: "new".into(),
10873 },
10874 };
10875 let verdict = reload_verdict(
10876 std::path::Path::new("/bin/same"),
10877 Some(std::path::Path::new("/bin/same")),
10878 image.clone(),
10879 );
10880 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
10881 assert_eq!(verdict.image, image);
10882 }
10883
10884 #[test]
10885 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
10886 let image = subc_control::RunningImageAgreement::Unavailable {
10887 reason: subc_control::RunningImageUnavailableReason::NotRunning,
10888 };
10889 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
10890 assert_eq!(
10891 verdict.path,
10892 subc_control::ReloadPathAgreement::Unavailable {
10893 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
10894 }
10895 );
10896 assert_eq!(verdict.image, image);
10897
10898 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
10899 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
10900 };
10901 let verdict = reload_verdict(
10902 std::path::Path::new("/bin/same"),
10903 Some(std::path::Path::new("/bin/same")),
10904 unconfirmed.clone(),
10905 );
10906 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
10907 assert_eq!(verdict.image, unconfirmed);
10908 }
10909
10910 #[test]
10911 fn reload_verdict_preserves_each_image_unavailability_reason() {
10912 use subc_control::RunningImageUnavailableReason as Reason;
10913
10914 for reason in [
10915 Reason::NotRunning,
10916 Reason::UnsupportedPlatform,
10917 Reason::RunningExecutableUnreadable,
10918 Reason::SpawnedPathUnreadable,
10919 Reason::HashFailed,
10920 Reason::ProcessIdentityUnconfirmed,
10921 Reason::Unknown("future_probe_reason".to_string()),
10922 ] {
10923 let image = subc_control::RunningImageAgreement::Unavailable {
10924 reason: reason.clone(),
10925 };
10926 let verdict = reload_verdict(
10927 std::path::Path::new("/bin/same"),
10928 Some(std::path::Path::new("/bin/same")),
10929 image.clone(),
10930 );
10931 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
10932 assert_eq!(verdict.image, image, "{reason:?}");
10933 }
10934 }
10935
10936 #[tokio::test]
10937 async fn malformed_control_bodies_return_invalid_control_body() {
10938 let handler = ControlHandler::default();
10939 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
10940
10941 for (corr, body) in [
10942 (56, br#"{"route_channel":1}"#.as_slice()),
10943 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
10944 (
10945 58,
10946 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
10947 ),
10948 ] {
10949 let request = Frame::build(
10950 FrameType::Request,
10951 control_flags(),
10952 0,
10953 0,
10954 corr,
10955 body.to_vec(),
10956 )
10957 .unwrap();
10958 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
10959
10960 assert_eq!(response.len(), 1);
10961 assert_eq!(response[0].header.ty, FrameType::Error);
10962 assert_eq!(response[0].header.corr, corr);
10963 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
10964 }
10965 }
10966
10967 #[tokio::test]
10968 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
10969 let registry = Arc::new(Registry::default());
10970 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
10971 let router = Router::with_control_handler(Arc::clone(&control));
10972 let connection = router.begin_connection();
10973 let (ctx, mut rx) = route_ctx(connection.id());
10974
10975 router
10976 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
10977 .await
10978 .unwrap();
10979 let response = rx.recv().await.unwrap();
10980 let ack = parse_ack(&response);
10981 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
10982 let channel = 1;
10983
10984 let goodbye =
10985 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
10986 router.route_for_connection(&ctx, goodbye).await.unwrap();
10987 assert!(rx.try_recv().is_err());
10988 assert!(registry.get_module("aft").unwrap().is_none());
10989
10990 router
10991 .route_for_connection(&ctx, channel_request(channel, 13))
10992 .await
10993 .unwrap();
10994 let error_frame = rx.recv().await.unwrap();
10995 assert_eq!(error_frame.header.ty, FrameType::Error);
10996 assert_eq!(error_frame.header.channel, channel);
10997 }
10998
10999 #[tokio::test]
11000 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
11001 let registry = Arc::new(Registry::default());
11002 let handler = ControlHandler::new(registry).with_capability_config(
11003 [("prov".to_string(), true), ("cons".to_string(), true)],
11004 BTreeMap::new(),
11005 );
11006 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
11007 register_capability_manifest(
11008 &handler,
11009 &provider_ctx,
11010 &mut provider_rx,
11011 capability_manifest("prov", &["thing/v1"], &[]),
11012 1,
11013 )
11014 .await;
11015 let mut consumer = capability_manifest("cons", &[], &[]);
11016 consumer.capabilities.as_mut().unwrap().requires.push(
11017 subc_protocol::manifest::CapabilityRequirement {
11018 capability: "thing/v1".to_string(),
11019 need: subc_protocol::manifest::CapabilityNeed::Required,
11020 },
11021 );
11022 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
11023 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
11024 assert_eq!(
11025 handler.capability_evaluator.verdict("cons", "thing/v1"),
11026 Some(CapabilityVerdict::Provided)
11027 );
11028 let (mut client_rx, _) = open_route_for_capability_test(
11029 &handler,
11030 &provider_ctx,
11031 &mut provider_rx,
11032 703,
11033 3,
11034 "prov",
11035 None,
11036 )
11037 .await;
11038 let goodbye =
11039 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
11040 handler
11041 .handle_control_frame(&provider_ctx, goodbye)
11042 .await
11043 .unwrap();
11044 assert_eq!(
11045 handler.capability_evaluator.verdict("cons", "thing/v1"),
11046 Some(CapabilityVerdict::NeverProvided)
11047 );
11048 let closed = client_rx
11049 .try_recv()
11050 .expect("GOODBYE pushes route.closed before route GOODBYE");
11051 assert!(
11052 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
11053 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
11054 );
11055 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
11056 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
11057 }
11058
11059 #[tokio::test]
11060 async fn dropping_router_connection_releases_registration() {
11061 let registry = Arc::new(Registry::default());
11062 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
11063 let router = Router::with_control_handler(control);
11064 let connection = router.begin_connection();
11065 let (ctx, mut rx) = route_ctx(connection.id());
11066
11067 router
11068 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
11069 .await
11070 .unwrap();
11071 let response = rx.recv().await.unwrap();
11072 let ack = parse_ack(&response);
11073 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
11074 assert!(registry.get_module("aft").unwrap().is_some());
11075
11076 drop(connection);
11077
11078 assert!(registry.get_module("aft").unwrap().is_none());
11079 assert_eq!(registry.active_registration_count().unwrap(), 0);
11080 }
11081
11082 fn capability_manifest(
11083 module_id: &str,
11084 provides: &[&str],
11085 must_never_reach: &[&str],
11086 ) -> ModuleManifest {
11087 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
11088 manifest.capabilities = Some(CapabilityDeclarations {
11089 provides: provides
11090 .iter()
11091 .map(|capability| (*capability).to_string())
11092 .collect(),
11093 requires: Vec::new(),
11094 must_never_reach: must_never_reach
11095 .iter()
11096 .map(|capability| (*capability).to_string())
11097 .collect(),
11098 });
11099 manifest
11100 }
11101
11102 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
11103 Frame::build(
11104 FrameType::Hello,
11105 control_flags(),
11106 0,
11107 0,
11108 corr,
11109 serde_json::to_vec(&ModuleHelloBody {
11110 protocol_ver: manifest.protocol_ver,
11111 manifest,
11112 control_ops: None,
11113 launch_nonce: None,
11114 })
11115 .expect("capability test HELLO serializes"),
11116 )
11117 .expect("capability test HELLO frame builds")
11118 }
11119
11120 fn catalog_update_with_capabilities_frame(
11121 corr: u64,
11122 capabilities: CapabilityDeclarations,
11123 ) -> Frame {
11124 Frame::build(
11125 FrameType::Request,
11126 control_flags(),
11127 0,
11128 0,
11129 corr,
11130 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
11131 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
11132 capabilities: Some(capabilities),
11133 ready: None,
11134 })
11135 .expect("capability catalog.update serializes"),
11136 )
11137 .expect("capability catalog.update frame builds")
11138 }
11139
11140 async fn register_capability_manifest(
11141 handler: &ControlHandler,
11142 ctx: &RouteCtx,
11143 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
11144 manifest: ModuleManifest,
11145 corr: u64,
11146 ) {
11147 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
11148 }
11149
11150 async fn open_route_for_capability_test(
11151 handler: &ControlHandler,
11152 target_ctx: &RouteCtx,
11153 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
11154 client_connection_id: u64,
11155 corr: u64,
11156 target_module_id: &str,
11157 consumer_identity: Option<ConsumerIdentity>,
11158 ) -> (
11159 mpsc::Receiver<crate::router::OutboundFrame>,
11160 ModuleControlRequest,
11161 ) {
11162 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
11163 let route_handler = handler.clone();
11164 let target_module_id = target_module_id.to_string();
11165 let route_task = tokio::spawn(async move {
11166 route_handler
11167 .handle_control_frame(
11168 &client_ctx,
11169 route_open_frame_with_admission_facts(
11170 corr,
11171 &target_module_id,
11172 unique_project_root("admission-facts"),
11173 consumer_identity,
11174 None,
11175 ),
11176 )
11177 .await
11178 .expect("capability test route.open succeeds")
11179 });
11180 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
11181 .await
11182 .expect("capability test route.open must reach route.bind")
11183 .expect("target control receiver stays open");
11184 let bind_request: ModuleControlRequest =
11185 serde_json::from_slice(&bind.body).expect("route.bind decodes");
11186 handler
11187 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
11188 .await
11189 .expect("capability test route.bind ACK succeeds");
11190 assert!(route_task.await.expect("route.open task joins").is_empty());
11191 let opened = client_rx
11192 .recv()
11193 .await
11194 .expect("successful route.open publishes a response");
11195 assert!(matches!(
11196 serde_json::from_slice::<ClientControlResponse>(&opened.body),
11197 Ok(ClientControlResponse::RouteOpen { .. })
11198 ));
11199 (client_rx, bind_request)
11200 }
11201
11202 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
11203 assert_eq!(frame.header.ty, FrameType::Push);
11204 assert_eq!(frame.header.channel, 0);
11205 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
11206 .expect("route.closed control push decodes");
11207 let ClientControlPush::RouteClosed { channels, .. } = &push else {
11208 panic!("expected route.closed");
11209 };
11210 assert_eq!(channels.len(), 1, "exactly one violating route closed");
11211 let channels = channels.clone();
11212 assert_eq!(
11213 push,
11214 ClientControlPush::RouteClosed {
11215 module_id: target_module_id.to_string(),
11216 channels,
11217 reason: RouteCloseReason::CapabilityDenied,
11218 drained: false,
11219 abandoned: 0,
11220 excluded_subscriptions: 0,
11221 terminal: Some(false),
11222 }
11223 );
11224 }
11225
11226 #[tokio::test]
11227 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
11228 let registry = Arc::new(Registry::default());
11229 let forwarding = Arc::new(ForwardingTable::default());
11230 let supervisor = SupervisorHandle::new();
11231 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11232 let handler =
11233 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11234 .with_supervisor(supervisor);
11235 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
11236 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
11237 register_capability_manifest(
11238 &handler,
11239 &target_ctx,
11240 &mut target_rx,
11241 capability_manifest("target", &["credentials-provider/v1"], &[]),
11242 1,
11243 )
11244 .await;
11245 register_capability_manifest(
11246 &handler,
11247 &opener_ctx,
11248 &mut opener_rx,
11249 capability_manifest("opener", &[], &["credentials-provider/v1"]),
11250 2,
11251 )
11252 .await;
11253
11254 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
11255 let replies = handler
11256 .handle_control_frame(
11257 &client_ctx,
11258 route_open_frame_with_admission_facts(
11259 3,
11260 "target",
11261 unique_project_root("admission-facts"),
11262 Some(ConsumerIdentity {
11263 module_id: "opener".to_string(),
11264 launch_nonce: "opener-nonce".to_string(),
11265 }),
11266 None,
11267 ),
11268 )
11269 .await
11270 .expect("denied route.open returns a typed frame");
11271 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
11272 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
11273 assert!(
11274 target_rx.try_recv().is_err(),
11275 "forbidden route.open must not relay route.bind"
11276 );
11277 }
11278
11279 #[tokio::test]
11280 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
11281 let registry = Arc::new(Registry::default());
11282 let forwarding = Arc::new(ForwardingTable::default());
11283 let supervisor = SupervisorHandle::new();
11284 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11285 let handler =
11286 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11287 .with_supervisor(supervisor);
11288 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
11289 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
11290 register_capability_manifest(
11291 &handler,
11292 &target_ctx,
11293 &mut target_rx,
11294 capability_manifest("target", &["credentials-provider/v1"], &[]),
11295 1,
11296 )
11297 .await;
11298 register_capability_manifest(
11299 &handler,
11300 &old_opener_ctx,
11301 &mut old_opener_rx,
11302 capability_manifest("opener", &[], &[]),
11303 2,
11304 )
11305 .await;
11306 let (mut client_rx, _) = open_route_for_capability_test(
11307 &handler,
11308 &target_ctx,
11309 &mut target_rx,
11310 712,
11311 3,
11312 "target",
11313 Some(ConsumerIdentity {
11314 module_id: "opener".to_string(),
11315 launch_nonce: "opener-nonce".to_string(),
11316 }),
11317 )
11318 .await;
11319 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
11320
11321 handler
11322 .cleanup_connection(old_opener_ctx.connection_id)
11323 .expect("old opener registration cleans up");
11324 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
11325 register_capability_manifest(
11326 &handler,
11327 &new_opener_ctx,
11328 &mut new_opener_rx,
11329 capability_manifest("opener", &[], &["credentials-provider/v1"]),
11330 4,
11331 )
11332 .await;
11333
11334 assert_capability_denied_push(
11335 client_rx
11336 .try_recv()
11337 .expect("HELLO deny addition must emit route.closed")
11338 .frame,
11339 "target",
11340 );
11341 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
11342 assert!(matches!(
11343 target_rx.try_recv(),
11344 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
11345 ));
11346 }
11347
11348 #[tokio::test]
11349 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
11350 let registry = Arc::new(Registry::default());
11351 let forwarding = Arc::new(ForwardingTable::default());
11352 let supervisor = SupervisorHandle::new();
11353 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11354 let handler =
11355 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11356 .with_supervisor(supervisor);
11357 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
11358 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
11359 register_capability_manifest(
11360 &handler,
11361 &target_ctx,
11362 &mut target_rx,
11363 capability_manifest("target", &[], &[]),
11364 1,
11365 )
11366 .await;
11367 register_capability_manifest(
11368 &handler,
11369 &opener_ctx,
11370 &mut opener_rx,
11371 capability_manifest("opener", &[], &["credentials-provider/v1"]),
11372 2,
11373 )
11374 .await;
11375 let (mut client_rx, _) = open_route_for_capability_test(
11376 &handler,
11377 &target_ctx,
11378 &mut target_rx,
11379 722,
11380 3,
11381 "target",
11382 Some(ConsumerIdentity {
11383 module_id: "opener".to_string(),
11384 launch_nonce: "opener-nonce".to_string(),
11385 }),
11386 )
11387 .await;
11388 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
11389
11390 let replies = handler
11391 .handle_control_frame(
11392 &target_ctx,
11393 catalog_update_with_capabilities_frame(
11394 4,
11395 CapabilityDeclarations {
11396 provides: vec!["credentials-provider/v1".to_string()],
11397 requires: Vec::new(),
11398 must_never_reach: Vec::new(),
11399 },
11400 ),
11401 )
11402 .await
11403 .expect("claim catalog.update succeeds");
11404 assert!(matches!(
11405 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
11406 Ok(ModuleControlResponseToModule::CatalogUpdate {})
11407 ));
11408 assert_capability_denied_push(
11409 client_rx
11410 .try_recv()
11411 .expect("claim addition must emit route.closed")
11412 .frame,
11413 "target",
11414 );
11415 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
11416 assert!(matches!(
11417 target_rx.try_recv(),
11418 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
11419 ));
11420 }
11421
11422 #[tokio::test]
11423 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
11424 let registry = Arc::new(Registry::default());
11425 let forwarding = Arc::new(ForwardingTable::default());
11426 let supervisor = SupervisorHandle::new();
11427 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11428 let handler =
11429 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11430 .with_supervisor(supervisor);
11431 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
11432 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
11433 register_capability_manifest(
11434 &handler,
11435 &target_ctx,
11436 &mut target_rx,
11437 capability_manifest("target", &["credentials-provider/v1"], &[]),
11438 1,
11439 )
11440 .await;
11441 register_capability_manifest(
11442 &handler,
11443 &opener_ctx,
11444 &mut opener_rx,
11445 capability_manifest("opener", &[], &[]),
11446 2,
11447 )
11448 .await;
11449 let (mut client_rx, _) = open_route_for_capability_test(
11450 &handler,
11451 &target_ctx,
11452 &mut target_rx,
11453 732,
11454 3,
11455 "target",
11456 Some(ConsumerIdentity {
11457 module_id: "opener".to_string(),
11458 launch_nonce: "opener-nonce".to_string(),
11459 }),
11460 )
11461 .await;
11462 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
11463
11464 handler
11465 .handle_control_frame(
11466 &target_ctx,
11467 catalog_update_with_capabilities_frame(
11468 4,
11469 CapabilityDeclarations {
11470 provides: Vec::new(),
11471 requires: Vec::new(),
11472 must_never_reach: Vec::new(),
11473 },
11474 ),
11475 )
11476 .await
11477 .expect("claim removal catalog.update succeeds");
11478 assert_eq!(
11479 forwarding.active_binding_count().unwrap(),
11480 1,
11481 "removing an attested target claim must leave the route census unchanged"
11482 );
11483 assert!(
11484 client_rx.try_recv().is_err(),
11485 "claim removal must not emit route.closed capability_denied"
11486 );
11487 assert!(
11488 target_rx.try_recv().is_err(),
11489 "claim removal must not send the target a route GOODBYE"
11490 );
11491 }
11492
11493 #[tokio::test]
11496 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
11497 let registry = Arc::new(Registry::default());
11498 let forwarding = Arc::new(ForwardingTable::default());
11499 let supervisor = SupervisorHandle::new();
11500 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11501 let handler =
11502 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11503 .with_supervisor(supervisor);
11504 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
11505 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
11506 register_capability_manifest(
11507 &handler,
11508 &target_ctx,
11509 &mut target_rx,
11510 capability_manifest("target", &["credentials-provider/v1"], &[]),
11511 1,
11512 )
11513 .await;
11514 register_capability_manifest(
11515 &handler,
11516 &opener_ctx,
11517 &mut opener_rx,
11518 capability_manifest("opener", &[], &["credentials-provider/v1"]),
11519 2,
11520 )
11521 .await;
11522
11523 let (_client_rx, bind) = open_route_for_capability_test(
11524 &handler,
11525 &target_ctx,
11526 &mut target_rx,
11527 742,
11528 3,
11529 "target",
11530 None,
11531 )
11532 .await;
11533 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
11534 panic!("direct scope-honesty route must bind");
11535 };
11536 assert_eq!(principal, Some(Principal::Direct));
11537 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
11538 }
11539
11540 #[tokio::test]
11543 async fn must_never_reach_self_route_is_capability_forbidden() {
11544 let registry = Arc::new(Registry::default());
11545 let forwarding = Arc::new(ForwardingTable::default());
11546 let supervisor = SupervisorHandle::new();
11547 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
11548 let handler =
11549 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11550 .with_supervisor(supervisor);
11551 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
11552 register_capability_manifest(
11553 &handler,
11554 &self_ctx,
11555 &mut self_rx,
11556 capability_manifest(
11557 "self-provider",
11558 &["credentials-provider/v1"],
11559 &["credentials-provider/v1"],
11560 ),
11561 1,
11562 )
11563 .await;
11564
11565 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
11566 let replies = handler
11567 .handle_control_frame(
11568 &client_ctx,
11569 route_open_frame_with_admission_facts(
11570 2,
11571 "self-provider",
11572 unique_project_root("admission-facts"),
11573 Some(ConsumerIdentity {
11574 module_id: "self-provider".to_string(),
11575 launch_nonce: "self-nonce".to_string(),
11576 }),
11577 None,
11578 ),
11579 )
11580 .await
11581 .expect("self-route refusal returns a typed frame");
11582 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
11583 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
11584 assert!(
11585 self_rx.try_recv().is_err(),
11586 "self denial must not relay route.bind"
11587 );
11588 }
11589
11590 #[test]
11591 fn unsupported_channel_zero_frame_returns_error() {
11592 let handler = ControlHandler::default();
11593 let request = Frame::build(
11594 FrameType::Request,
11595 control_flags(),
11596 0,
11597 0,
11598 21,
11599 b"opaque".to_vec(),
11600 )
11601 .unwrap();
11602
11603 let response = handler
11604 .handle_control(ConnectionId::new(1), request)
11605 .unwrap();
11606
11607 assert_eq!(response[0].header.ty, FrameType::Error);
11608 assert_eq!(
11609 parse_error(&response[0])["code"],
11610 "unsupported_control_frame"
11611 );
11612 }
11613
11614 mod swap {
11619 use super::*;
11620
11621 const INCUMBENT: ConnectionId = ConnectionId::new(30);
11622 const CANDIDATE: ConnectionId = ConnectionId::new(40);
11623
11624 struct Swap {
11625 registry: Arc<Registry>,
11626 forwarding: Arc<ForwardingTable>,
11627 handler: ControlHandler,
11628 incumbent_ctx: RouteCtx,
11629 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
11630 candidate_ctx: RouteCtx,
11631 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
11632 }
11633
11634 async fn swap_with_incumbent() -> Swap {
11635 let registry = Arc::new(Registry::default());
11636 let forwarding = Arc::new(ForwardingTable::default());
11637 let handler =
11638 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11639 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
11640 hello_via_sink(
11641 &handler,
11642 &incumbent_ctx,
11643 &mut incumbent_rx,
11644 hello_frame("aft", PROTOCOL_VERSION, 7),
11645 )
11646 .await;
11647 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
11648 Swap {
11649 registry,
11650 forwarding,
11651 handler,
11652 incumbent_ctx,
11653 incumbent_rx,
11654 candidate_ctx,
11655 candidate_rx,
11656 }
11657 }
11658
11659 fn register_candidate(swap: &Swap, ready: Option<bool>) {
11660 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
11661 candidate_manifest.ready = ready;
11662 let registration = swap
11663 .registry
11664 .register_candidate_with_control_ops(
11665 candidate_manifest,
11666 PROTOCOL_VERSION,
11667 CANDIDATE,
11668 module_baseline_control_ops(),
11669 )
11670 .unwrap();
11671 swap.forwarding
11672 .register_candidate_module_connection(
11673 CANDIDATE,
11674 "aft".to_string(),
11675 PROTOCOL_VERSION,
11676 manifest_concurrency(®istration.manifest),
11677 swap.candidate_ctx.egress.clone(),
11678 )
11679 .unwrap();
11680 }
11681
11682 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
11683 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
11684 swap.registry.promote_candidate("aft").unwrap().unwrap();
11685 cutover.incumbent.unwrap()
11686 }
11687
11688 fn keyed_total(counters: &Value, key: &str) -> u64 {
11689 counters[key]
11690 .as_object()
11691 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
11692 .unwrap_or(0)
11693 }
11694
11695 #[tokio::test]
11701 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
11702 let mut swap = swap_with_incumbent().await;
11703 let handler = swap.handler.clone();
11704
11705 let cotenant = ConnectionId::new(31);
11707 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
11708 let (cotenant_task, cotenant_bind) = relay_route_open(
11709 &handler,
11710 cotenant,
11711 &cotenant_ctx.egress,
11712 &mut swap.incumbent_rx,
11713 100,
11714 "aft",
11715 "swap-cotenant",
11716 )
11717 .await;
11718 handler
11719 .handle_control_frame(
11720 &swap.incumbent_ctx,
11721 route_bind_ack(cotenant_bind.header.corr),
11722 )
11723 .await
11724 .unwrap();
11725 assert!(cotenant_task.await.unwrap().is_empty());
11726 let (cotenant_channel, cotenant_epoch) =
11727 published_route(&cotenant_rx.recv().await.unwrap());
11728
11729 let caller = ConnectionId::new(32);
11731 let (caller_ctx, mut caller_rx) = route_ctx(caller);
11732 let (caller_task, caller_bind) = relay_route_open(
11733 &handler,
11734 caller,
11735 &caller_ctx.egress,
11736 &mut swap.incumbent_rx,
11737 101,
11738 "aft",
11739 "swap-caller",
11740 )
11741 .await;
11742 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
11743
11744 register_candidate(&swap, None);
11745 cutover(&swap);
11746
11747 let ack = handler
11749 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
11750 .await;
11751 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
11752 if module_loop_error.is_some() {
11753 handler.cleanup_connection(INCUMBENT).unwrap();
11756 }
11757
11758 assert!(
11760 cotenant_rx.try_recv().is_err(),
11761 "the co-tenant route on the incumbent was torn down by one late ack: \
11762 {module_loop_error:?}"
11763 );
11764 assert!(matches!(
11765 swap.forwarding
11766 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
11767 .unwrap(),
11768 DataRoute::Client(DataRouteState::Bound(_))
11769 ));
11770 assert_eq!(module_loop_error, None);
11771 assert!(swap
11772 .registry
11773 .get_module_by_connection(INCUMBENT)
11774 .unwrap()
11775 .is_some());
11776
11777 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
11779 .await
11780 .expect("the incumbent is told to drop the abandoned binding")
11781 .unwrap()
11782 .frame;
11783 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11784 assert_eq!(goodbye.header.channel, abandoned_channel);
11785 assert_eq!(goodbye.header.epoch, abandoned_epoch);
11786 assert!(swap.incumbent_rx.try_recv().is_err());
11787
11788 let response = caller_task.await.unwrap();
11790 assert_eq!(response.len(), 1);
11791 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
11792 assert!(caller_rx.try_recv().is_err());
11793
11794 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
11798 let counters = handler.counters().snapshot();
11799 assert_eq!(
11800 keyed_total(&counters, "route_open_accepted_by_principal"),
11801 1
11802 );
11803 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
11804 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
11805 }
11806
11807 #[tokio::test]
11811 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
11812 let mut swap = swap_with_incumbent().await;
11813 register_candidate(&swap, None);
11814 let incumbent = cutover(&swap);
11815 swap.forwarding
11816 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
11817 .unwrap()
11818 .expect("the incumbent is still registered");
11819
11820 let client = ConnectionId::new(33);
11821 let (client_ctx, mut client_rx) = route_ctx(client);
11822 let route_handler = swap.handler.clone();
11823 let open_ctx = RouteCtx {
11824 connection_id: client,
11825 egress: client_ctx.egress.clone(),
11826 };
11827 let mut route_task = tokio::spawn(async move {
11828 route_handler
11829 .handle_control_frame(
11830 &open_ctx,
11831 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
11832 )
11833 .await
11834 .unwrap()
11835 });
11836 let bind = tokio::select! {
11837 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
11838 response = &mut route_task => {
11839 let response = response.unwrap();
11840 panic!(
11841 "post-cutover route.open was refused instead of relayed to the candidate: {}",
11842 parse_error(&response[0])["code"]
11843 );
11844 }
11845 };
11846 swap.handler
11847 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
11848 .await
11849 .unwrap();
11850 assert!(route_task.await.unwrap().is_empty());
11851 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
11852 match swap
11853 .forwarding
11854 .lookup_data_route(client, channel, epoch)
11855 .unwrap()
11856 {
11857 DataRoute::Client(DataRouteState::Bound(route)) => {
11858 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
11859 }
11860 other => panic!("expected a bound route on the candidate, got {other:?}"),
11861 }
11862 assert!(swap.incumbent_rx.try_recv().is_err());
11863 }
11864
11865 #[tokio::test]
11870 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
11871 let swap = swap_with_incumbent().await;
11872 register_candidate(&swap, Some(false));
11873 let update = Frame::build(
11874 FrameType::Request,
11875 control_flags(),
11876 0,
11877 0,
11878 55,
11879 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
11880 provides: manifest("aft", PROTOCOL_VERSION).provides,
11881 capabilities: None,
11882 ready: Some(true),
11883 })
11884 .unwrap(),
11885 )
11886 .unwrap();
11887
11888 let replies = swap
11889 .handler
11890 .handle_control_frame(&swap.candidate_ctx, update)
11891 .await
11892 .unwrap();
11893
11894 assert_eq!(replies.len(), 1);
11895 assert_eq!(
11896 replies[0].header.ty,
11897 FrameType::Response,
11898 "candidate catalog.update was refused: {:?}",
11899 serde_json::from_slice::<Value>(&replies[0].body).ok()
11900 );
11901 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
11902 assert_eq!(
11903 swap.registry
11904 .get_module("aft")
11905 .unwrap()
11906 .unwrap()
11907 .connection_id,
11908 INCUMBENT
11909 );
11910 }
11911 }
11912
11913 mod swap_admission {
11917 use super::*;
11918
11919 const INCUMBENT_NONCE: &str = "incumbent-nonce";
11920 const CANDIDATE_NONCE: &str = "candidate-nonce";
11921
11922 fn handler_with_incumbent(
11923 module_id: &str,
11924 reserved: bool,
11925 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
11926 let registry = Arc::new(Registry::default());
11927 let supervisor = SupervisorHandle::new();
11928 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
11929 if reserved {
11930 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
11931 }
11932 let handler =
11933 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
11934 let incumbent = handler
11935 .handle_control(
11936 ConnectionId::new(1),
11937 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
11938 )
11939 .unwrap();
11940 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
11941 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
11942 (registry, supervisor, handler)
11943 }
11944
11945 #[test]
11951 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
11952 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
11953
11954 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
11955 let replies = handler
11956 .handle_control(
11957 ConnectionId::new(connection),
11958 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
11959 )
11960 .unwrap();
11961 assert_eq!(replies[0].header.ty, FrameType::Error);
11962 assert_eq!(
11963 parse_error(&replies[0])["code"],
11964 "swap_token_invalid",
11965 "nonce {nonce:?}"
11966 );
11967 }
11968 assert!(registry.get_candidate("aft").unwrap().is_none());
11969 assert_eq!(
11970 registry.get_module("aft").unwrap().unwrap().connection_id,
11971 ConnectionId::new(1)
11972 );
11973
11974 let admitted = handler
11977 .handle_control(
11978 ConnectionId::new(4),
11979 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
11980 )
11981 .unwrap();
11982 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
11983 assert_eq!(
11984 registry
11985 .get_candidate("aft")
11986 .unwrap()
11987 .unwrap()
11988 .connection_id,
11989 ConnectionId::new(4)
11990 );
11991 assert_eq!(
11992 registry.get_module("aft").unwrap().unwrap().connection_id,
11993 ConnectionId::new(1),
11994 "the candidate must not take the active slot"
11995 );
11996 let replayed = handler
11997 .handle_control(
11998 ConnectionId::new(5),
11999 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
12000 )
12001 .unwrap();
12002 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
12003
12004 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
12008 let squatter = handler
12009 .handle_control(
12010 ConnectionId::new(6),
12011 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
12012 )
12013 .unwrap();
12014 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
12015 assert!(
12016 registry.get_module("aft").unwrap().is_none(),
12017 "a squatter took the active slot of an id being swapped"
12018 );
12019 }
12020
12021 #[test]
12026 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
12027 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
12028
12029 let replies = handler
12030 .handle_control(
12031 ConnectionId::new(2),
12032 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
12033 )
12034 .unwrap();
12035
12036 assert_eq!(
12037 replies[0].header.ty,
12038 FrameType::HelloAck,
12039 "reserved candidate refused: {:?}",
12040 serde_json::from_slice::<Value>(&replies[0].body).ok()
12041 );
12042 assert_eq!(
12043 registry
12044 .get_candidate("vault")
12045 .unwrap()
12046 .unwrap()
12047 .connection_id,
12048 ConnectionId::new(2)
12049 );
12050 }
12051
12052 #[test]
12055 fn without_an_open_swap_the_ordinary_gates_decide() {
12056 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
12057 supervisor.close_swap("vault");
12058
12059 let candidate = handler
12060 .handle_control(
12061 ConnectionId::new(2),
12062 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
12063 )
12064 .unwrap();
12065 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
12066 let duplicate = handler
12067 .handle_control(
12068 ConnectionId::new(3),
12069 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
12070 )
12071 .unwrap();
12072 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
12073 assert!(registry.get_candidate("vault").unwrap().is_none());
12074 }
12075 }
12076
12077 mod scopes {
12081 use subc_protocol::scope::{
12082 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
12083 ScopeStatus,
12084 };
12085
12086 use super::*;
12087
12088 const OWNER: &str = "prefrontal-core";
12089
12090 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
12091 ScopeRecord {
12092 scope_ref: scope_ref.to_string(),
12093 scope_epoch,
12094 kind: ScopeKind::Head,
12095 parent: None,
12096 child_owners: Vec::new(),
12097 carriers: Vec::new(),
12098 attributes: Default::default(),
12099 }
12100 }
12101
12102 async fn call(
12103 handler: &ControlHandler,
12104 ctx: &RouteCtx,
12105 request: &ModuleControlRequestFromModule,
12106 ) -> Frame {
12107 let body = serde_json::to_vec(request).unwrap();
12108 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
12109 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
12110 assert_eq!(replies.len(), 1, "{replies:?}");
12111 replies.pop().unwrap()
12112 }
12113
12114 async fn sync(
12115 handler: &ControlHandler,
12116 ctx: &RouteCtx,
12117 generation: u64,
12118 scopes: Vec<ScopeRecord>,
12119 ) -> Result<ModuleControlResponseToModule, String> {
12120 let reply = call(
12121 handler,
12122 ctx,
12123 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
12124 )
12125 .await;
12126 match reply.header.ty {
12127 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
12128 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
12129 }
12130 }
12131
12132 async fn describe(
12133 handler: &ControlHandler,
12134 ctx: &RouteCtx,
12135 owner: &str,
12136 scope_ref: &str,
12137 ) -> ModuleControlResponseToModule {
12138 let reply = call(
12139 handler,
12140 ctx,
12141 &ModuleControlRequestFromModule::ScopeDescribe {
12142 owner: Principal::Reserved {
12143 module_id: owner.to_string(),
12144 },
12145 scope_ref: scope_ref.to_string(),
12146 },
12147 )
12148 .await;
12149 assert_eq!(
12150 reply.header.ty,
12151 FrameType::Response,
12152 "{:?}",
12153 parse_error(&reply)
12154 );
12155 serde_json::from_slice(&reply.body).unwrap()
12156 }
12157
12158 async fn module(
12160 handler: &ControlHandler,
12161 connection: u64,
12162 module_id: &str,
12163 nonce: Option<&str>,
12164 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
12165 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
12166 hello_via_sink(
12167 handler,
12168 &ctx,
12169 &mut rx,
12170 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
12171 )
12172 .await;
12173 (ctx, rx)
12174 }
12175
12176 #[tokio::test]
12179 async fn a_client_connection_cannot_sync_or_describe() {
12180 let handler = ControlHandler::new(Arc::new(Registry::default()));
12181 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
12182 for request in [
12183 ModuleControlRequestFromModule::ScopeSync {
12184 generation: 1,
12185 scopes: vec![head("s", 1)],
12186 },
12187 ModuleControlRequestFromModule::ScopeDescribe {
12188 owner: Principal::Direct,
12189 scope_ref: "s".to_string(),
12190 },
12191 ] {
12192 let reply = call(&handler, &ctx, &request).await;
12193 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
12194 }
12195 assert!(
12196 !handler
12197 .scopes
12198 .read()
12199 .unwrap()
12200 .describe(
12201 &Principal::Reserved {
12202 module_id: OWNER.to_string()
12203 },
12204 "s"
12205 )
12206 .owner_synced
12207 );
12208 }
12209
12210 #[tokio::test]
12213 async fn a_module_without_a_supervised_launch_cannot_sync() {
12214 let handler = ControlHandler::new(Arc::new(Registry::default()));
12215 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
12216 assert_eq!(
12217 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
12218 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
12219 );
12220 }
12221
12222 #[tokio::test]
12223 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
12224 let supervisor = SupervisorHandle::new();
12225 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
12226 let handler = ControlHandler::new(Arc::new(Registry::default()))
12227 .with_supervisor(supervisor.clone());
12228 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
12229 sync(&handler, &incumbent, 1, vec![head("s", 1)])
12230 .await
12231 .expect("the current launch syncs");
12232
12233 supervisor.open_swap(OWNER, "n2".to_string());
12236 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
12237 assert_eq!(
12238 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
12239 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
12240 );
12241 sync(&handler, &incumbent, 2, vec![head("s", 1)])
12242 .await
12243 .expect("the serving owner syncs during the swap");
12244
12245 supervisor.close_swap(OWNER);
12248 assert_eq!(
12249 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
12250 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
12251 );
12252 sync(&handler, &incumbent, 3, vec![head("s", 1)])
12253 .await
12254 .expect("the serving owner syncs after the rollback");
12255 handler.cleanup_connection(candidate.connection_id).unwrap();
12256
12257 supervisor.open_swap(OWNER, "n3".to_string());
12262 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
12263 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
12264 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
12265 .await
12266 .expect("the promoted launch takes authority");
12267 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
12268 panic!("unexpected reply {reply:?}");
12269 };
12270 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
12271 assert_eq!(
12272 sync(&handler, &incumbent, 4, Vec::new()).await,
12273 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
12274 );
12275 }
12276
12277 #[tokio::test]
12280 async fn closing_the_authority_connection_frees_sync_authority() {
12281 let supervisor = SupervisorHandle::new();
12282 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
12283 let handler = ControlHandler::new(Arc::new(Registry::default()))
12284 .with_supervisor(supervisor.clone());
12285 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
12286 sync(&handler, &first, 10, vec![head("s", 1)])
12287 .await
12288 .unwrap();
12289 handler.cleanup_connection(first.connection_id).unwrap();
12290
12291 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
12292 sync(&handler, &second, 1, vec![head("s", 1)])
12293 .await
12294 .expect("the next connection takes the released authority");
12295 }
12296
12297 #[tokio::test]
12298 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
12299 let supervisor = SupervisorHandle::new();
12300 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
12301 let handler =
12302 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
12303 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
12304 sync(&handler, &first, 10, vec![head("s", 1)])
12305 .await
12306 .unwrap();
12307 handler
12308 .handle_control_frame(
12309 &first,
12310 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
12311 )
12312 .await
12313 .unwrap();
12314 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
12315 sync(&handler, &second, 1, vec![head("s", 1)])
12316 .await
12317 .expect("GOODBYE releases authority even if the old socket remains open");
12318 }
12319
12320 #[tokio::test]
12321 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
12322 let registry = Arc::new(Registry::default());
12323 let supervisor_handle = SupervisorHandle::new();
12324 let supervisor = Supervisor::new(Arc::clone(®istry), RestartPolicy::default())
12325 .with_handle(supervisor_handle.clone())
12326 .with_daemon_incarnation("incarnation-7".to_string());
12327 supervisor
12330 .supervise_configured(
12331 ModuleSpec {
12332 module_id: OWNER.to_string(),
12333 program: PathBuf::from("/nonexistent/prefrontal-core"),
12334 args: Vec::new(),
12335 env: Vec::new(),
12336 reserved: false,
12337 reserved_prefixes: Vec::new(),
12338 protocol: ModuleProtocol::Subc,
12339 overlap: Default::default(),
12340 },
12341 false,
12342 )
12343 .unwrap();
12344 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
12345 let handler =
12346 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12347 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
12348
12349 let ModuleControlResponseToModule::ScopeDescribe {
12351 status,
12352 daemon_incarnation,
12353 owner_synced,
12354 owner_configured,
12355 scope,
12356 ..
12357 } = describe(&handler, &reader, OWNER, "s").await
12358 else {
12359 panic!("not a describe reply");
12360 };
12361 assert_eq!(status, ScopeStatus::NotLive);
12362 assert_eq!(daemon_incarnation, "incarnation-7");
12363 assert!(!owner_synced);
12364 assert!(owner_configured);
12365 assert!(scope.is_none());
12366
12367 let ModuleControlResponseToModule::ScopeDescribe {
12370 status,
12371 owner_configured,
12372 ..
12373 } = describe(&handler, &reader, "ghost", "s").await
12374 else {
12375 panic!("not a describe reply");
12376 };
12377 assert_eq!(status, ScopeStatus::NotLive);
12378 assert!(!owner_configured);
12379
12380 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
12382 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
12383 let ModuleControlResponseToModule::ScopeDescribe {
12384 status,
12385 scope_epoch,
12386 owner_synced,
12387 scope,
12388 ..
12389 } = describe(&handler, &reader, OWNER, "s").await
12390 else {
12391 panic!("not a describe reply");
12392 };
12393 assert_eq!(status, ScopeStatus::Live);
12394 assert_eq!(scope_epoch, Some(4));
12395 assert!(owner_synced);
12396 let stamp = scope.expect("a live scope carries its stamp");
12397 assert!(
12398 stamp.owner_authorized,
12399 "prefrontal-core is the default authority"
12400 );
12401 assert_eq!(stamp.kind, ScopeKind::Head);
12402 }
12403
12404 #[tokio::test]
12405 async fn scope_authority_owners_decides_owner_authorized() {
12406 let supervisor = SupervisorHandle::new();
12407 supervisor.set_spawn_nonce("broca", "b1".to_string());
12408 let handler = ControlHandler::new(Arc::new(Registry::default()))
12409 .with_supervisor(supervisor)
12410 .with_scope_authority_owners(vec!["broca".to_string()]);
12411 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
12412 let mut gated = head("s", 1);
12413 gated.attributes.agent_id = Some("agent".to_string());
12414 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
12415 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
12416 describe(&handler, &broca, "broca", "s").await
12417 else {
12418 panic!("not a describe reply");
12419 };
12420 assert!(scope.unwrap().owner_authorized);
12421 }
12422
12423 #[tokio::test]
12427 async fn scope_ops_and_the_scopes_capability_are_advertised() {
12428 let handler = ControlHandler::new(Arc::new(Registry::default()));
12429 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
12430 let ack = hello_via_sink(
12431 &handler,
12432 &ctx,
12433 &mut rx,
12434 hello_frame("m", PROTOCOL_VERSION, 1),
12435 )
12436 .await;
12437 let ack = parse_ack(&ack);
12438 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
12439 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
12440 }
12441 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
12442
12443 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
12444 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12445 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12446 let reply = handler
12447 .handle_control_frame(&client, frame)
12448 .await
12449 .unwrap()
12450 .pop()
12451 .unwrap();
12452 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12453 serde_json::from_slice(&reply.body).unwrap()
12454 else {
12455 panic!("not a server.describe reply");
12456 };
12457 assert!(
12458 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
12459 "{capabilities:?}"
12460 );
12461 }
12462
12463 const PLEXUS: &str = "plexus";
12466 const OTHER: &str = "other";
12467 const AFT: &str = "aft";
12468 const BROCA: &str = "broca";
12469 const MAGIC: &str = "magic-context";
12470
12471 fn nonce(module_id: &str) -> String {
12472 format!("nonce-{module_id}")
12473 }
12474
12475 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
12476 let (tx, rx) = mpsc::channel(64);
12477 (
12478 RouteCtx {
12479 connection_id: ConnectionId::new(connection),
12480 egress: FrameSink::new(tx),
12481 },
12482 rx,
12483 )
12484 }
12485
12486 struct Rig {
12490 handler: ControlHandler,
12491 forwarding: Arc<ForwardingTable>,
12492 owner: RouteCtx,
12493 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12494 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
12495 generation: u64,
12496 next_connection: u64,
12497 _supervisor: Supervisor,
12498 }
12499
12500 async fn rig() -> Rig {
12501 let registry = Arc::new(Registry::default());
12502 let forwarding = Arc::new(ForwardingTable::default());
12503 let supervisor_handle = SupervisorHandle::new();
12504 let supervisor = Supervisor::new(Arc::clone(®istry), RestartPolicy::default())
12505 .with_handle(supervisor_handle.clone());
12506 supervisor
12507 .supervise_configured(
12508 ModuleSpec {
12509 module_id: OWNER.to_string(),
12510 program: PathBuf::from("/nonexistent/prefrontal-core"),
12511 args: Vec::new(),
12512 env: Vec::new(),
12513 reserved: false,
12514 reserved_prefixes: Vec::new(),
12515 protocol: ModuleProtocol::Subc,
12516 overlap: Default::default(),
12517 },
12518 false,
12519 )
12520 .unwrap();
12521 for module_id in [OWNER, AFT, BROCA, MAGIC] {
12522 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
12523 }
12524 let handler =
12525 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12526 .with_supervisor(supervisor_handle);
12527 let (owner, mut owner_rx) = wide_ctx(1);
12528 hello_via_sink(
12529 &handler,
12530 &owner,
12531 &mut owner_rx,
12532 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
12533 )
12534 .await;
12535 let mut modules = BTreeMap::new();
12536 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
12537 let (ctx, mut rx) = wide_ctx(connection);
12538 hello_via_sink(
12539 &handler,
12540 &ctx,
12541 &mut rx,
12542 hello_frame(module_id, PROTOCOL_VERSION, connection),
12543 )
12544 .await;
12545 modules.insert(module_id.to_string(), (ctx, rx));
12546 }
12547 Rig {
12548 handler,
12549 forwarding,
12550 owner,
12551 _owner_rx: owner_rx,
12552 modules,
12553 generation: 0,
12554 next_connection: 100,
12555 _supervisor: supervisor,
12556 }
12557 }
12558
12559 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
12560 ScopeCarrier {
12561 principal: Principal::Reserved {
12562 module_id: module_id.to_string(),
12563 },
12564 targets: targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()),
12565 }
12566 }
12567
12568 fn session(scope_epoch: u64) -> ScopeRecord {
12571 let mut record = head("s", scope_epoch);
12572 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
12573 record.attributes.agent_id = Some("agent-1".to_string());
12574 record.attributes.delegates = true;
12575 record
12576 }
12577
12578 impl Rig {
12579 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
12580 self.generation += 1;
12581 sync(&self.handler, &self.owner, self.generation, scopes)
12582 .await
12583 .expect("the owner's sync is accepted");
12584 }
12585
12586 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
12587 ScopeSelector {
12588 owner: Principal::Reserved {
12589 module_id: OWNER.to_string(),
12590 },
12591 scope_ref: scope_ref.to_string(),
12592 scope_epoch,
12593 }
12594 }
12595
12596 fn open_frame(
12597 &mut self,
12598 opener: Option<&str>,
12599 target: &str,
12600 scope: Option<ScopeSelector>,
12601 ) -> (
12602 RouteCtx,
12603 mpsc::Receiver<crate::router::OutboundFrame>,
12604 Frame,
12605 ) {
12606 self.next_connection += 1;
12607 let (ctx, rx) = wide_ctx(self.next_connection);
12608 let root = unique_project_root("scoped-open");
12609 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
12610 target: RouteTarget::ToolProvider {
12611 module_id: target.to_string(),
12612 },
12613 identity: BindIdentity::new(
12614 root.path().to_path_buf(),
12615 "unit".to_string(),
12616 "session".to_string(),
12617 ),
12618 consumer_identity: opener.map(|module_id| ConsumerIdentity {
12619 module_id: module_id.to_string(),
12620 launch_nonce: nonce(module_id),
12621 }),
12622 consumer_capabilities: None,
12623 role_versions: None,
12624 admission_facts: None,
12625 scope,
12626 })
12627 .unwrap();
12628 let frame = Frame::build(
12629 FrameType::Request,
12630 control_flags(),
12631 0,
12632 0,
12633 self.next_connection,
12634 body,
12635 )
12636 .unwrap();
12637 (ctx, rx, frame)
12638 }
12639
12640 async fn refused(
12642 &mut self,
12643 opener: Option<&str>,
12644 target: &str,
12645 scope: Option<ScopeSelector>,
12646 ) -> String {
12647 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
12648 let replies = self
12649 .handler
12650 .handle_control_frame(&ctx, frame)
12651 .await
12652 .unwrap();
12653 assert_eq!(replies.len(), 1, "{replies:?}");
12654 assert_eq!(replies[0].header.ty, FrameType::Error);
12655 let (_, module_rx) = self.modules.get_mut(target).unwrap();
12656 assert!(
12657 module_rx.try_recv().is_err(),
12658 "a refused open relays nothing"
12659 );
12660 parse_error(&replies[0])["code"]
12661 .as_str()
12662 .unwrap()
12663 .to_string()
12664 }
12665
12666 async fn relayed(
12668 &mut self,
12669 opener: Option<&str>,
12670 target: &str,
12671 scope: Option<ScopeSelector>,
12672 ) -> Relayed {
12673 let (ctx, rx, frame) = self.open_frame(opener, target, scope);
12674 let handler = self.handler.clone();
12675 let task_ctx = ctx.clone();
12676 let task = tokio::spawn(async move {
12677 handler
12678 .handle_control_frame(&task_ctx, frame)
12679 .await
12680 .unwrap()
12681 });
12682 let (_, module_rx) = self.modules.get_mut(target).unwrap();
12683 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
12684 .await
12685 .expect("the target receives the relayed route.bind")
12686 .unwrap()
12687 .frame;
12688 Relayed {
12689 target: target.to_string(),
12690 client: ctx,
12691 client_rx: rx,
12692 task,
12693 bind,
12694 }
12695 }
12696
12697 async fn ack(&self, relayed: &Relayed) {
12698 let (module, _) = &self.modules[&relayed.target];
12699 self.handler
12700 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
12701 .await
12702 .unwrap();
12703 }
12704
12705 async fn bound(
12707 &mut self,
12708 opener: Option<&str>,
12709 target: &str,
12710 scope: Option<ScopeSelector>,
12711 ) -> Bound {
12712 let relayed = self.relayed(opener, target, scope).await;
12713 self.ack(&relayed).await;
12714 let Relayed {
12715 target,
12716 client,
12717 mut client_rx,
12718 task,
12719 bind,
12720 } = relayed;
12721 assert!(
12722 task.await.unwrap().is_empty(),
12723 "the open is answered by commit"
12724 );
12725 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
12726 Bound {
12727 target,
12728 client,
12729 client_rx,
12730 channel,
12731 epoch,
12732 bind,
12733 }
12734 }
12735
12736 fn live(&self, route: &Bound) -> bool {
12737 matches!(
12738 self.forwarding
12739 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
12740 .unwrap(),
12741 DataRoute::Client(DataRouteState::Bound(_))
12742 )
12743 }
12744 }
12745
12746 struct Relayed {
12747 target: String,
12748 client: RouteCtx,
12749 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12750 task: tokio::task::JoinHandle<Vec<Frame>>,
12751 bind: Frame,
12752 }
12753
12754 struct Bound {
12755 target: String,
12756 client: RouteCtx,
12757 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12758 channel: u16,
12759 epoch: u32,
12760 bind: Frame,
12761 }
12762
12763 impl Bound {
12764 fn closed_reason(&mut self) -> RouteCloseReason {
12767 let mut reason = None;
12768 let mut goodbye = false;
12769 while let Ok(outbound) = self.client_rx.try_recv() {
12770 let frame = outbound.frame;
12771 match frame.header.ty {
12772 FrameType::Goodbye => {
12773 assert_eq!(
12774 (frame.header.channel, frame.header.epoch),
12775 (self.channel, self.epoch)
12776 );
12777 goodbye = true;
12778 }
12779 FrameType::Push => {
12780 let ClientControlPush::RouteClosed {
12781 reason: r,
12782 module_id,
12783 ..
12784 } = serde_json::from_slice(&frame.body).unwrap()
12785 else {
12786 panic!("unexpected push");
12787 };
12788 assert_eq!(module_id, self.target);
12789 reason = Some(r);
12790 }
12791 other => panic!("unexpected frame {other:?}"),
12792 }
12793 }
12794 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
12795 reason.expect("the client is told why the route closed")
12796 }
12797
12798 fn untouched(&mut self) -> bool {
12799 self.client_rx.try_recv().is_err()
12800 }
12801
12802 fn stamp(&self) -> Option<ScopeStamp> {
12803 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
12804 ModuleControlRequest::RouteBind { scope, .. } => scope,
12805 other => panic!("expected a route.bind, got {other:?}"),
12806 }
12807 }
12808 }
12809
12810 #[tokio::test]
12811 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
12812 ) {
12813 let mut rig = rig().await;
12814 let mut record = session(1);
12815 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
12816 record.child_owners = vec![Principal::Reserved {
12817 module_id: MAGIC.to_string(),
12818 }];
12819 rig.sync(vec![record]).await;
12820 let scope = || Some(rig_selector("s", Some(1)));
12821
12822 rig.bound(Some(OWNER), PLEXUS, scope()).await;
12825 rig.bound(Some(AFT), OTHER, scope()).await;
12826 rig.bound(Some(BROCA), PLEXUS, scope()).await;
12827
12828 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
12832 assert_eq!(
12833 rig.refused(opener, target, scope()).await,
12834 error_codes::SCOPE_NOT_CARRIER,
12835 "{opener:?} -> {target}"
12836 );
12837 }
12838 }
12839
12840 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
12841 ScopeSelector {
12842 owner: Principal::Reserved {
12843 module_id: OWNER.to_string(),
12844 },
12845 scope_ref: scope_ref.to_string(),
12846 scope_epoch,
12847 }
12848 }
12849
12850 #[tokio::test]
12851 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
12852 let mut rig = rig().await;
12853 rig.sync(vec![session(1)]).await;
12854 for opener in [OWNER, AFT] {
12855 assert_eq!(
12856 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
12857 .await,
12858 error_codes::SCOPE_EPOCH_REQUIRED,
12859 "{opener}"
12860 );
12861 }
12862 }
12863
12864 #[tokio::test]
12865 async fn admission_separates_not_synced_not_live_and_ended() {
12866 let mut rig = rig().await;
12867 let code = rig
12869 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12870 .await;
12871 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
12872 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
12873
12874 let ghost = ScopeSelector {
12876 owner: Principal::Reserved {
12877 module_id: "ghost".to_string(),
12878 },
12879 scope_ref: "s".to_string(),
12880 scope_epoch: Some(1),
12881 };
12882 assert_eq!(
12883 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
12884 error_codes::SCOPE_NOT_LIVE
12885 );
12886
12887 rig.sync(vec![session(2)]).await;
12888 assert_eq!(
12889 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
12890 .await,
12891 error_codes::SCOPE_NOT_LIVE
12892 );
12893 for epoch in [1, 3] {
12894 assert_eq!(
12895 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
12896 .await,
12897 error_codes::SCOPE_ENDED,
12898 "epoch {epoch}"
12899 );
12900 }
12901 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
12903 .await;
12904 }
12905
12906 #[tokio::test]
12907 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
12908 let mut rig = rig().await;
12909 rig.sync(vec![session(1)]).await;
12910 let route = rig
12911 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12912 .await;
12913 let stamp = route.stamp().expect("a scoped bind carries the stamp");
12914 assert_eq!(stamp.scope_ref, "s");
12915 assert_eq!(stamp.scope_epoch, 1);
12916 assert_eq!(stamp.kind, ScopeKind::Head);
12917 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
12918 assert!(stamp.attributes.delegates);
12919 assert!(stamp.owner_authorized);
12920 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
12921 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
12922
12923 let (broca, mut broca_rx) = wide_ctx(50);
12926 hello_via_sink(
12927 &rig.handler,
12928 &broca,
12929 &mut broca_rx,
12930 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
12931 )
12932 .await;
12933 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
12934 .await
12935 .unwrap();
12936 let own = ScopeSelector {
12937 owner: Principal::Reserved {
12938 module_id: BROCA.to_string(),
12939 },
12940 scope_ref: "b".to_string(),
12941 scope_epoch: Some(1),
12942 };
12943 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
12944 assert!(!route.stamp().unwrap().owner_authorized);
12945 }
12946
12947 #[tokio::test]
12948 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
12949 let mut rig = rig().await;
12950 let mut record = head("s", 1);
12951 record.carriers = vec![carrier(AFT, None)];
12952 let flow_id = "Flow:run-7/step_2!~";
12953 record.attributes.flow_id = Some(flow_id.to_string());
12954 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
12955 .await
12956 .unwrap();
12957 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
12958 panic!("not a sync reply");
12959 };
12960 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
12961 let route = rig
12962 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12963 .await;
12964 assert!(rig.live(&route), "the stamped bind committed");
12965 let stamp = route.stamp().expect("a flow scope carries a stamp");
12966 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
12967 assert_eq!(stamp.attributes.agent_id, None);
12968 assert!(!stamp.attributes.delegates);
12969 assert!(stamp.owner_authorized);
12970 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
12971 assert_eq!(unscoped.stamp(), None);
12972 }
12973
12974 #[tokio::test]
12975 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
12976 let mut rig = rig().await;
12977 let mut record = session(1);
12978 record.attributes.flow_id = Some("flow:7".to_string());
12979 rig.sync(vec![record.clone()]).await;
12980 let before = rig.forwarding.published_scope_tag(OWNER, "s");
12981 let mut owner_route = rig
12982 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
12983 .await;
12984 let mut carrier_route = rig
12985 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
12986 .await;
12987 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
12988 rig.sync(vec![record.clone()]).await;
12989 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
12990 assert!(rig.live(&owner_route) && owner_route.untouched());
12991 assert!(rig.live(&carrier_route) && carrier_route.untouched());
12992
12993 record.attributes.flow_id = Some("flow:8".to_string());
12994 rig.sync(vec![record]).await;
12995 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
12996 let before = before.unwrap();
12997 assert_eq!(after.scope_epoch, before.scope_epoch);
12998 assert!(after.version > before.version);
12999 for route in [&mut owner_route, &mut carrier_route] {
13000 assert!(!rig.live(route));
13001 assert_eq!(
13002 route.closed_reason(),
13003 RouteCloseReason::ScopeDelegationChanged
13004 );
13005 }
13006 assert!(rig.live(&unscoped) && unscoped.untouched());
13007 for target in [PLEXUS, OTHER] {
13010 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
13011 let goodbye = module_rx
13012 .try_recv()
13013 .expect("the provider sees the drain")
13014 .frame;
13015 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
13016 assert!(module_rx.try_recv().is_err());
13017 }
13018 let rebound = rig
13019 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13020 .await;
13021 assert_eq!(
13022 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
13023 Some("flow:8")
13024 );
13025 }
13026
13027 #[tokio::test]
13032 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
13033 let mut rig = rig().await;
13034 rig.sync(vec![session(1)]).await;
13035 let mut cotenant = rig
13036 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
13037 .await;
13038
13039 let mut changed = session(1);
13040 changed.child_owners.push(Principal::Reserved {
13041 module_id: MAGIC.to_string(),
13042 });
13043 let mut ended = None;
13044 for (code, next) in [
13045 (error_codes::SCOPE_CHANGED, vec![changed]),
13046 (error_codes::SCOPE_ENDED, Vec::new()),
13047 ] {
13048 let relayed = rig
13049 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13050 .await;
13051 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
13052 ended = Some(next.is_empty());
13053 rig.sync(next).await;
13054 rig.ack(&relayed).await;
13055 let replies = relayed.task.await.unwrap();
13056 assert_eq!(replies.len(), 1, "{replies:?}");
13057 assert_eq!(parse_error(&replies[0])["code"], code);
13058 assert_eq!(
13059 subc_protocol::error_codes::is_retryable_route_open(code),
13060 code == error_codes::SCOPE_CHANGED
13061 );
13062 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
13063 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
13065 let mut goodbyes = Vec::new();
13068 while let Ok(outbound) = plexus_rx.try_recv() {
13069 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
13070 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
13071 }
13072 assert!(
13073 goodbyes.contains(&(bind_channel, bind_epoch)),
13074 "{goodbyes:?}"
13075 );
13076 assert!(rig
13077 .handler
13078 .registry
13079 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
13080 .unwrap()
13081 .is_some());
13082 }
13083 assert_eq!(ended, Some(true));
13084 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
13087 }
13088
13089 #[tokio::test]
13092 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
13093 struct Case {
13094 name: &'static str,
13095 change: fn(&mut ScopeRecord),
13096 closed: [Option<RouteCloseReason>; 4],
13099 }
13100 use RouteCloseReason::*;
13101 let cases = [
13102 Case {
13103 name: "a carrier entry removed",
13104 change: |r| {
13105 r.carriers.retain(|c| {
13106 c.principal
13107 != Principal::Reserved {
13108 module_id: AFT.to_string(),
13109 }
13110 })
13111 },
13112 closed: [None, Some(ScopeCarrierRemoved), None, None],
13113 },
13114 Case {
13115 name: "a target removed from a carrier",
13116 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
13117 closed: [None, None, None, Some(ScopeCarrierRemoved)],
13118 },
13119 Case {
13120 name: "a bare carrier narrowed to targets",
13121 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
13122 closed: [None, Some(ScopeCarrierRemoved), None, None],
13123 },
13124 Case {
13125 name: "delegates turned off",
13126 change: |r| r.attributes.delegates = false,
13127 closed: [Some(ScopeDelegationChanged); 4],
13128 },
13129 Case {
13130 name: "agent_id changed",
13131 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
13132 closed: [Some(ScopeDelegationChanged); 4],
13133 },
13134 Case {
13135 name: "a carrier added, child owners changed, the record re-sent",
13136 change: |r| {
13137 r.carriers.push(carrier(MAGIC, None));
13138 r.child_owners.push(Principal::Reserved {
13139 module_id: MAGIC.to_string(),
13140 });
13141 },
13142 closed: [None; 4],
13143 },
13144 Case {
13145 name: "a target added",
13146 change: |r| {
13147 r.carriers[1]
13148 .targets
13149 .as_mut()
13150 .unwrap()
13151 .push("third".to_string())
13152 },
13153 closed: [None; 4],
13154 },
13155 Case {
13156 name: "delegates turned on",
13157 change: |r| r.attributes.delegates = true,
13158 closed: [None; 4],
13159 },
13160 ];
13161 for case in cases {
13162 let mut rig = rig().await;
13163 rig.sync(vec![session(1)]).await;
13164 let scope = || Some(rig_selector("s", Some(1)));
13165 let mut routes = [
13166 rig.bound(Some(OWNER), PLEXUS, scope()).await,
13167 rig.bound(Some(AFT), PLEXUS, scope()).await,
13168 rig.bound(Some(BROCA), PLEXUS, scope()).await,
13169 rig.bound(Some(BROCA), OTHER, scope()).await,
13170 ];
13171 let mut record = session(1);
13172 (case.change)(&mut record);
13173 rig.sync(vec![record]).await;
13174 for (index, expected) in case.closed.iter().enumerate() {
13175 let route = &mut routes[index];
13176 match expected {
13177 Some(reason) => {
13178 assert!(!rig.live(route), "{}: route {index} still live", case.name);
13179 assert_eq!(
13180 route.closed_reason(),
13181 *reason,
13182 "{}: route {index}",
13183 case.name
13184 );
13185 }
13186 None => {
13187 assert!(rig.live(route), "{}: route {index} closed", case.name);
13188 assert!(
13189 route.untouched(),
13190 "{}: route {index} was told something",
13191 case.name
13192 );
13193 }
13194 }
13195 }
13196 }
13197 }
13198
13199 #[tokio::test]
13200 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
13201 for next in [Vec::new(), vec![session(2)]] {
13203 let mut rig = rig().await;
13204 rig.sync(vec![session(1)]).await;
13205 let mut route = rig
13206 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13207 .await;
13208 rig.sync(next).await;
13209 assert!(!rig.live(&route));
13210 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
13211 }
13212
13213 let mut rig = rig().await;
13216 let mut child = session(1);
13217 child.scope_ref = "child".to_string();
13218 child.kind = ScopeKind::Worker;
13219 child.parent = Some(ScopeParent {
13220 owner: Principal::Reserved {
13221 module_id: OWNER.to_string(),
13222 },
13223 scope_ref: "s".to_string(),
13224 scope_epoch: 1,
13225 });
13226 rig.sync(vec![session(1), child.clone()]).await;
13227 let mut child_route = rig
13228 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
13229 .await;
13230 assert_eq!(
13231 child_route.stamp().unwrap().parent_state,
13232 Some(ParentState::Linked)
13233 );
13234 rig.sync(vec![child]).await;
13235 assert!(!rig.live(&child_route));
13236 assert_eq!(
13237 child_route.closed_reason(),
13238 RouteCloseReason::ScopeParentEnded
13239 );
13240 }
13241
13242 #[tokio::test]
13243 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
13244 ) {
13245 let mut rig = rig().await;
13246 rig.sync(vec![session(1)]).await;
13247 let mut route = rig
13248 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13249 .await;
13250 let before = rig.forwarding.published_scope_tag(OWNER, "s");
13251 rig.sync(vec![session(1)]).await;
13252 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
13253 assert!(rig.live(&route) && route.untouched());
13254
13255 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
13259 .forwarding
13260 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
13261 .unwrap()
13262 else {
13263 panic!("the route is bound");
13264 };
13265 binding.flow.acquire_tagged(9, false).await.unwrap();
13266 let mut widened = session(1);
13267 widened.carriers.push(carrier(MAGIC, None));
13268 rig.sync(vec![widened]).await;
13269 assert!(rig.live(&route) && route.untouched());
13270 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
13271 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
13272 assert_eq!(binding.flow.in_flight(), 1);
13275 binding
13276 .flow
13277 .acquire_tagged(10, false)
13278 .await
13279 .expect("the flow is still open");
13280 }
13281
13282 #[tokio::test]
13285 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
13286 let mut rig = rig().await;
13287 rig.sync(vec![session(1)]).await;
13288 let mut on_incumbent = rig
13289 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13290 .await;
13291
13292 let (candidate, _candidate_rx) = wide_ctx(9);
13295 let registration = rig
13296 .handler
13297 .registry
13298 .register_candidate_with_control_ops(
13299 manifest(PLEXUS, PROTOCOL_VERSION),
13300 PROTOCOL_VERSION,
13301 candidate.connection_id,
13302 module_baseline_control_ops(),
13303 )
13304 .unwrap();
13305 rig.forwarding
13306 .register_candidate_module_connection(
13307 candidate.connection_id,
13308 PLEXUS.to_string(),
13309 PROTOCOL_VERSION,
13310 manifest_concurrency(®istration.manifest),
13311 candidate.egress.clone(),
13312 )
13313 .unwrap();
13314 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
13315 rig.handler
13316 .registry
13317 .promote_candidate(PLEXUS)
13318 .unwrap()
13319 .unwrap();
13320 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
13321
13322 rig.sync(Vec::new()).await;
13323 assert!(!rig.live(&on_incumbent));
13324 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
13325 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
13326 let goodbye = incumbent_rx
13327 .try_recv()
13328 .expect("the superseded endpoint is told")
13329 .frame;
13330 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
13331 }
13332 }
13333}
13334
13335#[cfg(test)]
13336mod concurrency_default_exposure_tests {
13337 use super::*;
13338
13339 fn hello_body(role_json: &str) -> Vec<u8> {
13340 format!(
13341 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
13342 )
13343 .into_bytes()
13344 }
13345
13346 fn manifest_from(body: &[u8]) -> ModuleManifest {
13347 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
13348 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
13349 .expect("manifest parses")
13350 }
13351
13352 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
13353
13354 #[test]
13355 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
13356 let body = hello_body(&format!(
13357 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
13358 ));
13359 let manifest = manifest_from(&body);
13360 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
13363 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
13364 }
13365
13366 #[test]
13367 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
13368 let body = hello_body(&format!(
13369 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
13370 ));
13371 let manifest = manifest_from(&body);
13372 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
13373 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
13374 }
13375
13376 #[test]
13377 fn non_management_roles_are_never_reported() {
13378 let body = hello_body(
13379 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
13380 );
13381 let manifest = manifest_from(&body);
13382 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
13383 }
13384}