1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 path::PathBuf,
5 sync::{Arc, Mutex},
6 time::{Duration, Instant as StdInstant},
7};
8
9use serde::{Deserialize, Serialize};
10use subc_control::{
11 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
12 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
13 ModuleDeclaredProvenance, ModuleProtocol, PollKind, RouteCloseReason, SpawnCursor,
14 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
15 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
16 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
17};
18use subc_protocol::{
19 error_codes,
20 manifest::{
21 validate_hello_capability_grammar, validate_hello_self_signal_declarations,
22 CapabilityDeclarations, Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
23 },
24 session::{
25 HealthReport, ModuleControlPush, ModuleControlRequest, ModuleControlRequestFromModule,
26 ModuleControlResponse, ModuleControlResponseToModule, MODULE_CONTROL_OP_HEALTH_CHECK,
27 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
28 },
29 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
30 Priority, RouteTarget, PROTOCOL_VERSION,
31};
32use tokio::time::{timeout_at, Instant};
33use tracing::{debug, info, warn};
34
35use crate::{
36 capability_requirements::{
37 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
38 DuplicateClaimSource, RegisteredModule, RequirementStatus, RuntimeModule,
39 },
40 daemon_config::RestartRequiredSection,
41 forwarding::{
42 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
43 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
44 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
45 },
46 observability::ROUTE_OPEN_REFUSED_DECLARED_NOT_READY,
47 provenance::{
48 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
49 },
50 registry::{ChannelState, ConnectionId, Registry, RegistryError},
51 router::{RouteCtx, RouterError},
52 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
53 stderr_tail::{CaptureState, TailEntry},
54 supervise::{
55 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
56 SupervisorHandle, SwapHelloAdmission,
57 },
58 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
59};
60
61pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
67
68const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
69const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
70const CAP_PING_PONG: &str = "ping_pong_v1";
71const CAP_SESSION_ATTACH: &str = "session_attach_v1";
72const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
73
74const SUBC_CONTROL_OPS: &[&str] = &[
75 ops::SERVER_DESCRIBE,
76 ops::CATALOG_LIST,
77 ops::ROUTE_OPEN,
78 ops::ROUTE_POLL,
79 ops::ROUTE_CLOSING,
80 ops::ROUTE_CLOSED,
81 ops::SUPERVISOR_LIST,
82 ops::SUPERVISOR_RESTART,
83 ops::SUPERVISOR_SWAP,
84 ops::SUPERVISOR_RELOAD,
85 ops::SUPERVISOR_RESCAN,
86 ops::SUPERVISOR_RELEASE_RESERVED,
87 ops::SUPERVISOR_SET_ENABLED,
88 ops::SUPERVISOR_HEALTH_PROBE,
89 ops::SUPERVISOR_HEALTH,
90 ops::SUPERVISOR_STDERR_TAIL,
91 ops::SUPERVISOR_TERMINALS,
92 ops::SUPERVISOR_ROUTES,
93 ops::SUPERVISOR_PROVENANCE,
94 ops::SUPERVISOR_SPAWN_SNAPSHOT,
95 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
96];
97
98const MODULE_TO_SUBC_CONTROL_OPS: &[&str] =
99 &[MODULE_TO_SUBC_OP_CATALOG_UPDATE, "supervisor.live_roots"];
100
101const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
102
103pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
111
112pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
127
128pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
146
147const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
148const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
149
150#[derive(Clone)]
151struct DaemonProvenanceFacts {
152 build: DaemonBuildProvenance,
153 pid: Option<u32>,
154 started_at_ms: Option<u64>,
155 start_clock: Option<crate::clock::StartClock>,
156 executable_path: Option<PathBuf>,
157 executable_identity: Option<SpawnedFileIdentity>,
158 process_start_time: Option<u64>,
159 probe: ExecutableIdentityProbe,
160}
161
162impl Default for DaemonProvenanceFacts {
163 fn default() -> Self {
164 Self {
165 build: DaemonBuildProvenance {
166 build_git_sha: None,
167 build_lock_digest: None,
168 },
169 pid: None,
170 started_at_ms: None,
171 start_clock: None,
172 executable_path: None,
173 executable_identity: None,
174 process_start_time: None,
175 probe: ExecutableIdentityProbe::default(),
176 }
177 }
178}
179
180#[derive(Debug, Clone)]
181struct SupervisorRescanContext {
182 supervisor: Supervisor,
183 config_path: PathBuf,
184 configured_port: Option<u16>,
185 storage_config: Option<crate::daemon_config::StorageConfig>,
186 admission_facts_carrier_module_id: Option<String>,
187 admission_facts_targets: Option<Vec<String>>,
188}
189
190#[derive(Clone)]
192pub struct ControlHandler {
193 registry: Arc<Registry>,
194 forwarding: Arc<ForwardingTable>,
195 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
196 supervisor: SupervisorHandle,
197 subc_capabilities: Arc<[String]>,
198 route_bind_relay_timeout: Duration,
202 route_bind_relay_timeouts: BTreeMap<String, Duration>,
206 route_bind_breakers: RouteBindBreakers,
209 route_bind_concurrency: RouteBindConcurrency,
212 route_bind_breaker_threshold: u32,
214 route_bind_breaker_cooldown: Duration,
216 health_probe_timeout: Duration,
217 storage_config: Option<crate::daemon_config::StorageConfig>,
220 admission_facts_carrier_module_id: Option<String>,
221 admission_facts_targets: Option<Vec<String>>,
222 rescan: Option<SupervisorRescanContext>,
223 connected_clients: ConnectedClients,
224 counters: DaemonCounters,
225 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
226 daemon_provenance: DaemonProvenanceFacts,
227 #[cfg(test)]
228 control_dispatch_delay: Option<Duration>,
229 #[cfg(test)]
230 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
231}
232
233impl fmt::Debug for ControlHandler {
234 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
235 f.debug_struct("ControlHandler")
236 .field("registry", &self.registry)
237 .field("forwarding", &self.forwarding)
238 .field("process_liveness", &self.process_liveness.is_some())
239 .field("supervisor", &self.supervisor)
240 .field("subc_capabilities", &self.subc_capabilities)
241 .finish()
242 }
243}
244
245struct RouteOpenRequest {
246 target: RouteTarget,
247 identity: BindIdentity,
248 consumer_identity: Option<ConsumerIdentity>,
249 consumer_capabilities: Option<Vec<String>>,
250 admission_facts: Option<serde_json::Value>,
251}
252
253struct RouteBindReservationGuard {
254 forwarding: Arc<ForwardingTable>,
255 endpoint: ModuleEndpointId,
256 relay_corr: u64,
257 armed: bool,
258}
259
260struct ModuleControlRpcGuard {
261 forwarding: Arc<ForwardingTable>,
262 endpoint: ModuleEndpointId,
263 corr: u64,
264 armed: bool,
265}
266
267impl ModuleControlRpcGuard {
268 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
269 Self {
270 forwarding,
271 endpoint,
272 corr,
273 armed: true,
274 }
275 }
276
277 fn disarm(&mut self) {
278 self.armed = false;
279 }
280}
281
282impl Drop for ModuleControlRpcGuard {
283 fn drop(&mut self) {
284 if self.armed {
285 let _ = self
286 .forwarding
287 .cancel_module_control_rpc(self.endpoint, self.corr);
288 }
289 }
290}
291
292impl RouteBindReservationGuard {
293 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
294 Self {
295 forwarding,
296 endpoint,
297 relay_corr,
298 armed: true,
299 }
300 }
301
302 fn release_and_disarm(&mut self) {
303 if !self.armed {
304 return;
305 }
306 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
307 self.endpoint,
308 self.relay_corr,
309 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
310 ) {
311 send_goodbye_target_best_effort(&target, "canceled route.bind");
312 }
313 self.armed = false;
314 }
315
316 fn disarm(&mut self) {
317 self.armed = false;
318 }
319}
320
321impl Drop for RouteBindReservationGuard {
322 fn drop(&mut self) {
323 self.release_and_disarm();
324 }
325}
326
327#[derive(Debug, Clone, Default)]
357pub(crate) struct RouteBindBreakers {
358 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
359}
360
361#[derive(Debug, Clone, Default)]
362pub(crate) struct RouteBindConcurrency {
363 modules: Arc<Mutex<HashMap<String, usize>>>,
364}
365
366struct RouteBindConcurrencyGuard {
367 concurrency: RouteBindConcurrency,
368 module_id: String,
369}
370
371impl RouteBindConcurrency {
372 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
375 let mut modules = self
376 .modules
377 .lock()
378 .expect("route.bind concurrency mutex poisoned");
379 let in_flight = modules.entry(module_id.to_string()).or_default();
380 if *in_flight >= limit {
381 return Err(*in_flight);
382 }
383 *in_flight += 1;
384 Ok(RouteBindConcurrencyGuard {
385 concurrency: self.clone(),
386 module_id: module_id.to_string(),
387 })
388 }
389}
390
391impl Drop for RouteBindConcurrencyGuard {
392 fn drop(&mut self) {
393 let mut modules = self
394 .concurrency
395 .modules
396 .lock()
397 .expect("route.bind concurrency mutex poisoned");
398 let remove = {
399 let in_flight = modules
400 .get_mut(&self.module_id)
401 .expect("admitted route.bind has a concurrency entry");
402 *in_flight -= 1;
403 *in_flight == 0
404 };
405 if remove {
406 modules.remove(&self.module_id);
407 }
408 }
409}
410
411#[derive(Debug, Default)]
412struct ModuleBreakerState {
413 consecutive_timeouts: u32,
415 cooldown_until: Option<Instant>,
418 probe_in_flight: bool,
423}
424
425enum RouteBindAdmission<'a> {
427 Admitted {
428 guard: RouteBindBreakerGuard<'a>,
429 probe: bool,
432 },
433 Refused {
434 consecutive_timeouts: u32,
435 retry_in: Duration,
439 probe_in_flight: bool,
440 },
441}
442
443struct RouteBindBreakerGuard<'a> {
450 breakers: RouteBindBreakers,
451 module_id: &'a str,
452 settled: bool,
453}
454
455impl RouteBindBreakerGuard<'_> {
456 fn record_accepted(&mut self) -> bool {
460 self.settled = true;
461 self.breakers.record_accepted(self.module_id)
462 }
463
464 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
467 self.settled = true;
468 self.breakers
469 .record_timeout(self.module_id, threshold, cooldown)
470 }
471
472 fn record_inconclusive(&mut self) {
481 self.settled = true;
482 self.breakers.record_inconclusive(self.module_id);
483 }
484}
485
486impl Drop for RouteBindBreakerGuard<'_> {
487 fn drop(&mut self) {
488 if !self.settled {
489 self.breakers.record_inconclusive(self.module_id);
490 }
491 }
492}
493
494struct BreakerOpened {
498 consecutive_timeouts: u32,
499 reopened_after_probe: bool,
502}
503
504impl RouteBindBreakers {
505 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
506 self.modules
507 .lock()
508 .expect("route.bind breaker mutex poisoned")
509 }
510
511 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
514 let admitted = |probe| RouteBindAdmission::Admitted {
515 guard: RouteBindBreakerGuard {
516 breakers: self.clone(),
517 module_id,
518 settled: false,
519 },
520 probe,
521 };
522
523 let mut modules = self.lock();
524 let Some(state) = modules.get_mut(module_id) else {
525 return admitted(false);
526 };
527 let Some(cooldown_until) = state.cooldown_until else {
528 return admitted(false);
529 };
530 if state.probe_in_flight {
531 return RouteBindAdmission::Refused {
532 consecutive_timeouts: state.consecutive_timeouts,
533 retry_in: Duration::ZERO,
534 probe_in_flight: true,
535 };
536 }
537 let now = Instant::now();
538 if now < cooldown_until {
539 return RouteBindAdmission::Refused {
540 consecutive_timeouts: state.consecutive_timeouts,
541 retry_in: cooldown_until - now,
542 probe_in_flight: false,
543 };
544 }
545 state.probe_in_flight = true;
546 admitted(true)
547 }
548
549 fn record_accepted(&self, module_id: &str) -> bool {
550 self.lock()
551 .remove(module_id)
552 .is_some_and(|state| state.cooldown_until.is_some())
553 }
554
555 fn record_timeout(
556 &self,
557 module_id: &str,
558 threshold: u32,
559 cooldown: Duration,
560 ) -> Option<BreakerOpened> {
561 let mut modules = self.lock();
562 let state = modules.entry(module_id.to_string()).or_default();
563 let was_open = state.cooldown_until.is_some();
564 let was_probe = state.probe_in_flight;
565 state.probe_in_flight = false;
566 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
567 if state.consecutive_timeouts < threshold {
568 return None;
569 }
570 state.cooldown_until = Some(Instant::now() + cooldown);
571 Some(BreakerOpened {
572 consecutive_timeouts: state.consecutive_timeouts,
573 reopened_after_probe: was_open && was_probe,
574 })
575 }
576
577 fn record_inconclusive(&self, module_id: &str) {
578 if let Some(state) = self.lock().get_mut(module_id) {
579 state.probe_in_flight = false;
580 }
581 }
582
583 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
598 self.lock()
599 .remove(module_id)
600 .map(|state| state.consecutive_timeouts)
601 .filter(|discarded| *discarded > 0)
602 }
603
604 fn open_snapshot(&self) -> Option<serde_json::Value> {
611 let now = Instant::now();
612 let modules = self.lock();
613 let open = modules
614 .iter()
615 .filter_map(|(module_id, state)| {
616 let cooldown_until = state.cooldown_until?;
617 Some((
618 module_id.clone(),
619 serde_json::json!({
620 "consecutive_timeouts": state.consecutive_timeouts,
621 "cooldown_remaining_ms":
622 cooldown_until.saturating_duration_since(now).as_millis() as u64,
623 "probe_in_flight": state.probe_in_flight,
624 }),
625 ))
626 })
627 .collect::<serde_json::Map<String, serde_json::Value>>();
628 (!open.is_empty()).then_some(serde_json::Value::Object(open))
629 }
630}
631
632impl ControlHandler {
633 pub fn new(registry: Arc<Registry>) -> Self {
634 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
635 }
636
637 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
638 let counters = forwarding.counters();
639 let route_bind_breakers = forwarding.route_bind_breakers();
643 let route_bind_concurrency = forwarding.route_bind_concurrency();
644 Self {
645 registry,
646 forwarding,
647 process_liveness: None,
648 supervisor: SupervisorHandle::new(),
649 subc_capabilities: Arc::from([
650 CAP_MANIFEST_REGISTRATION.to_string(),
651 CAP_CHANNEL_LIFECYCLE.to_string(),
652 CAP_PING_PONG.to_string(),
653 CAP_SESSION_ATTACH.to_string(),
654 CAP_ADMISSION_FACTS_RELAY.to_string(),
655 ]),
656 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
657 route_bind_relay_timeouts: BTreeMap::new(),
658 route_bind_breakers,
659 route_bind_concurrency,
660 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
661 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
662 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
663 storage_config: None,
664 admission_facts_carrier_module_id: None,
665 admission_facts_targets: None,
666 rescan: None,
667 connected_clients: ConnectedClients::new(),
668 counters,
669 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
670 daemon_provenance: DaemonProvenanceFacts::default(),
671 #[cfg(test)]
672 control_dispatch_delay: None,
673 #[cfg(test)]
674 provenance_probe_override: None,
675 }
676 }
677
678 pub fn with_storage_config(
681 mut self,
682 storage_config: Option<crate::daemon_config::StorageConfig>,
683 ) -> Self {
684 self.storage_config = storage_config;
685 self
686 }
687
688 pub fn with_admission_facts_config(
692 mut self,
693 carrier_module_id: Option<String>,
694 targets: Option<Vec<String>>,
695 ) -> Self {
696 self.admission_facts_carrier_module_id = carrier_module_id;
697 self.admission_facts_targets = targets;
698 self
699 }
700
701 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
704 self.route_bind_relay_timeout = timeout;
705 self
706 }
707
708 pub fn with_route_bind_relay_timeouts(
714 mut self,
715 timeouts: impl IntoIterator<Item = (String, Duration)>,
716 ) -> Self {
717 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
718 self
719 }
720
721 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
727 self.route_bind_relay_timeouts
728 .get(module_id)
729 .copied()
730 .unwrap_or(self.route_bind_relay_timeout)
731 }
732
733 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
741 self.route_bind_breaker_threshold = threshold.max(1);
742 self.route_bind_breaker_cooldown = cooldown;
743 self
744 }
745
746 #[cfg(test)]
747 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
748 self.health_probe_timeout = timeout;
749 self
750 }
751
752 #[cfg(test)]
753 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
754 self.control_dispatch_delay = Some(delay);
755 self
756 }
757
758 pub fn with_process_liveness(
759 mut self,
760 process_liveness: Arc<dyn ModuleProcessLiveness>,
761 ) -> Self {
762 self.process_liveness = Some(process_liveness);
763 self
764 }
765
766 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
767 self.supervisor = supervisor;
768 self
769 }
770
771 pub fn with_daemon_provenance(
772 mut self,
773 pid: u32,
774 started_at_ms: u64,
775 executable_path: Option<PathBuf>,
776 build_git_sha: Option<String>,
777 build_lock_digest: Option<String>,
778 ) -> Self {
779 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
780 let process_start_time = process_start_time(pid);
781 self.daemon_provenance = DaemonProvenanceFacts {
782 build: DaemonBuildProvenance {
783 build_git_sha,
784 build_lock_digest,
785 },
786 pid: Some(pid),
787 started_at_ms: Some(started_at_ms),
788 start_clock: None,
789 executable_path,
790 executable_identity,
791 process_start_time,
792 probe: ExecutableIdentityProbe::default(),
793 };
794 self
795 }
796
797 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
798 self.daemon_provenance.start_clock = Some(clock);
799 self
800 }
801
802 #[cfg(test)]
803 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
804 self.provenance_probe_override = Some(result);
805 self
806 }
807
808 pub fn with_capability_config(
812 self,
813 modules: impl IntoIterator<Item = (String, bool)>,
814 reserved_capabilities: BTreeMap<String, String>,
815 ) -> Self {
816 self.capability_evaluator
817 .configure(modules, reserved_capabilities);
818 self
819 }
820
821 pub fn with_supervisor_rescan(
822 mut self,
823 supervisor: Supervisor,
824 config_path: impl Into<PathBuf>,
825 configured_port: Option<u16>,
826 ) -> Self {
827 self.rescan = Some(SupervisorRescanContext {
828 supervisor,
829 config_path: config_path.into(),
830 configured_port,
831 storage_config: self.storage_config.clone(),
832 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
833 admission_facts_targets: self.admission_facts_targets.clone(),
834 });
835 self
836 }
837
838 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
839 self.connected_clients = connected_clients;
840 self
841 }
842
843 pub fn forwarding(&self) -> Arc<ForwardingTable> {
844 Arc::clone(&self.forwarding)
845 }
846
847 pub(crate) fn counters(&self) -> DaemonCounters {
848 self.counters.clone()
849 }
850
851 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
854 tokio::spawn(async move {
855 loop {
856 self.capability_evaluator
857 .wait_for_change_or_deadline()
858 .await;
859 self.refresh_capability_requirements();
860 }
861 });
862 }
863
864 fn runtime_capability_snapshot(
865 &self,
866 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
867 let runtime = self
868 .supervisor
869 .list()
870 .into_iter()
871 .map(|module| {
872 let status = module.status().map_err(|err| {
873 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
874 })?;
875 Ok(RuntimeModule {
876 module_id: status.module_id,
877 state: status.state,
878 enabled: status.enabled,
879 })
880 })
881 .collect::<Result<Vec<_>, RouterError>>()?;
882 let (_, registrations) = self.registry.list_modules().map_err(|err| {
883 RouterError::backend(
884 0,
885 0,
886 format!("failed to list capability registrations: {err}"),
887 )
888 })?;
889 let registrations = registrations
890 .into_iter()
891 .map(|registration| RegisteredModule {
892 module_id: registration.manifest.module_id,
893 module_version: registration.manifest.module_version,
894 capabilities: registration.manifest.capabilities,
895 })
896 .collect();
897 Ok((runtime, registrations))
898 }
899
900 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
907 let cached_registration = RegisteredModule {
908 module_id: registration.manifest.module_id.clone(),
909 module_version: registration.manifest.module_version.clone(),
910 capabilities: registration.manifest.capabilities.clone(),
911 };
912 if self.capability_evaluator.record_hello(&cached_registration) {
913 warn!(
914 module_id = %cached_registration.module_id,
915 "capability claims drifted from the cached manifest"
916 );
917 }
918 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
919 self.enforce_capability_denies();
920 }
921 self.refresh_capability_requirements();
922 }
923
924 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
928 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
929 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
930 self.supervisor.set_swap_promotion_observer(observer);
931 }
932
933 pub fn refresh_capability_requirements(&self) {
934 match self.runtime_capability_snapshot() {
935 Ok((runtime, registrations)) => {
936 log_requirement_events(
937 self.capability_evaluator
938 .evaluate_now(&runtime, ®istrations),
939 );
940 }
941 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
942 }
943 }
944
945 fn enforce_capability_denies(&self) {
949 let (_, registrations) = match self.registry.list_modules() {
950 Ok(snapshot) => snapshot,
951 Err(err) => {
952 warn!(error = %err, "failed to read registrations for capability deny census");
953 return;
954 }
955 };
956 let manifests = registrations
957 .into_iter()
958 .map(|registration| {
959 (
960 registration.manifest.module_id.clone(),
961 registration.manifest,
962 )
963 })
964 .collect::<BTreeMap<_, _>>();
965 let census = match self.forwarding.route_census(None) {
966 Ok(census) => census,
967 Err(err) => {
968 warn!(error = %err, "failed to read route census for capability deny enforcement");
969 return;
970 }
971 };
972
973 for (target_module_id, routes) in census {
974 let Some(target_manifest) = manifests.get(&target_module_id) else {
975 continue;
976 };
977 let mut closed_routes = Vec::new();
978 let mut module_goodbyes = Vec::new();
979 for route in routes {
980 let Principal::Reserved {
981 module_id: opening_module_id,
982 } = &route.principal
983 else {
984 continue;
985 };
986 let Some(opening_manifest) = manifests.get(opening_module_id) else {
987 continue;
988 };
989 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
990 continue;
991 };
992
993 match self.forwarding.release_client_route(
994 route.goodbye_target.connection_id,
995 route.goodbye_target.channel,
996 route.goodbye_target.epoch,
997 ) {
998 Ok(RouteRelease::Removed(module_goodbye)) => {
999 warn!(
1000 opening_module_id,
1001 target_module_id,
1002 capability,
1003 "force-closing route because an attested capability deny edge now matches"
1004 );
1005 closed_routes.push(route);
1006 module_goodbyes.push(module_goodbye);
1007 }
1008 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1009 Err(err) => warn!(
1010 opening_module_id,
1011 target_module_id,
1012 capability,
1013 error = %err,
1014 "failed to force-close capability-denied route"
1015 ),
1016 }
1017 }
1018
1019 if closed_routes.is_empty() {
1020 continue;
1021 }
1022 send_route_control_pushes(
1023 &self.forwarding,
1024 closed_routes,
1025 ClientControlPush::RouteClosed {
1026 module_id: target_module_id,
1027 reason: RouteCloseReason::CapabilityDenied,
1028 drained: false,
1029 abandoned: 0,
1030 excluded_subscriptions: 0,
1031 terminal: Some(false),
1032 },
1033 );
1034 self.emit_route_goodbyes(module_goodbyes);
1035 }
1036 }
1037
1038 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1039 self.capability_evaluator
1040 .statuses()
1041 .into_iter()
1042 .map(capability_requirement_status)
1043 .collect()
1044 }
1045
1046 fn deregister_connection(
1053 &self,
1054 connection_id: ConnectionId,
1055 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1056 self.registry.deregister_connection(connection_id)
1057 }
1058
1059 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1060 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1061 return None;
1062 }
1063 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1064 parse_client_control_request(&frame.body)
1065 else {
1066 return None;
1067 };
1068 Some(target_module_id(&target).to_string())
1069 }
1070
1071 pub(crate) fn route_open_capacity_refusal(
1072 &self,
1073 ctx: &RouteCtx,
1074 frame: &Frame,
1075 target_module_id: &str,
1076 limit: usize,
1077 ) -> Result<Frame, RouterError> {
1078 self.route_open_admission_refusal_frame(
1079 ctx,
1080 frame,
1081 target_module_id,
1082 format!(
1083 "connection already has {limit} route.open binds in flight; retry after one settles"
1084 ),
1085 )
1086 }
1087
1088 fn route_open_admission_refusal_frame(
1096 &self,
1097 ctx: &RouteCtx,
1098 frame: &Frame,
1099 target_module_id: &str,
1100 message: impl Into<String>,
1101 ) -> Result<Frame, RouterError> {
1102 self.route_open_refusal_frame(
1103 ctx,
1104 frame,
1105 target_module_id,
1106 "open_admission_full",
1107 error_codes::TARGET_UNAVAILABLE,
1108 message,
1109 )
1110 }
1111
1112 #[cfg(test)]
1118 pub fn handle_control(
1119 &self,
1120 connection_id: ConnectionId,
1121 frame: Frame,
1122 ) -> Result<Vec<Frame>, RouterError> {
1123 match frame.header.ty {
1124 FrameType::Ping => Ok(vec![pong(&frame)?]),
1125 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1126 FrameType::Goodbye => self.handle_goodbye(connection_id),
1127 ty => Ok(vec![control_error_frame(
1128 &frame,
1129 "unsupported_control_frame",
1130 format!("unsupported channel-0 frame {ty:?}"),
1131 )?]),
1132 }
1133 }
1134
1135 pub async fn handle_control_frame(
1136 &self,
1137 ctx: &RouteCtx,
1138 frame: Frame,
1139 ) -> Result<Vec<Frame>, RouterError> {
1140 self.handle_control_frame_timed(ctx, frame, None).await
1141 }
1142
1143 pub(crate) async fn handle_control_frame_timed(
1144 &self,
1145 ctx: &RouteCtx,
1146 frame: Frame,
1147 dispatch_started_at: Option<StdInstant>,
1148 ) -> Result<Vec<Frame>, RouterError> {
1149 match frame.header.ty {
1150 FrameType::Ping => Ok(vec![pong(&frame)?]),
1151 FrameType::Hello => {
1152 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1153 }
1154 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1155 FrameType::Cancel => {
1156 if self
1157 .supervisor
1158 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1159 {
1160 Ok(Vec::new())
1161 } else {
1162 Ok(vec![control_error_frame(
1163 &frame,
1164 "unknown_subscription",
1165 "no supervisor spawn subscription has this correlation id",
1166 )?])
1167 }
1168 }
1169 FrameType::Request => {
1170 if self
1171 .forwarding
1172 .module_endpoint_for_connection(ctx.connection_id)
1173 .map_err(RouterError::Forwarding)?
1174 .is_some()
1175 {
1176 if !is_known_module_request_op(&frame.body) {
1177 return Ok(vec![control_error_frame(
1178 &frame,
1179 "unsupported_control_frame",
1180 "module-originated channel-0 REQUEST is not supported",
1181 )?]);
1182 }
1183 let request = match parse_module_control_request_from_module(&frame.body) {
1184 Ok(request) => request,
1185 Err((err, ControlRequestBodyError::UnknownOp)) => {
1186 return Ok(vec![control_error_frame(
1187 &frame,
1188 "unsupported_control_frame",
1189 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1190 )?])
1191 }
1192 Err((err, ControlRequestBodyError::InvalidBody)) => {
1193 return Ok(vec![control_error_frame(
1194 &frame,
1195 "invalid_control_body",
1196 format!("malformed module control body: {err}"),
1197 )?])
1198 }
1199 };
1200 let op = module_control_request_op(&request);
1201 let corr = frame.header.corr;
1202 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1203 let result =
1204 self.handle_module_control_request(ctx.connection_id, frame, request);
1205 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1206 return result;
1207 }
1208
1209 if is_known_module_request_op(&frame.body) {
1210 return Ok(vec![control_error_frame(
1211 &frame,
1212 "not_registered",
1213 "catalog.update requires an active module registration owned by this connection",
1214 )?]);
1215 }
1216
1217 let request = match parse_client_control_request(&frame.body) {
1218 Ok(request) => request,
1219 Err((err, ControlRequestBodyError::UnknownOp)) => {
1220 return Ok(vec![control_error_frame(
1221 &frame,
1222 "unknown_control_op",
1223 format!("unknown client control op: {err}"),
1224 )?])
1225 }
1226 Err((err, ControlRequestBodyError::InvalidBody)) => {
1227 return Ok(vec![control_error_frame(
1228 &frame,
1229 "invalid_control_body",
1230 format!("malformed client control body: {err}"),
1231 )?])
1232 }
1233 };
1234 let op = client_control_request_op(&request);
1235 let corr = frame.header.corr;
1236 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1237 #[cfg(test)]
1238 if let Some(delay) = self.control_dispatch_delay {
1239 tokio::time::sleep(delay).await;
1240 }
1241 let result = self
1242 .handle_client_control_request(ctx, frame, request)
1243 .await;
1244 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1245 result
1246 }
1247 FrameType::Push => {
1248 let Some(endpoint) = self
1249 .forwarding
1250 .module_endpoint_for_connection(ctx.connection_id)
1251 .map_err(RouterError::Forwarding)?
1252 else {
1253 return Ok(vec![control_error_frame(
1254 &frame,
1255 "unsupported_control_frame",
1256 "client-originated channel-0 PUSH is not supported",
1257 )?]);
1258 };
1259 self.handle_status_update(endpoint, frame)
1260 }
1261 FrameType::Response | FrameType::Error
1262 if self
1263 .forwarding
1264 .module_endpoint_for_connection(ctx.connection_id)
1265 .map_err(RouterError::Forwarding)?
1266 .is_some() =>
1267 {
1268 self.handle_module_relay_response(ctx.connection_id, frame)
1269 }
1270 ty => Ok(vec![control_error_frame(
1271 &frame,
1272 "unsupported_control_frame",
1273 format!("unsupported channel-0 frame {ty:?}"),
1274 )?]),
1275 }
1276 }
1277
1278 pub fn cleanup_connection(
1279 &self,
1280 connection_id: ConnectionId,
1281 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1282 let crash_closed = self
1283 .registry
1284 .get_module_by_connection(connection_id)?
1285 .and_then(|registration| {
1286 self.forwarding
1287 .module_endpoint_for_connection(connection_id)
1288 .ok()
1289 .flatten()
1290 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1291 .map(|routes| (registration.manifest.module_id, routes))
1292 });
1293 if let Some((module_id, routes)) = crash_closed {
1294 let terminal = match self.supervisor.get(&module_id) {
1295 None => false,
1296 Some(module) => match module.will_recover_after_connection_loss() {
1297 Ok(will_recover) => !will_recover,
1298 Err(err) => {
1299 warn!(
1300 %module_id,
1301 error = %err,
1302 "failed to read crash recovery verdict; reporting non-terminal conservatively"
1303 );
1304 false
1305 }
1306 },
1307 };
1308 send_route_control_pushes(
1309 &self.forwarding,
1310 routes,
1311 ClientControlPush::RouteClosed {
1312 module_id,
1313 reason: RouteCloseReason::Crash,
1314 drained: false,
1315 abandoned: 0,
1316 excluded_subscriptions: 0,
1317 terminal: Some(terminal),
1318 },
1319 );
1320 }
1321 let registrations = self.deregister_connection(connection_id);
1322 if let Ok(released_routes) = self.forwarding.cleanup_connection(connection_id) {
1323 self.emit_route_goodbyes(released_routes);
1324 }
1325 if matches!(®istrations, Ok(r) if !r.is_empty()) {
1329 crate::supervise::notify_registration_release();
1330 self.capability_evaluator.wake_deadline_loop();
1331 self.refresh_capability_requirements();
1332 }
1333 self.supervisor.remove_spawn_subscribers(connection_id);
1334 registrations
1335 }
1336
1337 pub(crate) fn handle_route_goodbye(
1338 &self,
1339 connection_id: ConnectionId,
1340 route_channel: u16,
1341 route_epoch: u32,
1342 ) -> Result<bool, RouterError> {
1343 debug!(
1344 connection_id = connection_id.get(),
1345 route_channel, route_epoch, "handling route GOODBYE"
1346 );
1347 let RouteRelease::Removed(released_route) = self
1348 .forwarding
1349 .release_client_route(connection_id, route_channel, route_epoch)
1350 .map_err(RouterError::Forwarding)?
1351 else {
1352 return Ok(false);
1353 };
1354 self.emit_route_goodbyes(vec![released_route]);
1355 Ok(true)
1356 }
1357
1358 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
1359 for released in released_routes {
1360 let frame = match Frame::build_with_version(
1361 released.negotiated_ver,
1362 FrameType::Goodbye,
1363 control_flags(),
1364 released.channel,
1365 released.epoch,
1366 0,
1367 Vec::new(),
1368 ) {
1369 Ok(frame) => frame,
1370 Err(err) => {
1371 warn!(
1372 route_channel = released.channel,
1373 error = %err,
1374 "failed to build route GOODBYE frame"
1375 );
1376 continue;
1377 }
1378 };
1379 if let Err(err) = released.sink.try_send(frame) {
1380 if released.close_on_delivery_failure() {
1381 warn!(
1382 target_connection_id = released.connection_id.get(),
1383 route_channel = released.channel,
1384 error = %err,
1385 "route GOODBYE was not delivered to client; closing target connection"
1386 );
1387 if self
1388 .forwarding
1389 .escalate_client_delivery_failure(
1390 released.connection_id,
1391 released.channel,
1392 released.epoch,
1393 CloseReason::new(
1394 "route_goodbye_delivery_failed",
1395 format!(
1396 "failed to enqueue route GOODBYE for channel {}: {err}",
1397 released.channel
1398 ),
1399 ),
1400 )
1401 .unwrap_or(false)
1402 {
1403 self.counters.increment_goodbye_relay_client_failed();
1404 }
1405 } else {
1406 self.counters
1407 .increment_goodbye_relay_module_dropped(released.module_id.as_deref());
1408 warn!(
1409 target_connection_id = released.connection_id.get(),
1410 route_channel = released.channel,
1411 error = %err,
1412 "route GOODBYE to module dropped under backpressure; not closing shared module connection"
1413 );
1414 }
1415 }
1416 }
1417 }
1418
1419 fn send_abandoned_route_bind_goodbye(
1431 &self,
1432 module_sink: &crate::FrameSink,
1433 negotiated_ver: u8,
1434 module_channel: u16,
1435 module_epoch: u32,
1436 ) {
1437 let frame = match Frame::build_with_version(
1438 negotiated_ver,
1439 FrameType::Goodbye,
1440 control_flags(),
1441 module_channel,
1442 module_epoch,
1443 0,
1444 Vec::new(),
1445 ) {
1446 Ok(frame) => frame,
1447 Err(err) => {
1448 warn!(
1449 route_channel = module_channel,
1450 error = %err,
1451 "failed to build GOODBYE for abandoned route.bind"
1452 );
1453 return;
1454 }
1455 };
1456 if let Err(err) = module_sink.try_send(frame) {
1457 warn!(
1458 route_channel = module_channel,
1459 error = %err,
1460 "GOODBYE for abandoned route.bind dropped; module idle reaper will reclaim the binding"
1461 );
1462 }
1463 }
1464
1465 fn handle_hello(
1466 &self,
1467 connection_id: ConnectionId,
1468 sink: Option<crate::FrameSink>,
1469 frame: Frame,
1470 ) -> Result<Vec<Frame>, RouterError> {
1471 debug!(
1472 connection_id = connection_id.get(),
1473 corr = frame.header.corr,
1474 "handling HELLO"
1475 );
1476 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
1477 Ok(value) => value,
1478 Err(err) => {
1479 return Ok(vec![control_error_frame(
1480 &frame,
1481 "invalid_hello",
1482 format!("malformed HELLO body: {err}"),
1483 )?])
1484 }
1485 };
1486 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
1487 return Ok(vec![control_error_frame(
1488 &frame,
1489 "invalid_capability_grammar",
1490 err.to_string(),
1491 )?]);
1492 }
1493 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
1494 return Ok(vec![control_error_frame(
1495 &frame,
1496 "invalid_manifest",
1497 err.to_string(),
1498 )?]);
1499 }
1500 if let Some(provenance) = hello_value
1501 .get("manifest")
1502 .and_then(|manifest| manifest.get("provenance"))
1503 {
1504 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
1505 return Ok(vec![control_error_frame(
1506 &frame,
1507 "invalid_manifest",
1508 format!("malformed manifest provenance: {err}"),
1509 )?]);
1510 }
1511 }
1512 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
1513 Ok(hello) => hello,
1514 Err(err) => {
1515 return Ok(vec![control_error_frame(
1516 &frame,
1517 "invalid_hello",
1518 format!("malformed HELLO body: {err}"),
1519 )?])
1520 }
1521 };
1522
1523 if hello.protocol_ver != hello.manifest.protocol_ver {
1524 return Ok(vec![control_error_frame(
1525 &frame,
1526 "invalid_manifest",
1527 format!(
1528 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
1529 hello.protocol_ver, hello.manifest.protocol_ver
1530 ),
1531 )?]);
1532 }
1533
1534 if hello.manifest.module_id.trim().is_empty() {
1535 return Ok(vec![control_error_frame(
1536 &frame,
1537 "invalid_manifest",
1538 "manifest module_id must not be empty",
1539 )?]);
1540 }
1541
1542 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
1543 Ok(negotiated_ver) => negotiated_ver,
1544 Err(message) => {
1545 return Ok(vec![control_error_frame(
1546 &frame,
1547 "version_unsupported",
1548 message,
1549 )?])
1550 }
1551 };
1552
1553 let swap_admission = self
1561 .supervisor
1562 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
1563 if swap_admission == SwapHelloAdmission::Refused {
1564 warn!(
1565 module_id = %hello.manifest.module_id,
1566 connection_id = connection_id.get(),
1567 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
1568 );
1569 return Ok(vec![control_error_frame(
1570 &frame,
1571 "swap_token_invalid",
1572 format!(
1573 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
1574 hello.manifest.module_id
1575 ),
1576 )?]);
1577 }
1578 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
1579
1580 if let Some(rejection) = (!swap_candidate)
1588 .then(|| {
1589 self.supervisor.reserved_hello_rejection(
1590 &hello.manifest.module_id,
1591 hello.launch_nonce.as_deref(),
1592 )
1593 })
1594 .flatten()
1595 {
1596 let message = match rejection {
1597 ReservedHelloRejection::Exact { module_id } => format!(
1598 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
1599 ),
1600 ReservedHelloRejection::Prefix {
1601 prefix,
1602 owner_module_id,
1603 } => format!(
1604 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
1605 hello.manifest.module_id
1606 ),
1607 };
1608 return Ok(vec![control_error_frame(
1609 &frame,
1610 "reserved_module",
1611 message,
1612 )?]);
1613 }
1614
1615 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
1616 &hello.manifest.module_id,
1617 hello.manifest.capabilities.as_ref(),
1618 );
1619 if let Some(refusal) = reserved_capability_refusals.first() {
1620 let capability = refusal.capability.clone();
1621 let bound_module = refusal.claimants[0].clone();
1622 log_duplicate_claim_events(reserved_capability_refusals);
1623 return Ok(vec![control_error_frame(
1624 &frame,
1625 "reserved_capability",
1626 format!(
1627 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
1628 capability, bound_module, hello.manifest.module_id
1629 ),
1630 )?]);
1631 }
1632
1633 if self
1636 .forwarding
1637 .connection_has_client_routes(connection_id)
1638 .map_err(RouterError::Forwarding)?
1639 {
1640 return Ok(vec![control_error_frame(
1641 &frame,
1642 "invalid_hello",
1643 "connection has open client routes and cannot also register as a module",
1644 )?]);
1645 }
1646
1647 let control_ops = effective_module_control_ops(hello.control_ops);
1648 if swap_candidate {
1649 return self.register_swap_candidate(
1650 connection_id,
1651 sink,
1652 &frame,
1653 hello.manifest,
1654 negotiated_ver,
1655 control_ops,
1656 );
1657 }
1658 let registration = match self.registry.register_with_control_ops(
1659 hello.manifest,
1660 negotiated_ver,
1661 connection_id,
1662 control_ops,
1663 ) {
1664 Ok(registration) => registration,
1665 Err(RegistryError::DuplicateModuleId { module_id }) => {
1666 return Ok(vec![control_error_frame(
1667 &frame,
1668 "duplicate_module_id",
1669 format!(
1670 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
1671 ),
1672 )?])
1673 }
1674 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
1675 return Ok(vec![control_error_frame(
1676 &frame,
1677 "invalid_module_id",
1678 err.to_string(),
1679 )?])
1680 }
1681 Err(err) => {
1682 return Ok(vec![control_error_frame(
1683 &frame,
1684 "registry_error",
1685 err.to_string(),
1686 )?])
1687 }
1688 };
1689
1690 if let Some(sink) = sink {
1691 let concurrency = manifest_concurrency(®istration.manifest);
1700 if let Err(err) = self.forwarding.register_module_connection(
1701 connection_id,
1702 registration.manifest.module_id.clone(),
1703 negotiated_ver,
1704 concurrency,
1705 sink,
1706 ) {
1707 if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
1711 crate::supervise::notify_registration_release();
1712 }
1713 return Ok(vec![control_error_frame(
1714 &frame,
1715 forwarding_error_code(&err),
1716 err.to_string(),
1717 )?]);
1718 }
1719 }
1720
1721 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
1730 info!(
1731 module_id = %registration.manifest.module_id,
1732 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
1733 );
1734 }
1735
1736 self.apply_registration_capabilities(®istration);
1737
1738 info!(
1739 module_id = %registration.manifest.module_id,
1740 module_version = %registration.manifest.module_version,
1741 negotiated_ver,
1742 routable_provider = manifest_provides_routable_role(®istration.manifest),
1743 connection_id = connection_id.get(),
1744 "module registered"
1745 );
1746
1747 self.hello_ack(&frame, negotiated_ver, ®istration.manifest.module_id)
1748 }
1749
1750 fn register_swap_candidate(
1760 &self,
1761 connection_id: ConnectionId,
1762 sink: Option<crate::FrameSink>,
1763 frame: &Frame,
1764 manifest: ModuleManifest,
1765 negotiated_ver: u8,
1766 control_ops: Vec<String>,
1767 ) -> Result<Vec<Frame>, RouterError> {
1768 let module_id = manifest.module_id.clone();
1769 let registration = match self.registry.register_candidate_with_control_ops(
1770 manifest,
1771 negotiated_ver,
1772 connection_id,
1773 control_ops,
1774 ) {
1775 Ok(registration) => registration,
1776 Err(RegistryError::DuplicateModuleId { module_id }) => {
1777 return Ok(vec![control_error_frame(
1778 frame,
1779 "duplicate_module_id",
1780 format!(
1781 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
1782 ),
1783 )?])
1784 }
1785 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
1786 return Ok(vec![control_error_frame(
1787 frame,
1788 "invalid_module_id",
1789 err.to_string(),
1790 )?])
1791 }
1792 Err(err) => {
1793 return Ok(vec![control_error_frame(
1794 frame,
1795 "registry_error",
1796 err.to_string(),
1797 )?])
1798 }
1799 };
1800 if let Some(sink) = sink {
1801 let concurrency = manifest_concurrency(®istration.manifest);
1802 if let Err(err) = self.forwarding.register_candidate_module_connection(
1803 connection_id,
1804 module_id.clone(),
1805 negotiated_ver,
1806 concurrency,
1807 sink,
1808 ) {
1809 if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
1810 crate::supervise::notify_registration_release();
1811 }
1812 return Ok(vec![control_error_frame(
1813 frame,
1814 forwarding_error_code(&err),
1815 err.to_string(),
1816 )?]);
1817 }
1818 }
1819 self.supervisor.mark_swap_candidate_admitted(&module_id);
1820 info!(
1821 module_id = %module_id,
1822 module_version = %registration.manifest.module_version,
1823 negotiated_ver,
1824 ready = registration.ready,
1825 connection_id = connection_id.get(),
1826 "swap candidate registered; not routable until cutover"
1827 );
1828 self.hello_ack(frame, negotiated_ver, &module_id)
1829 }
1830
1831 fn hello_ack(
1832 &self,
1833 frame: &Frame,
1834 negotiated_ver: u8,
1835 module_id: &str,
1836 ) -> Result<Vec<Frame>, RouterError> {
1837 let ack = ModuleHelloAckBody {
1838 negotiated_ver,
1839 subc_ops: module_subc_ops(),
1840 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
1841 storage: self
1842 .storage_config
1843 .as_ref()
1844 .map(|cfg| cfg.descriptor_for(module_id)),
1845 };
1846 let body = serde_json::to_vec(&ack).map_err(|err| {
1847 RouterError::backend(
1848 0,
1849 frame.header.corr,
1850 format!("failed to encode HELLO_ACK: {err}"),
1851 )
1852 })?;
1853
1854 Ok(vec![Frame::build_with_version(
1855 negotiated_ver,
1856 FrameType::HelloAck,
1857 control_flags(),
1858 0,
1859 0,
1860 frame.header.corr,
1861 body,
1862 )
1863 .map_err(RouterError::FrameBuild)?])
1864 }
1865
1866 async fn handle_client_control_request(
1867 &self,
1868 ctx: &RouteCtx,
1869 frame: Frame,
1870 request: ClientControlRequest,
1871 ) -> Result<Vec<Frame>, RouterError> {
1872 match request {
1873 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
1874 ClientControlRequest::CatalogList { module_id } => {
1875 self.handle_catalog_list(frame, module_id)
1876 }
1877 ClientControlRequest::RouteOpen {
1878 target,
1879 identity,
1880 consumer_identity,
1881 consumer_capabilities,
1882 admission_facts,
1883 } => {
1884 self.handle_route_open(
1885 ctx,
1886 frame,
1887 RouteOpenRequest {
1888 target,
1889 identity,
1890 consumer_identity,
1891 consumer_capabilities,
1892 admission_facts,
1893 },
1894 )
1895 .await
1896 }
1897 ClientControlRequest::RoutePoll {
1898 route_channel,
1899 route_epoch,
1900 kind,
1901 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
1902 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame),
1903 ClientControlRequest::SupervisorSpawnSnapshot {} => {
1904 self.handle_supervisor_spawn_snapshot(frame)
1905 }
1906 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
1907 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
1908 }
1909 ClientControlRequest::SupervisorRestart {
1910 module_id,
1911 drain_timeout_ms,
1912 } => {
1913 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms)
1914 .await
1915 }
1916 ClientControlRequest::SupervisorSwap {
1917 module_id,
1918 ready_timeout_ms,
1919 } => {
1920 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms)
1921 .await
1922 }
1923 ClientControlRequest::SupervisorReload { module_id } => {
1924 self.handle_supervisor_reload(frame, module_id).await
1925 }
1926 ClientControlRequest::SupervisorRescan { preview } => {
1927 self.handle_supervisor_rescan(frame, preview).await
1928 }
1929 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
1930 self.handle_supervisor_release_reserved(frame, module_id)
1931 .await
1932 }
1933 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
1934 self.handle_supervisor_set_enabled(frame, module_id, enabled)
1935 .await
1936 }
1937 ClientControlRequest::SupervisorHealthProbe { module_id } => {
1938 self.handle_supervisor_health_probe(frame, module_id).await
1939 }
1940 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
1941 ClientControlRequest::SupervisorRoutes { module_id } => {
1942 self.handle_supervisor_routes(frame, module_id)
1943 }
1944 ClientControlRequest::SupervisorProvenance { module_id } => {
1945 self.handle_supervisor_provenance(frame, module_id).await
1946 }
1947 ClientControlRequest::SupervisorStderrTail {
1948 module_id,
1949 max_lines,
1950 max_bytes,
1951 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
1952 ClientControlRequest::SupervisorTerminals { module_id } => {
1953 self.handle_supervisor_terminals(frame, module_id)
1954 }
1955 }
1956 }
1957
1958 fn handle_module_control_request(
1959 &self,
1960 connection_id: ConnectionId,
1961 frame: Frame,
1962 request: ModuleControlRequestFromModule,
1963 ) -> Result<Vec<Frame>, RouterError> {
1964 match request {
1965 ModuleControlRequestFromModule::CatalogUpdate {
1966 provides,
1967 capabilities,
1968 ready,
1969 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
1970 ModuleControlRequestFromModule::LiveRoots {} => {
1971 let registered = self
1972 .registry
1973 .get_module_by_connection(connection_id)
1974 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
1975 let Some(registration) = registered else {
1976 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
1977 };
1978 let response = self
1979 .forwarding
1980 .live_roots(®istration.manifest.module_id)
1981 .map_err(RouterError::Forwarding)?;
1982 Ok(vec![control_response_body_frame(
1983 &frame,
1984 &response,
1985 "ModuleControlResponseToModule::LiveRoots",
1986 )?])
1987 }
1988 }
1989 }
1990
1991 fn handle_catalog_update(
1992 &self,
1993 connection_id: ConnectionId,
1994 frame: Frame,
1995 provides: Vec<ProviderRole>,
1996 capabilities: Option<CapabilityDeclarations>,
1997 ready: Option<bool>,
1998 ) -> Result<Vec<Frame>, RouterError> {
1999 self.refresh_capability_requirements();
2000 let Some(registration) = self
2001 .registry
2002 .get_module_by_connection(connection_id)
2003 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2004 else {
2005 return Ok(vec![control_error_frame(
2006 &frame,
2007 "not_registered",
2008 "catalog.update requires an active module registration owned by this connection",
2009 )?]);
2010 };
2011
2012 if let Some(message) =
2013 catalog_update_frozen_field_message(®istration.manifest, &provides)
2014 {
2015 return Ok(vec![control_error_frame(
2016 &frame,
2017 "catalog_update_frozen_field",
2018 message,
2019 )?]);
2020 }
2021
2022 let mut candidate = registration.manifest.clone();
2023 candidate.provides = provides.clone();
2024 candidate.capabilities = capabilities
2025 .clone()
2026 .or_else(|| registration.manifest.capabilities.clone());
2027 if let Err(err) = candidate.validate_capability_grammar() {
2028 return Ok(vec![control_error_frame(
2029 &frame,
2030 "invalid_capability_grammar",
2031 err.to_string(),
2032 )?]);
2033 }
2034
2035 let updated = self
2036 .registry
2037 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
2038 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2039 if updated.is_none() {
2040 return Ok(vec![control_error_frame(
2041 &frame,
2042 "not_registered",
2043 "catalog.update requires an active module registration owned by this connection",
2044 )?]);
2045 }
2046 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
2047 log_duplicate_claim_events(
2048 self.capability_evaluator
2049 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
2050 );
2051 }
2052 if capability_census_trigger(
2053 registration.manifest.capabilities.as_ref(),
2054 updated
2055 .as_ref()
2056 .and_then(|entry| entry.manifest.capabilities.as_ref()),
2057 ) {
2058 self.enforce_capability_denies();
2059 }
2060 self.refresh_capability_requirements();
2061
2062 let response = ModuleControlResponseToModule::CatalogUpdate {};
2063 control_response_body_frame(
2064 &frame,
2065 &response,
2066 "ModuleControlResponseToModule::CatalogUpdate",
2067 )
2068 .map(|frame| vec![frame])
2069 }
2070
2071 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
2072 self.refresh_capability_requirements();
2073 let mut counters = self.counters.snapshot();
2079 if let (Ok((connections_with_routes, max)), Some(obj)) = (
2080 self.forwarding.client_route_concentration(),
2081 counters.as_object_mut(),
2082 ) {
2083 obj.insert(
2084 "client_connections_with_routes".into(),
2085 connections_with_routes.into(),
2086 );
2087 obj.insert("max_routes_on_one_connection".into(), max.into());
2088 }
2089 if let (Some(open_breakers), Some(obj)) = (
2095 self.route_bind_breakers.open_snapshot(),
2096 counters.as_object_mut(),
2097 ) {
2098 obj.insert("route_bind_breakers_open".into(), open_breakers);
2099 }
2100 let response = ClientControlResponse::ServerDescribe {
2101 protocol_ver: PROTOCOL_VERSION,
2102 subc_ops: subc_ops(),
2103 capabilities: self.subc_capabilities.as_ref().to_vec(),
2104 connected_clients: self.connected_clients.count(),
2105 counters: Some(counters),
2106 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
2107 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
2108 capability_requirements: self.capability_requirement_statuses(),
2109 };
2110 Ok(vec![control_response_body_frame(
2111 &frame,
2112 &response,
2113 "ClientControlResponse::ServerDescribe",
2114 )?])
2115 }
2116
2117 fn handle_catalog_list(
2118 &self,
2119 frame: Frame,
2120 module_id: Option<String>,
2121 ) -> Result<Vec<Frame>, RouterError> {
2122 let (generation, modules) = self.registry.list_modules().map_err(|err| {
2123 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
2124 })?;
2125 let entries = modules
2126 .into_iter()
2127 .filter(|registration| {
2128 module_id
2129 .as_deref()
2130 .map(|wanted| registration.manifest.module_id == wanted)
2131 .unwrap_or(true)
2132 })
2133 .map(|registration| {
2134 let roles = registration.manifest.provides;
2135 CatalogEntry {
2136 module_id: registration.manifest.module_id,
2137 ready: registration.ready,
2138 module_version: Some(registration.manifest.module_version),
2139 roles,
2140 control_ops: registration.control_ops,
2141 capabilities: registration.manifest.capabilities,
2142 self_signals: registration.manifest.self_signals,
2143 }
2144 })
2145 .collect();
2146 let response = ClientControlResponse::CatalogList {
2147 generation,
2148 modules: entries,
2149 subc_ops: subc_ops(),
2150 };
2151 Ok(vec![control_response_body_frame(
2152 &frame,
2153 &response,
2154 "ClientControlResponse::CatalogList",
2155 )?])
2156 }
2157
2158 fn route_open_principal(
2159 &self,
2160 frame: &Frame,
2161 consumer_identity: Option<ConsumerIdentity>,
2162 ) -> Result<Result<Principal, Frame>, RouterError> {
2163 let Some(consumer_identity) = consumer_identity else {
2164 return Ok(Ok(Principal::Direct));
2165 };
2166
2167 if self.supervisor.spawned_consumer_authorized(
2168 &consumer_identity.module_id,
2169 &consumer_identity.launch_nonce,
2170 ) {
2171 return Ok(Ok(Principal::Reserved {
2172 module_id: consumer_identity.module_id,
2173 }));
2174 }
2175
2176 Ok(Err(control_error_frame(
2177 frame,
2178 "bad_consumer_identity",
2179 format!(
2180 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
2181 consumer_identity.module_id
2182 ),
2183 )?))
2184 }
2185
2186 fn route_open_refusal_frame(
2192 &self,
2193 ctx: &RouteCtx,
2194 frame: &Frame,
2195 module_id: &str,
2196 reason: &'static str,
2197 code: &'static str,
2198 message: impl Into<String>,
2199 ) -> Result<Frame, RouterError> {
2200 self.observe_route_open_refusal(ctx, module_id, reason, code);
2201 control_error_frame(frame, code, message.into())
2202 }
2203
2204 fn route_open_breaker_refusal_frame(
2222 &self,
2223 ctx: &RouteCtx,
2224 frame: &Frame,
2225 module_id: &str,
2226 consecutive_timeouts: u32,
2227 retry_in: Duration,
2228 probe_in_flight: bool,
2229 ) -> Result<Frame, RouterError> {
2230 self.counters
2231 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
2232 debug!(
2233 target: "control",
2234 code = "module_timeout",
2235 module_id = ?module_id,
2236 connection_id = ctx.connection_id.get(),
2237 consecutive_timeouts,
2238 retry_in_ms = retry_in.as_millis() as u64,
2239 probe_in_flight,
2240 "route.open refused by open bind-relay breaker"
2241 );
2242 let detail = if probe_in_flight {
2243 "one probe bind is already in flight; retry once it settles".to_string()
2244 } else {
2245 format!("not relaying for another {retry_in:?}")
2246 };
2247 control_error_frame(
2248 frame,
2249 "module_timeout",
2250 format!(
2251 "module_id '{module_id}' failed {consecutive_timeouts} consecutive route.bind \
2252 relays; {detail}"
2253 ),
2254 )
2255 }
2256
2257 fn observe_route_open_refusal(
2270 &self,
2271 ctx: &RouteCtx,
2272 module_id: &str,
2273 reason: &'static str,
2274 code: &'static str,
2275 ) {
2276 self.counters.increment_route_open_refused(code);
2277 info!(
2278 target: "control",
2279 code,
2280 reason,
2281 module_id = ?module_id,
2282 connection_id = ctx.connection_id.get(),
2283 "route.open refused"
2284 );
2285 }
2286
2287 fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
2332 self.counters.increment_route_open_accepted(principal);
2333 info!(
2334 target: "control",
2335 principal,
2336 module_id,
2337 connection_id = ctx.connection_id.get(),
2338 "route.open accepted"
2339 );
2340 }
2341
2342 fn supervised_absent_route_open_refusal_frame(
2343 &self,
2344 ctx: &RouteCtx,
2345 frame: &Frame,
2346 module_id: &str,
2347 code: &'static str,
2348 status: &crate::supervise::ModuleStatus,
2349 ) -> Result<Frame, RouterError> {
2350 self.counters.increment_route_open_refused(code);
2351 info!(
2352 target: "control",
2353 code,
2354 reason = "supervised_not_registered",
2355 module_id = ?module_id,
2356 connection_id = ctx.connection_id.get(),
2357 state = %status.state,
2358 enabled = status.enabled,
2359 live = status.live,
2360 "route.open refused"
2361 );
2362 control_error_frame(
2363 frame,
2364 code,
2365 format!(
2366 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
2367 status.state, status.enabled, status.live
2368 ),
2369 )
2370 }
2371
2372 async fn handle_route_open(
2373 &self,
2374 ctx: &RouteCtx,
2375 frame: Frame,
2376 request: RouteOpenRequest,
2377 ) -> Result<Vec<Frame>, RouterError> {
2378 let RouteOpenRequest {
2379 target,
2380 mut identity,
2381 consumer_identity,
2382 consumer_capabilities,
2383 admission_facts,
2384 } = request;
2385 let target_module_id = target_module_id(&target).to_string();
2386 debug!(
2387 connection_id = ctx.connection_id.get(),
2388 corr = frame.header.corr,
2389 module_id = %target_module_id,
2390 "handling route.open"
2391 );
2392
2393 let Some(registration) = self
2412 .registry
2413 .get_module(&target_module_id)
2414 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2415 else {
2416 if let Some((status, warming)) =
2417 self.supervisor_status(&target_module_id, frame.header.corr)?
2418 {
2419 if status.protocol == ModuleProtocol::None {
2426 return Ok(vec![self.route_open_refusal_frame(
2427 ctx,
2428 &frame,
2429 &target_module_id,
2430 "protocol_none",
2431 error_codes::MODULE_NO_PROTOCOL,
2432 format!(
2433 "module_id '{target_module_id}' is declared protocol: none; \
2434 it speaks no subc wire and serves no routes"
2435 ),
2436 )?]);
2437 }
2438 let code = if warming {
2439 "module_warming"
2440 } else {
2441 "target_unavailable"
2442 };
2443 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
2444 ctx,
2445 &frame,
2446 &target_module_id,
2447 code,
2448 &status,
2449 )?]);
2450 }
2451 if let Some(removed_ago_ms) =
2452 self.supervisor.removal_tombstone_age_ms(&target_module_id)
2453 {
2454 return Ok(vec![self.route_open_refusal_frame(
2455 ctx,
2456 &frame,
2457 &target_module_id,
2458 "removed",
2459 error_codes::MODULE_REMOVED,
2460 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
2461 )?]);
2462 }
2463 return Ok(vec![self.route_open_refusal_frame(
2464 ctx,
2465 &frame,
2466 &target_module_id,
2467 "not_registered",
2468 error_codes::UNKNOWN_MODULE,
2469 format!("module_id '{target_module_id}' is not registered"),
2470 )?]);
2471 };
2472
2473 if !registration.ready {
2477 self.counters
2478 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
2479 info!(
2480 target: "control",
2481 code = error_codes::MODULE_WARMING,
2482 module_id = ?target_module_id,
2483 connection_id = ctx.connection_id.get(),
2484 reason = "declared_not_ready",
2485 "route.open refused"
2486 );
2487 return Ok(vec![control_error_body_frame(
2488 &frame,
2489 ErrorBody {
2490 code: error_codes::MODULE_WARMING.to_string(),
2491 message: format!(
2492 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
2493 ),
2494 detail: Some(serde_json::json!({
2495 "reason": "declared_not_ready"
2496 })),
2497 },
2498 )?]);
2499 }
2500
2501 if !target_has_required_role(&target, ®istration.manifest.provides) {
2502 return Ok(vec![self.route_open_refusal_frame(
2503 ctx,
2504 &frame,
2505 &target_module_id,
2506 "role_not_provided",
2507 "target_unavailable",
2508 format!("module_id '{target_module_id}' does not provide the requested target"),
2509 )?]);
2510 }
2511
2512 if registration.state != ChannelState::Active {
2513 return Ok(vec![self.route_open_refusal_frame(
2514 ctx,
2515 &frame,
2516 &target_module_id,
2517 "registration_not_active",
2518 "target_unavailable",
2519 format!("module_id '{target_module_id}' is not active"),
2520 )?]);
2521 }
2522
2523 if self
2524 .forwarding
2525 .module_is_draining(&target_module_id)
2526 .map_err(RouterError::Forwarding)?
2527 {
2528 return Ok(vec![self.route_open_refusal_frame(
2529 ctx,
2530 &frame,
2531 &target_module_id,
2532 "reloading",
2533 "module_reloading",
2534 format!("module_id '{target_module_id}' is reloading"),
2535 )?]);
2536 }
2537
2538 if self
2539 .process_liveness
2540 .as_ref()
2541 .and_then(|process_liveness| process_liveness.process_live(&target_module_id))
2542 == Some(false)
2543 {
2544 return Ok(vec![self.route_open_refusal_frame(
2545 ctx,
2546 &frame,
2547 &target_module_id,
2548 "supervisor_not_live",
2549 "target_unavailable",
2550 format!("module_id '{target_module_id}' is not live"),
2551 )?]);
2552 }
2553
2554 if !self
2555 .forwarding
2556 .has_live_module_connection(&target_module_id)
2557 .map_err(RouterError::Forwarding)?
2558 {
2559 return Ok(vec![self.route_open_refusal_frame(
2560 ctx,
2561 &frame,
2562 &target_module_id,
2563 "no_forwarding_connection",
2564 "target_unavailable",
2565 format!("module_id '{target_module_id}' has no live forwarding connection"),
2566 )?]);
2567 }
2568
2569 if let Some(error) =
2570 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
2571 {
2572 self.observe_route_open_refusal(
2573 ctx,
2574 &target_module_id,
2575 "op_not_allowed",
2576 "op_not_allowed",
2577 );
2578 return Ok(vec![error]);
2579 }
2580
2581 let principal = match self.route_open_principal(&frame, consumer_identity)? {
2582 Ok(principal) => principal,
2583 Err(error) => {
2584 self.observe_route_open_refusal(
2585 ctx,
2586 &target_module_id,
2587 "bad_consumer_identity",
2588 "bad_consumer_identity",
2589 );
2590 return Ok(vec![error]);
2591 }
2592 };
2593
2594 if let Principal::Reserved {
2598 module_id: opening_module_id,
2599 } = &principal
2600 {
2601 if let Some(opening_registration) = self
2602 .registry
2603 .get_module(opening_module_id)
2604 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2605 {
2606 if let Some(capability) =
2607 denied_capability(&opening_registration.manifest, ®istration.manifest)
2608 {
2609 warn!(
2610 opening_module_id,
2611 target_module_id,
2612 capability,
2613 "refusing route.open because an attested capability deny edge matches"
2614 );
2615 return Ok(vec![self.route_open_refusal_frame(
2616 ctx,
2617 &frame,
2618 &target_module_id,
2619 "capability_deny_edge",
2620 "capability_forbidden",
2621 format!(
2622 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
2623 ),
2624 )?]);
2625 }
2626 }
2627 }
2628
2629 if admission_facts.is_some() {
2630 let carrier_matches = matches!(
2631 &principal,
2632 Principal::Reserved { module_id }
2633 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
2634 );
2635 if !carrier_matches {
2636 return Ok(vec![self.route_open_refusal_frame(
2637 ctx,
2638 &frame,
2639 &target_module_id,
2640 "admission_facts_carrier_not_permitted",
2641 "admission_facts_not_permitted",
2642 "admission facts may only be carried by the configured reserved module",
2643 )?]);
2644 }
2645
2646 let target_allowed = self
2647 .admission_facts_targets
2648 .as_ref()
2649 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
2650 if !target_allowed {
2651 return Ok(vec![self.route_open_refusal_frame(
2652 ctx,
2653 &frame,
2654 &target_module_id,
2655 "admission_facts_target_not_listed",
2656 "admission_facts_target_not_allowed",
2657 format!(
2658 "admission facts are not permitted for target module_id '{target_module_id}'"
2659 ),
2660 )?]);
2661 }
2662
2663 }
2667
2668 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
2690 Ok(project_root) => project_root,
2691 Err(err) => {
2692 return Ok(vec![control_error_frame(
2693 &frame,
2694 "invalid_project_root",
2695 err.to_string(),
2696 )?])
2697 }
2698 };
2699 identity.project_root = project_root.as_path().to_path_buf();
2700
2701 let _concurrency_guard = match self
2712 .route_bind_concurrency
2713 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
2714 {
2715 Ok(guard) => guard,
2716 Err(in_flight) => {
2717 return Ok(vec![self.route_open_admission_refusal_frame(
2718 ctx,
2719 &frame,
2720 &target_module_id,
2721 format!(
2722 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
2723 ),
2724 )?]);
2725 }
2726 };
2727
2728 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
2731 RouteBindAdmission::Admitted { guard, probe } => {
2732 if probe {
2733 info!(
2734 module_id = %target_module_id,
2735 connection_id = ctx.connection_id.get(),
2736 "route.bind breaker half-open: admitting one probe"
2737 );
2738 }
2739 guard
2740 }
2741 RouteBindAdmission::Refused {
2742 consecutive_timeouts,
2743 retry_in,
2744 probe_in_flight,
2745 } => {
2746 return Ok(vec![self.route_open_breaker_refusal_frame(
2747 ctx,
2748 &frame,
2749 &target_module_id,
2750 consecutive_timeouts,
2751 retry_in,
2752 probe_in_flight,
2753 )?]);
2754 }
2755 };
2756
2757 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
2762 let relay_deadline = Instant::now() + route_bind_relay_timeout;
2763 let pending = match self
2764 .forwarding
2765 .begin_route_bind_relay_for(
2766 ctx.connection_id,
2767 ctx.egress.clone(),
2768 response_version(&frame),
2769 frame.header.corr,
2770 &target_module_id,
2771 principal.clone(),
2772 Some(project_root),
2773 relay_deadline,
2774 )
2775 .await
2776 {
2777 Ok(pending) => pending,
2778 Err(err) => {
2779 return Ok(vec![self.route_open_refusal_frame(
2780 ctx,
2781 &frame,
2782 &target_module_id,
2783 "relay_reservation_failed",
2784 forwarding_error_code(&err),
2785 err.to_string(),
2786 )?])
2787 }
2788 };
2789 let crate::forwarding::PendingRouteBindRelay {
2790 endpoint,
2791 module_sink,
2792 negotiated_ver,
2793 client_channel,
2794 client_epoch,
2795 module_channel,
2796 module_epoch,
2797 corr: relay_corr,
2798 receiver,
2799 } = pending;
2800 let mut reservation =
2801 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
2802
2803 debug!(
2804 connection_id = ctx.connection_id.get(),
2805 client_channel,
2806 client_epoch,
2807 module_channel,
2808 module_epoch,
2809 "reserved route handle pair"
2810 );
2811 let principal_label = match &principal {
2814 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
2815 Principal::Direct => "direct".to_string(),
2816 other => format!("{other:?}"),
2817 };
2818 let relay = ModuleControlRequest::RouteBind {
2819 route_channel: module_channel,
2820 epoch: module_epoch,
2821 target,
2822 identity,
2823 principal: Some(principal),
2824 consumer_capabilities,
2825 admission_facts,
2826 };
2827 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
2828 RouterError::backend(
2829 0,
2830 frame.header.corr,
2831 format!("failed to encode route.bind request: {err}"),
2832 )
2833 })?;
2834 let relay_frame = Frame::build_with_version(
2835 negotiated_ver,
2836 FrameType::Request,
2837 control_flags(),
2838 0,
2839 0,
2840 relay_corr,
2841 relay_body,
2842 )
2843 .map_err(RouterError::FrameBuild)?;
2844
2845 if let Err(err) = module_sink.send(relay_frame).await {
2846 reservation.release_and_disarm();
2847 return Ok(vec![self.route_open_refusal_frame(
2848 ctx,
2849 &frame,
2850 &target_module_id,
2851 "relay_send_failed",
2852 "target_unavailable",
2853 err.to_string(),
2854 )?]);
2855 }
2856
2857 if !self
2858 .forwarding
2859 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
2860 .map_err(RouterError::Forwarding)?
2861 {
2862 self.send_abandoned_route_bind_goodbye(
2863 &module_sink,
2864 negotiated_ver,
2865 module_channel,
2866 module_epoch,
2867 );
2868 }
2869
2870 match timeout_at(relay_deadline, receiver).await {
2871 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
2872 reservation.disarm();
2873 if breaker.record_accepted() {
2874 info!(
2875 module_id = %target_module_id,
2876 "route.bind breaker closed: the probe was accepted"
2877 );
2878 }
2879 self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
2880 Ok(Vec::new())
2881 }
2882 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
2883 reservation.release_and_disarm();
2884 breaker.record_inconclusive();
2888 self.counters
2889 .increment_route_open_refused("module_rejected");
2890 info!(
2891 target: "control",
2892 code = "module_rejected",
2893 module_code = ?body.code,
2894 module_id = ?target_module_id,
2895 connection_id = ctx.connection_id.get(),
2896 "route.open refused"
2897 );
2898 Ok(vec![control_error_body_frame(&frame, body)?])
2899 }
2900 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
2901 reservation.release_and_disarm();
2902 breaker.record_inconclusive();
2903 tracing::warn!(
2910 module_id = %target_module_id,
2911 "route.bind relay abandoned: {message}"
2912 );
2913 Ok(vec![self.route_open_refusal_frame(
2914 ctx,
2915 &frame,
2916 &target_module_id,
2917 "relay_abandoned",
2918 "target_unavailable",
2919 message,
2920 )?])
2921 }
2922 Ok(Err(_)) => {
2923 reservation.release_and_disarm();
2924 breaker.record_inconclusive();
2925 Ok(vec![self.route_open_refusal_frame(
2926 ctx,
2927 &frame,
2928 &target_module_id,
2929 "relay_waiter_canceled",
2930 "target_unavailable",
2931 "route.bind relay waiter was canceled before the module responded",
2932 )?])
2933 }
2934 Err(_) => {
2935 reservation.release_and_disarm();
2936 if let Some(opened) = breaker.record_timeout(
2940 self.route_bind_breaker_threshold,
2941 self.route_bind_breaker_cooldown,
2942 ) {
2943 warn!(
2944 module_id = %target_module_id,
2945 consecutive_timeouts = opened.consecutive_timeouts,
2946 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
2947 reopened_after_probe = opened.reopened_after_probe,
2948 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
2949 );
2950 }
2951 tracing::warn!(
2959 module_id = %target_module_id,
2960 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
2961 "route.bind relay timed out: module did not ack within budget"
2962 );
2963 Ok(vec![self.route_open_refusal_frame(
2964 ctx,
2965 &frame,
2966 &target_module_id,
2967 "relay_timed_out",
2968 "module_timeout",
2969 format!(
2970 "module_id '{target_module_id}' did not answer route.bind within {:?}",
2971 route_bind_relay_timeout
2972 ),
2973 )?])
2974 }
2975 }
2976 }
2977
2978 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
2979 let response = ClientControlResponse::SupervisorSpawnSnapshot {
2980 snapshot: self.supervisor.spawn_snapshot(),
2981 };
2982 Ok(vec![control_response_body_frame(
2983 &frame,
2984 &response,
2985 "ClientControlResponse::SupervisorSpawnSnapshot",
2986 )?])
2987 }
2988
2989 fn handle_supervisor_spawn_subscribe(
2990 &self,
2991 ctx: &RouteCtx,
2992 frame: Frame,
2993 since: Option<SpawnCursor>,
2994 ) -> Result<Vec<Frame>, RouterError> {
2995 match self.supervisor.subscribe_spawns(
2996 ctx.connection_id,
2997 frame.header.corr,
2998 response_version(&frame),
2999 since,
3000 ctx.egress.clone(),
3001 ) {
3002 Ok(()) => Ok(Vec::new()),
3003 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
3004 Ok(vec![control_error_body_frame(
3005 &frame,
3006 ErrorBody {
3007 code: "spawn_cursor_incarnation_mismatch".to_string(),
3008 message: "spawn cursor belongs to a different daemon incarnation"
3009 .to_string(),
3010 detail: Some(serde_json::json!({
3011 "current_daemon_incarnation": current
3012 })),
3013 },
3014 )?])
3015 }
3016 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
3017 &frame,
3018 ErrorBody {
3019 code: "spawn_cursor_too_old".to_string(),
3020 message: "spawn cursor predates the retained event ring".to_string(),
3021 detail: Some(serde_json::json!({
3022 "oldest_retained_cursor": oldest
3023 })),
3024 },
3025 )?]),
3026 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
3027 0,
3028 frame.header.corr,
3029 format!("failed to open supervisor spawn subscription: {error}"),
3030 )),
3031 }
3032 }
3033
3034 fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3035 let generation = self
3036 .registry
3037 .generation()
3038 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3039 let modules = self
3040 .supervisor
3041 .list()
3042 .into_iter()
3043 .map(|module| {
3044 let status = module.status_for_control("list").map_err(|err| {
3045 RouterError::backend(
3046 0,
3047 frame.header.corr,
3048 format!("failed to read supervisor status: {err}"),
3049 )
3050 })?;
3051 Ok(SupervisorEntry {
3052 module_id: status.module_id,
3053 state: status.state.to_string(),
3054 enabled: status.enabled,
3055 live: status.live,
3056 protocol: status.protocol,
3057 health: status.health.status,
3058 last_probe_ms: status.health.last_probe_ms,
3059 last_exit_code: status.last_exit.as_ref().and_then(|e| e.code),
3060 last_exit_signal: status.last_exit.as_ref().and_then(|e| e.signal),
3061 last_exit_ms: status.last_exit.as_ref().map(|e| e.at_ms),
3062 last_exit_kind: status.last_exit.as_ref().map(|e| e.kind.into()),
3063 restart_count: Some(status.restart_count),
3064 max_restarts: Some(status.max_restarts),
3065 lifetime_restarts: Some(status.lifetime_restarts),
3066 spawn_generation: Some(status.spawn_generation),
3067 restart_window_secs: Some(status.restart_window.as_secs()),
3068 drain_timeout_ms: Some(status.drain_timeout.as_millis() as u64),
3069 restart_backoff_ms: Some(status.restart_backoff.as_millis() as u64),
3070 restart_max_backoff_ms: Some(status.restart_max_backoff.as_millis() as u64),
3071 })
3072 })
3073 .collect::<Result<Vec<_>, RouterError>>()?;
3074 let response = ClientControlResponse::SupervisorList {
3075 generation,
3076 modules,
3077 };
3078 Ok(vec![control_response_body_frame(
3079 &frame,
3080 &response,
3081 "ClientControlResponse::SupervisorList",
3082 )?])
3083 }
3084
3085 fn handle_supervisor_stderr_tail(
3086 &self,
3087 frame: Frame,
3088 module_id: String,
3089 max_lines: Option<u32>,
3090 max_bytes: Option<u32>,
3091 ) -> Result<Vec<Frame>, RouterError> {
3092 let Some(module) = self.supervisor.get(&module_id) else {
3093 return Ok(vec![control_error_frame(
3094 &frame,
3095 "unknown_module",
3096 format!("module_id '{module_id}' is not supervised"),
3097 )?]);
3098 };
3099
3100 let snapshot = module.stderr_tail(
3101 max_lines.map(|value| value as usize),
3102 max_bytes.map(|value| value as usize),
3103 );
3104
3105 let response = ClientControlResponse::SupervisorStderrTail {
3106 module_id,
3107 tail: StderrTail {
3108 capture: match snapshot.capture {
3109 CaptureState::Captured => StderrCaptureState::Captured,
3110 CaptureState::Incomplete { reason } => {
3111 StderrCaptureState::Incomplete { reason }
3112 }
3113 CaptureState::NotCaptured { reason } => {
3114 StderrCaptureState::NotCaptured { reason }
3115 }
3116 },
3117 entries: snapshot
3118 .entries
3119 .into_iter()
3120 .map(|entry| match entry {
3121 TailEntry::Line { text, truncated } => {
3122 StderrTailEntry::Line { text, truncated }
3123 }
3124 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
3125 })
3126 .collect(),
3127 dropped_lines: snapshot.dropped_lines,
3128 },
3129 };
3130 Ok(vec![control_response_body_frame(
3131 &frame,
3132 &response,
3133 "ClientControlResponse::SupervisorStderrTail",
3134 )?])
3135 }
3136
3137 fn handle_supervisor_terminals(
3138 &self,
3139 frame: Frame,
3140 module_id: String,
3141 ) -> Result<Vec<Frame>, RouterError> {
3142 let Some(module) = self.supervisor.get(&module_id) else {
3143 return Ok(vec![control_error_frame(
3144 &frame,
3145 "unknown_module",
3146 format!("module_id '{module_id}' is not supervised"),
3147 )?]);
3148 };
3149
3150 let response = ClientControlResponse::SupervisorTerminals {
3151 module_id,
3152 terminals: module.durable_terminal_history(),
3153 };
3154 Ok(vec![control_response_body_frame(
3155 &frame,
3156 &response,
3157 "ClientControlResponse::SupervisorTerminals",
3158 )?])
3159 }
3160
3161 fn handle_supervisor_routes(
3162 &self,
3163 frame: Frame,
3164 module_id: Option<String>,
3165 ) -> Result<Vec<Frame>, RouterError> {
3166 let modules = self
3167 .forwarding
3168 .route_census(module_id.as_deref())
3169 .map_err(RouterError::Forwarding)?
3170 .into_iter()
3171 .map(|(module_id, routes)| SupervisorRouteModule {
3172 module_id,
3173 routes: routes
3174 .into_iter()
3175 .map(|route| SupervisorRoute {
3176 consumer: match route.principal {
3177 Principal::Reserved { module_id } => {
3178 SupervisorRouteConsumer::Reserved { module_id }
3179 }
3180 Principal::Direct | Principal::Unverified => {
3181 SupervisorRouteConsumer::Direct {
3182 connection_id: route.goodbye_target.connection_id.get(),
3183 }
3184 }
3185 },
3186 age_ms: Instant::now()
3187 .saturating_duration_since(route.bound_at)
3188 .as_millis()
3189 .try_into()
3190 .unwrap_or(u64::MAX),
3191 draining: route.draining,
3192 drain_reason: route.drain_reason,
3193 })
3194 .collect(),
3195 })
3196 .collect();
3197 let response = ClientControlResponse::SupervisorRoutes { modules };
3198 Ok(vec![control_response_body_frame(
3199 &frame,
3200 &response,
3201 "ClientControlResponse::SupervisorRoutes",
3202 )?])
3203 }
3204
3205 async fn handle_supervisor_provenance(
3206 &self,
3207 frame: Frame,
3208 module_id: Option<String>,
3209 ) -> Result<Vec<Frame>, RouterError> {
3210 let mut selected = if let Some(module_id) = module_id {
3211 let Some(module) = self.supervisor.get(&module_id) else {
3212 return Ok(vec![control_error_frame(
3213 &frame,
3214 "unknown_module",
3215 format!("module_id '{module_id}' is not supervised"),
3216 )?]);
3217 };
3218 vec![module]
3219 } else {
3220 self.supervisor.list()
3221 };
3222
3223 let mut modules = Vec::with_capacity(selected.len());
3224 for module in selected.drain(..) {
3225 let status = module.status().map_err(|err| {
3226 RouterError::backend(
3227 0,
3228 frame.header.corr,
3229 format!("failed to read supervisor status: {err}"),
3230 )
3231 })?;
3232 let module_declared = self
3233 .registry
3234 .get_module(&status.module_id)
3235 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3236 .and_then(|registration| registration.manifest.provenance)
3237 .map(|build| ModuleDeclaredProvenance::Reported { build })
3238 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
3239 #[cfg(test)]
3240 let running_image = match &self.provenance_probe_override {
3241 Some(result) => result.clone(),
3242 None => module.running_image_agreement().await,
3243 };
3244 #[cfg(not(test))]
3245 let running_image = module.running_image_agreement().await;
3246 modules.push(SupervisorModuleProvenance {
3247 module_id: status.module_id,
3248 module_declared,
3249 daemon_observed: SupervisorObservedProcess {
3250 pid: status.pid,
3251 spawned_at_ms: status.spawned_at_ms,
3252 spawned_from: status.spawned_from,
3253 running_image,
3254 },
3255 });
3256 }
3257 let daemon = SupervisorDaemonProvenance {
3258 daemon_build: self.daemon_provenance.build.clone(),
3259 daemon_observed: DaemonObservedProcess {
3260 pid: self.daemon_provenance.pid,
3261 started_at_ms: self
3262 .daemon_provenance
3263 .start_clock
3264 .map(|clock| clock.started_at_ms())
3265 .or(self.daemon_provenance.started_at_ms),
3266 running_image: self
3267 .daemon_provenance
3268 .probe
3269 .observe(
3270 self.daemon_provenance.pid,
3271 self.daemon_provenance.executable_path.as_deref(),
3272 self.daemon_provenance.executable_identity,
3273 self.daemon_provenance.process_start_time,
3274 )
3275 .await,
3276 },
3277 };
3278 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
3279 Ok(vec![control_response_body_frame(
3280 &frame,
3281 &response,
3282 "ClientControlResponse::SupervisorProvenance",
3283 )?])
3284 }
3285
3286 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3287 self.refresh_capability_requirements();
3288 let generation = self
3289 .registry
3290 .generation()
3291 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3292 let modules = self
3293 .supervisor
3294 .list()
3295 .into_iter()
3296 .map(|module| {
3297 let status = module.status_for_control("health").map_err(|err| {
3298 RouterError::backend(
3299 0,
3300 frame.header.corr,
3301 format!("failed to read supervisor health: {err}"),
3302 )
3303 })?;
3304 let module_id = status.module_id;
3305 let capability_detail = self
3306 .capability_evaluator
3307 .required_problem_detail(&module_id);
3308 Ok(SupervisorHealthEntry {
3309 module_id,
3310 status: status.health.status,
3311 detail: append_capability_problem_detail(
3312 status.health.detail,
3313 capability_detail,
3314 ),
3315 metrics: status.health.metrics,
3316 consecutive_failures: status.health.consecutive_failures,
3317 late_answer_count: status.health.late_answer_count,
3318 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
3319 last_action: status.health.last_action,
3320 last_action_ms: status.health.last_action_ms,
3321 last_probe_ms: status.health.last_probe_ms,
3322 })
3323 })
3324 .collect::<Result<Vec<_>, RouterError>>()?;
3325 let response = ClientControlResponse::SupervisorHealth {
3326 generation,
3327 modules,
3328 };
3329 Ok(vec![control_response_body_frame(
3330 &frame,
3331 &response,
3332 "ClientControlResponse::SupervisorHealth",
3333 )?])
3334 }
3335
3336 async fn handle_supervisor_restart(
3337 &self,
3338 frame: Frame,
3339 module_id: String,
3340 drain_timeout_ms: Option<u64>,
3341 ) -> Result<Vec<Frame>, RouterError> {
3342 let operation_lock = self.supervisor.operation_lock();
3343 let _operation_guard = operation_lock.lock().await;
3344 let Some(module) = self.supervisor.get(&module_id) else {
3345 return Ok(vec![control_error_frame(
3346 &frame,
3347 "unknown_module",
3348 format!("module_id '{module_id}' is not supervised"),
3349 )?]);
3350 };
3351
3352 if let Err(err) = module.restart(drain_timeout_ms).await {
3353 let (code, message) = match err {
3354 crate::supervise::SuperviseError::Disabled { .. } => {
3355 ("module_disabled", err.to_string())
3356 }
3357 crate::supervise::SuperviseError::SwapInProgress { .. } => {
3358 ("swap_in_progress", err.to_string())
3359 }
3360 _ => (
3361 "target_unavailable",
3362 format!("failed to restart module_id '{module_id}': {err}"),
3363 ),
3364 };
3365 return Ok(vec![control_error_frame(&frame, code, message)?]);
3366 }
3367
3368 let response = ClientControlResponse::SupervisorAck {
3369 module_id,
3370 applied: true,
3371 };
3372 Ok(vec![control_response_body_frame(
3373 &frame,
3374 &response,
3375 "ClientControlResponse::SupervisorAck",
3376 )?])
3377 }
3378
3379 async fn handle_supervisor_swap(
3383 &self,
3384 frame: Frame,
3385 module_id: String,
3386 ready_timeout_ms: Option<u64>,
3387 ) -> Result<Vec<Frame>, RouterError> {
3388 let module = {
3395 let operation_lock = self.supervisor.operation_lock();
3396 let _operation_guard = operation_lock.lock().await;
3397 self.supervisor.get(&module_id)
3398 };
3399 let Some(module) = module else {
3400 return Ok(vec![control_error_frame(
3401 &frame,
3402 "unknown_module",
3403 format!("module_id '{module_id}' is not supervised"),
3404 )?]);
3405 };
3406
3407 if let Err(err) = module
3408 .swap(ready_timeout_ms.map(Duration::from_millis))
3409 .await
3410 {
3411 use crate::supervise::SuperviseError;
3412 let message = err.to_string();
3413 let error = match err {
3414 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
3415 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
3416 code: "swap_refused".to_string(),
3417 message,
3418 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
3419 },
3420 SuperviseError::SwapFailed {
3421 arm,
3422 candidate_exit,
3423 ..
3424 } => ErrorBody {
3425 code: "swap_failed".to_string(),
3426 message,
3427 detail: Some(serde_json::json!({
3428 "arm": arm.as_str(),
3429 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
3430 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
3431 })),
3432 },
3433 _ => ErrorBody::new(
3434 "target_unavailable",
3435 format!("failed to swap module_id '{module_id}': {message}"),
3436 ),
3437 };
3438 return Ok(vec![control_error_body_frame(&frame, error)?]);
3439 }
3440
3441 let response = ClientControlResponse::SupervisorAck {
3442 module_id,
3443 applied: true,
3444 };
3445 Ok(vec![control_response_body_frame(
3446 &frame,
3447 &response,
3448 "ClientControlResponse::SupervisorAck",
3449 )?])
3450 }
3451
3452 async fn handle_supervisor_reload(
3453 &self,
3454 frame: Frame,
3455 module_id: String,
3456 ) -> Result<Vec<Frame>, RouterError> {
3457 let operation_lock = self.supervisor.operation_lock();
3458 let _operation_guard = operation_lock.lock().await;
3459 let Some(module) = self.supervisor.get(&module_id) else {
3460 return Ok(vec![control_error_frame(
3461 &frame,
3462 "unknown_module",
3463 format!("module_id '{module_id}' is not supervised"),
3464 )?]);
3465 };
3466
3467 if let Err(err) = module.reload().await {
3468 let (code, message) = match err {
3469 crate::supervise::SuperviseError::Disabled { .. } => {
3470 ("module_disabled", err.to_string())
3471 }
3472 crate::supervise::SuperviseError::SwapInProgress { .. } => {
3473 ("swap_in_progress", err.to_string())
3474 }
3475 _ => (
3476 "reload_failed",
3477 format!("failed to reload module_id '{module_id}': {err}"),
3478 ),
3479 };
3480 return Ok(vec![control_error_frame(&frame, code, message)?]);
3481 }
3482
3483 let response = ClientControlResponse::SupervisorAck {
3484 module_id,
3485 applied: true,
3486 };
3487 Ok(vec![control_response_body_frame(
3488 &frame,
3489 &response,
3490 "ClientControlResponse::SupervisorAck",
3491 )?])
3492 }
3493
3494 async fn handle_supervisor_rescan(
3495 &self,
3496 frame: Frame,
3497 preview: bool,
3498 ) -> Result<Vec<Frame>, RouterError> {
3499 let Some(context) = self.rescan.clone() else {
3500 return Ok(vec![control_error_frame(
3501 &frame,
3502 "rescan_unavailable",
3503 "the daemon was not started with a reloadable config path".to_string(),
3504 )?]);
3505 };
3506
3507 let operation_lock = self.supervisor.operation_lock();
3508 let _operation_guard = operation_lock.lock().await;
3509 let loaded = match crate::daemon_config::load(&context.config_path) {
3510 Ok(config) => config,
3511 Err(err) => {
3512 return Ok(vec![control_error_frame(
3513 &frame,
3514 "invalid_daemon_config",
3515 format!("supervisor rescan rejected daemon config: {err}"),
3516 )?])
3517 }
3518 };
3519 let Some(config) = loaded else {
3527 return Ok(vec![control_error_frame(
3528 &frame,
3529 "invalid_daemon_config",
3530 format!(
3531 "daemon config not found at {}; refusing to rescan (an absent config would \
3532 retire every supervised module)",
3533 context.config_path.display()
3534 ),
3535 )?]);
3536 };
3537 let (
3538 configured_port,
3539 storage_config,
3540 admission_facts_carrier_module_id,
3541 admission_facts_targets,
3542 modules,
3543 reserved_capabilities,
3544 ) = (
3545 config.port,
3546 config.storage,
3547 config.admission_facts_carrier_module_id,
3548 config.admission_facts_targets,
3549 config.modules,
3550 config.reserved_capabilities,
3551 );
3552
3553 let mut restart_required = Vec::new();
3562 for section in RestartRequiredSection::ALL {
3563 let changed = match section {
3564 RestartRequiredSection::Port => configured_port != context.configured_port,
3565 RestartRequiredSection::Storage => storage_config != context.storage_config,
3566 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
3567 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
3568 }
3569 RestartRequiredSection::AdmissionFactsTargets => {
3570 admission_facts_targets != context.admission_facts_targets
3571 }
3572 };
3573 if changed {
3574 restart_required.push(section.label().to_string());
3575 }
3576 }
3577 if !restart_required.is_empty() {
3578 warn!(
3579 config_path = %context.config_path.display(),
3580 sections = %restart_required.join(", "),
3581 "daemon config changed outside the modules section; restart the daemon to apply those changes"
3582 );
3583 }
3584
3585 for configured in &modules {
3586 if let Err(err) = validate_spec(&configured.module_spec()) {
3587 return Ok(vec![control_error_frame(
3588 &frame,
3589 "invalid_daemon_config",
3590 format!("supervisor rescan rejected daemon config: {err}"),
3591 )?]);
3592 }
3593 }
3594
3595 let configured_capabilities = modules
3596 .iter()
3597 .map(|module| (module.module_id.clone(), module.enabled))
3598 .collect::<Vec<_>>();
3599 let preview_capability_warnings = if preview {
3600 let (_, registrations) = self.runtime_capability_snapshot()?;
3601 let current_modules = self
3602 .supervisor
3603 .list()
3604 .into_iter()
3605 .map(|module| module.module_id().to_string())
3606 .collect::<BTreeSet<_>>();
3607 let resulting_modules = configured_capabilities.clone();
3608 let removed = current_modules
3609 .into_iter()
3610 .filter(|module_id| {
3611 !resulting_modules
3612 .iter()
3613 .any(|(configured_id, _)| configured_id == module_id)
3614 })
3615 .collect::<Vec<_>>();
3616 self.capability_evaluator.preview_removal_warnings(
3617 resulting_modules,
3618 &removed,
3619 ®istrations,
3620 )
3621 } else {
3622 Vec::new()
3623 };
3624 let result = match self
3625 .reconcile_supervised_modules(&context.supervisor, modules, preview)
3626 .await
3627 {
3628 Ok(result) => result,
3629 Err(message) => {
3630 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
3631 }
3632 };
3633 if !preview {
3634 self.capability_evaluator
3635 .configure(configured_capabilities, reserved_capabilities);
3636 self.capability_evaluator.wake_deadline_loop();
3637 self.refresh_capability_requirements();
3638 }
3639 let mut result = result;
3640 result.restart_required = restart_required;
3641 result.capability_warnings = preview_capability_warnings;
3642 let response = ClientControlResponse::SupervisorRescan { result };
3643 Ok(vec![control_response_body_frame(
3644 &frame,
3645 &response,
3646 "ClientControlResponse::SupervisorRescan",
3647 )?])
3648 }
3649
3650 async fn handle_supervisor_release_reserved(
3651 &self,
3652 frame: Frame,
3653 module_id: String,
3654 ) -> Result<Vec<Frame>, RouterError> {
3655 let Some(context) = self.rescan.clone() else {
3656 return Ok(vec![control_error_frame(
3657 &frame,
3658 "release_unavailable",
3659 "reserved-id release requires a daemon started with a reloadable config path",
3660 )?]);
3661 };
3662 let operation_lock = self.supervisor.operation_lock();
3663 let _operation_guard = operation_lock.lock().await;
3664 let loaded = match crate::daemon_config::load(&context.config_path) {
3665 Ok(Some(config)) => config,
3666 Ok(None) => {
3667 return Ok(vec![control_error_frame(
3668 &frame,
3669 "invalid_daemon_config",
3670 format!(
3671 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
3672 context.config_path.display()
3673 ),
3674 )?])
3675 }
3676 Err(err) => {
3677 return Ok(vec![control_error_frame(
3678 &frame,
3679 "invalid_daemon_config",
3680 format!("unable to verify reserved-id release against daemon config: {err}"),
3681 )?])
3682 }
3683 };
3684 if loaded
3685 .modules
3686 .iter()
3687 .any(|configured| configured.module_id == module_id)
3688 {
3689 return Ok(vec![control_error_frame(
3690 &frame,
3691 "reserved_module_configured",
3692 format!(
3693 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
3694 ),
3695 )?]);
3696 }
3697 if !self.supervisor.release_retained_reserved_gate(&module_id) {
3698 return Ok(vec![control_error_frame(
3699 &frame,
3700 "reserved_gate_not_retained",
3701 format!(
3702 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
3703 ),
3704 )?]);
3705 }
3706
3707 let response = ClientControlResponse::SupervisorAck {
3708 module_id,
3709 applied: true,
3710 };
3711 Ok(vec![control_response_body_frame(
3712 &frame,
3713 &response,
3714 "ClientControlResponse::SupervisorAck",
3715 )?])
3716 }
3717
3718 async fn reconcile_supervised_modules(
3727 &self,
3728 supervisor: &Supervisor,
3729 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
3730 preview: bool,
3731 ) -> Result<SupervisorRescanResult, String> {
3732 let mut current = BTreeMap::new();
3733 for module in self.supervisor.list() {
3734 let (spec, health) = module.configuration().map_err(|err| {
3735 format!(
3736 "failed to read configuration for module_id '{}': {err}",
3737 module.module_id()
3738 )
3739 })?;
3740 let enabled = module
3741 .status()
3742 .map_err(|err| {
3743 format!(
3744 "failed to read status for module_id '{}': {err}",
3745 module.module_id()
3746 )
3747 })?
3748 .enabled;
3749 current.insert(
3750 module.module_id().to_string(),
3751 (module, spec, health, enabled),
3752 );
3753 }
3754 let configured = configured_modules
3755 .into_iter()
3756 .map(|module| (module.module_id.clone(), module))
3757 .collect::<BTreeMap<_, _>>();
3758
3759 let added = configured
3760 .keys()
3761 .filter(|module_id| !current.contains_key(*module_id))
3762 .cloned()
3763 .collect::<Vec<_>>();
3764 let removed = current
3765 .keys()
3766 .filter(|module_id| !configured.contains_key(*module_id))
3767 .cloned()
3768 .collect::<Vec<_>>();
3769 let mut changed_pending_reload = Vec::new();
3770 let mut configuration_changes = BTreeSet::new();
3771 let mut enabled_changes = BTreeSet::new();
3772 let mut unchanged = 0_u32;
3773
3774 for (module_id, configured_module) in &configured {
3775 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
3776 else {
3777 continue;
3778 };
3779 let configuration_changed = *current_spec != configured_module.module_spec()
3780 || *current_health != configured_module.health;
3781 let enabled_changed = *current_enabled != configured_module.enabled;
3782 if configuration_changed {
3783 configuration_changes.insert(module_id.clone());
3784 changed_pending_reload.push(module_id.clone());
3785 }
3786 if enabled_changed {
3787 enabled_changes.insert(module_id.clone());
3788 }
3789 if !configuration_changed && !enabled_changed {
3790 unchanged = unchanged.saturating_add(1);
3791 }
3792 }
3793
3794 if preview {
3800 return Ok(SupervisorRescanResult {
3801 added,
3802 removed,
3803 changed_pending_reload,
3804 enabled_changes: enabled_changes.iter().cloned().collect(),
3805 unchanged,
3806 preview: true,
3807 restart_required: Vec::new(),
3811 capability_warnings: Vec::new(),
3812 });
3813 }
3814
3815 for module_id in &removed {
3816 let module = ¤t
3817 .get(module_id)
3818 .expect("removed module came from current supervisor state")
3819 .0;
3820 module.retire().await.map_err(|err| {
3821 format!("failed to retire module_id '{module_id}' during rescan: {err}")
3822 })?;
3823 self.supervisor.record_rescan_removal(module_id);
3857 self.supervisor.retire(module_id);
3858 }
3859
3860 for module_id in configured.keys() {
3861 let Some((module, _, _, _)) = current.get(module_id) else {
3862 continue;
3863 };
3864 let configured_module = configured
3865 .get(module_id)
3866 .expect("configured module id came from configured map");
3867 if configuration_changes.contains(module_id) {
3868 module
3869 .update_configuration(
3870 configured_module.module_spec(),
3871 configured_module.health,
3872 configured_module.drain_timeout_ms,
3873 )
3874 .await
3875 .map_err(|err| {
3876 format!(
3877 "failed to update module_id '{module_id}' configuration during rescan: {err}"
3878 )
3879 })?;
3880 }
3881 if enabled_changes.contains(module_id) {
3882 module
3883 .set_enabled(configured_module.enabled)
3884 .await
3885 .map_err(|err| {
3886 format!(
3887 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
3888 configured_module.enabled
3889 )
3890 })?;
3891 }
3892 }
3893
3894 for module_id in &added {
3895 let configured_module = configured
3896 .get(module_id)
3897 .expect("added module id came from configured map");
3898 supervisor
3899 .supervise_configured_with_health(
3900 configured_module.module_spec(),
3901 configured_module.enabled,
3902 configured_module.health,
3903 configured_module.drain_timeout_ms,
3904 configured_module.restart,
3905 )
3906 .map_err(|err| {
3907 format!("failed to add module_id '{module_id}' during rescan: {err}")
3908 })?;
3909 }
3910
3911 Ok(SupervisorRescanResult {
3912 added,
3913 removed,
3914 changed_pending_reload,
3915 enabled_changes: enabled_changes.iter().cloned().collect(),
3916 unchanged,
3917 preview: false,
3918 restart_required: Vec::new(),
3921 capability_warnings: Vec::new(),
3922 })
3923 }
3924
3925 async fn handle_supervisor_set_enabled(
3926 &self,
3927 frame: Frame,
3928 module_id: String,
3929 enabled: bool,
3930 ) -> Result<Vec<Frame>, RouterError> {
3931 let operation_lock = self.supervisor.operation_lock();
3932 let _operation_guard = operation_lock.lock().await;
3933 let Some(module) = self.supervisor.get(&module_id) else {
3934 return Ok(vec![control_error_frame(
3935 &frame,
3936 "unknown_module",
3937 format!("module_id '{module_id}' is not supervised"),
3938 )?]);
3939 };
3940
3941 let applied = match module.set_enabled(enabled).await {
3942 Ok(applied) => applied,
3943 Err(err) => {
3944 return Ok(vec![control_error_frame(
3945 &frame,
3946 "target_unavailable",
3947 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
3948 )?])
3949 }
3950 };
3951
3952 self.capability_evaluator.wake_deadline_loop();
3953 self.refresh_capability_requirements();
3954 let response = ClientControlResponse::SupervisorAck { module_id, applied };
3955 Ok(vec![control_response_body_frame(
3956 &frame,
3957 &response,
3958 "ClientControlResponse::SupervisorAck",
3959 )?])
3960 }
3961
3962 async fn handle_supervisor_health_probe(
3963 &self,
3964 frame: Frame,
3965 module_id: String,
3966 ) -> Result<Vec<Frame>, RouterError> {
3967 self.refresh_capability_requirements();
3968 let Some(registration) = self
3969 .registry
3970 .get_module(&module_id)
3971 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3972 else {
3973 return Ok(vec![control_error_frame(
3974 &frame,
3975 "unknown_module",
3976 format!("module_id '{module_id}' is not registered"),
3977 )?]);
3978 };
3979
3980 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
3993 {
3994 return Ok(vec![control_error_frame(
3995 &frame,
3996 "health_not_advertised",
3997 format!("module_id '{module_id}' did not advertise health.check"),
3998 )?]);
3999 }
4000
4001 let deadline = Instant::now() + self.health_probe_timeout;
4002 let pending = match self.forwarding.begin_module_control_rpc_for(
4003 &module_id,
4004 MODULE_CONTROL_OP_HEALTH_CHECK,
4005 deadline,
4006 ) {
4007 Ok(pending) => pending,
4008 Err(err) => {
4009 return Ok(vec![control_error_frame(
4010 &frame,
4011 forwarding_error_code(&err),
4012 err.to_string(),
4013 )?])
4014 }
4015 };
4016
4017 let PendingModuleControlRpc {
4018 endpoint,
4019 module_sink,
4020 negotiated_ver,
4021 corr: probe_corr,
4022 receiver,
4023 } = pending;
4024 let mut guard =
4025 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
4026 let probe_body =
4027 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
4028 RouterError::backend(
4029 0,
4030 frame.header.corr,
4031 format!("failed to encode health.check request: {err}"),
4032 )
4033 })?;
4034 let probe_frame = Frame::build_with_version(
4035 negotiated_ver,
4036 FrameType::Request,
4037 control_flags(),
4038 0,
4039 0,
4040 probe_corr,
4041 probe_body,
4042 )
4043 .map_err(RouterError::FrameBuild)?;
4044
4045 if let Err(err) = module_sink.send(probe_frame).await {
4046 return Ok(vec![control_error_frame(
4047 &frame,
4048 "target_unavailable",
4049 err.to_string(),
4050 )?]);
4051 }
4052
4053 match timeout_at(deadline, receiver).await {
4054 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
4055 guard.disarm();
4056 let Some(report) = response.health_report() else {
4057 return Ok(vec![control_error_frame(
4058 &frame,
4059 "invalid_control_body",
4060 "health.check RPC returned a non-health response",
4061 )?]);
4062 };
4063 let HealthReport {
4068 status,
4069 detail,
4070 metrics,
4071 } = report;
4072 let capability_detail = self
4073 .capability_evaluator
4074 .required_problem_detail(&module_id);
4075 let response = ClientControlResponse::SupervisorHealthProbe {
4076 module_id,
4077 status,
4078 detail: append_capability_problem_detail(detail, capability_detail),
4079 metrics,
4080 };
4081 Ok(vec![control_response_body_frame(
4082 &frame,
4083 &response,
4084 "ClientControlResponse::SupervisorHealthProbe",
4085 )?])
4086 }
4087 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
4088 guard.disarm();
4089 Ok(vec![control_error_body_frame(&frame, body)?])
4090 }
4091 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
4092 guard.disarm();
4093 Ok(vec![control_error_frame(
4094 &frame,
4095 "target_unavailable",
4096 message,
4097 )?])
4098 }
4099 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
4100 guard.disarm();
4101 Ok(vec![control_error_frame(
4102 &frame,
4103 "invalid_control_body",
4104 message,
4105 )?])
4106 }
4107 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
4108 guard.disarm();
4109 Ok(vec![control_error_frame(
4110 &frame,
4111 "invalid_control_body",
4112 format!("expected module-control op '{expected}', got '{actual}'"),
4113 )?])
4114 }
4115 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
4116 guard.disarm();
4117 Ok(vec![control_error_frame(
4118 &frame,
4119 "module_timeout",
4120 format!(
4121 "module_id '{module_id}' answered health.check after {:?}",
4122 self.health_probe_timeout
4123 ),
4124 )?])
4125 }
4126 Ok(Err(_)) => Ok(vec![control_error_frame(
4127 &frame,
4128 "target_unavailable",
4129 "health.check waiter was canceled before the module responded",
4130 )?]),
4131 Err(_) => Ok(vec![control_error_frame(
4132 &frame,
4133 "module_timeout",
4134 format!(
4135 "module_id '{module_id}' did not answer health.check within {:?}",
4136 self.health_probe_timeout
4137 ),
4138 )?]),
4139 }
4140 }
4141
4142 fn supervisor_status(
4143 &self,
4144 module_id: &str,
4145 corr: u64,
4146 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
4147 self.supervisor
4148 .get(module_id)
4149 .map(|module| {
4150 let warming = module.is_warming_for_control("status").map_err(|err| {
4151 RouterError::backend(
4152 0,
4153 corr,
4154 format!(
4155 "failed to read supervisor warming state for module_id '{module_id}': {err}"
4156 ),
4157 )
4158 })?;
4159 module.status_for_control("status").map_err(|err| {
4160 RouterError::backend(
4161 0,
4162 corr,
4163 format!(
4164 "failed to read supervisor status for module_id '{module_id}': {err}"
4165 ),
4166 )
4167 }).map(|status| (status, warming))
4168 })
4169 .transpose()
4170 }
4171
4172 fn guard_module_control_op(
4173 &self,
4174 frame: &Frame,
4175 module_id: &str,
4176 op: &str,
4177 ) -> Result<Option<Frame>, RouterError> {
4178 if self.module_grants_op(module_id, op, frame.header.corr)? {
4179 return Ok(None);
4180 }
4181
4182 Ok(Some(control_error_frame(
4183 frame,
4184 "op_not_allowed",
4185 format!("module_id '{module_id}' did not grant control op '{op}'"),
4186 )?))
4187 }
4188
4189 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
4190 let Some(registration) = self
4191 .registry
4192 .get_module(module_id)
4193 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
4194 else {
4195 return Ok(false);
4196 };
4197 Ok(module_registration_grants_op(®istration.control_ops, op))
4198 }
4199
4200 fn handle_status_update(
4201 &self,
4202 endpoint: ModuleEndpointId,
4203 frame: Frame,
4204 ) -> Result<Vec<Frame>, RouterError> {
4205 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
4206 Ok(update) => update,
4207 Err(err) => {
4208 if is_known_module_push_op(&frame.body) {
4213 return Ok(vec![control_error_frame(
4214 &frame,
4215 "invalid_control_body",
4216 format!("malformed module control push body: {err}"),
4217 )?]);
4218 }
4219 return Ok(Vec::new());
4220 }
4221 };
4222
4223 match update {
4224 ModuleControlPush::RouteStatus {
4225 route_channel,
4226 route_epoch,
4227 status,
4228 } => {
4229 self.forwarding
4230 .cache_status(endpoint, route_channel, route_epoch, status)
4231 .map_err(RouterError::Forwarding)?;
4232 }
4233 }
4234 Ok(Vec::new())
4235 }
4236
4237 fn handle_route_poll(
4238 &self,
4239 ctx: &RouteCtx,
4240 frame: Frame,
4241 route_channel: u16,
4242 route_epoch: u32,
4243 kind: PollKind,
4244 ) -> Result<Vec<Frame>, RouterError> {
4245 let snapshot = self
4246 .forwarding
4247 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
4248 .map_err(RouterError::Forwarding)?;
4249 let response = match (kind, snapshot) {
4250 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
4251 ClientControlResponse::RoutePoll {
4252 route_channel,
4253 route_epoch,
4254 status,
4255 live: None,
4256 }
4257 }
4258 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
4259 route_channel,
4260 route_epoch,
4261 status: None,
4262 live: None,
4263 },
4264 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
4265 let live = self
4291 .process_liveness
4292 .as_ref()
4293 .and_then(|source| source.process_live(&module_id))
4294 .unwrap_or(true);
4295 ClientControlResponse::RoutePoll {
4296 route_channel,
4297 route_epoch,
4298 status: None,
4299 live: Some(live),
4300 }
4301 }
4302 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
4303 route_channel,
4304 route_epoch,
4305 status: None,
4306 live: Some(false),
4307 },
4308 };
4309
4310 Ok(vec![control_response_body_frame(
4311 &frame,
4312 &response,
4313 "ClientControlResponse::RoutePoll",
4314 )?])
4315 }
4316
4317 pub(crate) fn observe_module_control_completion(
4318 &self,
4319 completion: ModuleControlRpcCompletion,
4320 ) -> bool {
4321 match completion {
4322 ModuleControlRpcCompletion::Unknown => false,
4323 ModuleControlRpcCompletion::Settled => true,
4324 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
4325 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
4326 info!(
4327 module_id = %module_id,
4328 latency_ms,
4329 "late health.check answer proves the module is alive"
4330 );
4331 match self
4332 .supervisor
4333 .record_late_health_answer(&module_id, latency_ms)
4334 {
4335 Ok(true) => {}
4336 Ok(false) => debug!(
4337 module_id = %module_id,
4338 latency_ms,
4339 "late health.check answer has no active supervisor snapshot"
4340 ),
4341 Err(err) => warn!(
4342 module_id = %module_id,
4343 latency_ms,
4344 error = %err,
4345 "failed to record late health.check answer"
4346 ),
4347 }
4348 true
4349 }
4350 }
4351 }
4352
4353 fn refuse_to_end_module_connection_for_a_client(
4374 &self,
4375 module_connection_id: ConnectionId,
4376 corr: u64,
4377 err: ForwardingError,
4378 ) -> Result<(), RouterError> {
4379 if let ForwardingError::ConnectionClosing { connection_id } = err {
4380 if connection_id != module_connection_id {
4381 warn!(
4382 module_connection_id = module_connection_id.get(),
4383 client_connection_id = connection_id.get(),
4384 corr,
4385 "dropping a route.bind response for a closing client; the module connection keeps serving"
4386 );
4387 return Ok(());
4388 }
4389 }
4390 Err(RouterError::Forwarding(err))
4391 }
4392
4393 fn handle_module_relay_response(
4394 &self,
4395 connection_id: ConnectionId,
4396 frame: Frame,
4397 ) -> Result<Vec<Frame>, RouterError> {
4398 let mut secondary_error = None;
4399 let outcome = match frame.header.ty {
4400 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
4401 Ok(probe) if probe.op == "route.bind" => {
4402 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
4403 Ok(ModuleControlResponse::RouteBindAck {}) => {
4404 RouteBindRelayOutcome::Accepted
4405 }
4406 Ok(other) => {
4407 let message =
4408 format!("route.bind response carried unexpected body: {other:?}");
4409 secondary_error = Some(control_error_frame(
4410 &frame,
4411 "invalid_control_body",
4412 message.clone(),
4413 )?);
4414 RouteBindRelayOutcome::ModuleGone(message)
4415 }
4416 Err(err) => {
4417 let message = format!("malformed route.bind response body: {err}");
4418 secondary_error = Some(control_error_frame(
4419 &frame,
4420 "invalid_control_body",
4421 message.clone(),
4422 )?);
4423 RouteBindRelayOutcome::ModuleGone(message)
4424 }
4425 }
4426 }
4427 Ok(probe) => {
4428 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
4429 {
4430 Ok(response) => ModuleControlRpcOutcome::Response(response),
4431 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
4432 "malformed {} response body: {err}",
4433 probe.op
4434 )),
4435 };
4436 let completion = self
4437 .forwarding
4438 .complete_module_control_rpc(
4439 connection_id,
4440 frame.header.corr,
4441 Some(&probe.op),
4442 outcome,
4443 )
4444 .map_err(RouterError::Forwarding)?;
4445 if !self.observe_module_control_completion(completion) {
4446 debug!(
4447 connection_id = connection_id.get(),
4448 corr = frame.header.corr,
4449 op = %probe.op,
4450 "dropping late or unknown module-control RPC response"
4451 );
4452 }
4453 return Ok(Vec::new());
4454 }
4455 Err(err) => {
4456 if let Some(expected_op) = self
4457 .forwarding
4458 .pending_module_control_op(connection_id, frame.header.corr)
4459 .map_err(RouterError::Forwarding)?
4460 {
4461 let completion = self
4462 .forwarding
4463 .complete_module_control_rpc(
4464 connection_id,
4465 frame.header.corr,
4466 None,
4467 ModuleControlRpcOutcome::MalformedResponse(format!(
4468 "malformed {expected_op} response body: {err}"
4469 )),
4470 )
4471 .map_err(RouterError::Forwarding)?;
4472 if !self.observe_module_control_completion(completion) {
4473 debug!(
4474 connection_id = connection_id.get(),
4475 corr = frame.header.corr,
4476 "dropping late malformed module-control RPC response"
4477 );
4478 }
4479 return Ok(Vec::new());
4480 }
4481 let message = format!("malformed route.bind response body: {err}");
4482 secondary_error = Some(control_error_frame(
4483 &frame,
4484 "invalid_control_body",
4485 message.clone(),
4486 )?);
4487 RouteBindRelayOutcome::ModuleGone(message)
4488 }
4489 },
4490 FrameType::Error => {
4491 if self
4492 .forwarding
4493 .pending_module_control_op(connection_id, frame.header.corr)
4494 .map_err(RouterError::Forwarding)?
4495 .is_some()
4496 {
4497 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
4498 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
4499 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
4500 "malformed module-control ERROR body: {err}"
4501 )),
4502 };
4503 let completion = self
4504 .forwarding
4505 .complete_module_control_rpc(
4506 connection_id,
4507 frame.header.corr,
4508 None,
4509 outcome,
4510 )
4511 .map_err(RouterError::Forwarding)?;
4512 if !self.observe_module_control_completion(completion) {
4513 debug!(
4514 connection_id = connection_id.get(),
4515 corr = frame.header.corr,
4516 "dropping late or unknown module-control RPC error"
4517 );
4518 }
4519 return Ok(Vec::new());
4520 }
4521 match serde_json::from_slice::<ErrorBody>(&frame.body) {
4522 Ok(body) => RouteBindRelayOutcome::Rejected(body),
4523 Err(err) => {
4524 let message = format!("malformed route.bind ERROR body: {err}");
4525 secondary_error = Some(control_error_frame(
4526 &frame,
4527 "invalid_control_body",
4528 message.clone(),
4529 )?);
4530 RouteBindRelayOutcome::ModuleGone(message)
4531 }
4532 }
4533 }
4534 ty => {
4535 return Ok(vec![control_error_frame(
4536 &frame,
4537 "unsupported_control_frame",
4538 format!("unsupported module channel-0 frame {ty:?}"),
4539 )?])
4540 }
4541 };
4542
4543 let settled =
4544 self.forwarding
4545 .complete_pending_relay(connection_id, frame.header.corr, outcome);
4546 let completion = match settled {
4547 Ok(completion) => completion,
4548 Err(err) => {
4549 self.refuse_to_end_module_connection_for_a_client(
4550 connection_id,
4551 frame.header.corr,
4552 err,
4553 )?;
4554 return Ok(secondary_error.into_iter().collect());
4555 }
4556 };
4557 if let Some(target) = completion.abandoned.as_ref() {
4558 send_goodbye_target_best_effort(target, "late accepted route.bind");
4559 }
4560 if !completion.settled {
4561 debug!(
4562 connection_id = connection_id.get(),
4563 corr = frame.header.corr,
4564 frame_type = ?frame.header.ty,
4565 "dropping late or unknown route.bind relay response"
4566 );
4567 }
4568 Ok(secondary_error.into_iter().collect())
4569 }
4570
4571 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
4572 debug!(connection_id = connection_id.get(), "handling GOODBYE");
4573 let registrations = self
4574 .deregister_connection(connection_id)
4575 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
4576 let released_routes = self
4577 .forwarding
4578 .cleanup_connection(connection_id)
4579 .map_err(RouterError::Forwarding)?;
4580 self.emit_route_goodbyes(released_routes);
4581 if !registrations.is_empty() {
4583 crate::supervise::notify_registration_release();
4584 }
4585 Ok(Vec::new())
4586 }
4587}
4588
4589impl Default for ControlHandler {
4590 fn default() -> Self {
4591 Self::new(Arc::new(Registry::default()))
4592 }
4593}
4594
4595impl crate::supervise::SwapPromotionObserver for ControlHandler {
4596 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
4597 self.apply_registration_capabilities(registration);
4598 }
4599}
4600
4601fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
4602 CapabilityRequirementStatus {
4603 consumer: status.consumer,
4604 capability: status.capability,
4605 need: match status.need {
4606 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
4607 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
4608 },
4609 verdict: status.verdict.as_str().to_string(),
4610 episode_seq: status.episode_seq,
4611 config_satisfiable: status.config_satisfiable,
4612 runtime_available: status.runtime_available,
4613 detail: status.detail,
4614 }
4615}
4616
4617fn append_capability_problem_detail(
4618 detail: Option<String>,
4619 capability_detail: Option<String>,
4620) -> Option<String> {
4621 match (detail, capability_detail) {
4622 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
4623 (Some(detail), None) => Some(detail),
4624 (None, Some(capability_detail)) => Some(capability_detail),
4625 (None, None) => None,
4626 }
4627}
4628
4629fn subc_ops() -> Vec<String> {
4630 SUBC_CONTROL_OPS
4631 .iter()
4632 .map(|op| (*op).to_string())
4633 .collect()
4634}
4635
4636fn module_subc_ops() -> Vec<String> {
4637 SUBC_CONTROL_OPS
4638 .iter()
4639 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
4640 .map(|op| (*op).to_string())
4641 .collect()
4642}
4643
4644#[cfg(test)]
4645fn module_baseline_control_ops() -> Vec<String> {
4646 MODULE_BASELINE_CONTROL_OPS
4647 .iter()
4648 .map(|op| (*op).to_string())
4649 .collect()
4650}
4651
4652fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
4653 let mut seen = HashSet::new();
4654 let mut effective = Vec::new();
4655 for op in MODULE_BASELINE_CONTROL_OPS {
4656 if seen.insert((*op).to_string()) {
4657 effective.push((*op).to_string());
4658 }
4659 }
4660 for op in declared.unwrap_or_default() {
4661 if seen.insert(op.clone()) {
4662 effective.push(op);
4663 }
4664 }
4665 effective
4666}
4667
4668fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
4669 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
4670}
4671
4672fn target_module_id(target: &RouteTarget) -> &str {
4673 match target {
4674 RouteTarget::ToolProvider { module_id }
4675 | RouteTarget::ManagementSurface { module_id }
4676 | RouteTarget::InternalService { module_id, .. } => module_id,
4677 }
4678}
4679
4680fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
4681 roles.iter().any(|role| match (target, role) {
4682 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
4683 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
4684 (
4685 RouteTarget::InternalService { service_id, .. },
4686 ProviderRole::InternalService {
4687 service_id: provided,
4688 ..
4689 },
4690 ) => service_id == provided,
4691 _ => false,
4692 })
4693}
4694
4695fn is_routable_role(role: &ProviderRole) -> bool {
4696 matches!(
4697 role,
4698 ProviderRole::ToolProvider { .. }
4699 | ProviderRole::ManagementSurface { .. }
4700 | ProviderRole::InternalService { .. }
4701 )
4702}
4703
4704#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4705enum ControlRequestBodyError {
4706 UnknownOp,
4707 InvalidBody,
4708}
4709
4710#[derive(Debug, Deserialize)]
4711struct ControlOpProbe {
4712 op: String,
4713}
4714
4715const MODULE_PUSH_OPS: &[&str] = &["route.status"];
4718
4719fn is_known_module_push_op(body: &[u8]) -> bool {
4720 serde_json::from_slice::<ControlOpProbe>(body)
4721 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
4722 .unwrap_or(false)
4723}
4724
4725fn is_known_module_request_op(body: &[u8]) -> bool {
4726 serde_json::from_slice::<ControlOpProbe>(body)
4727 .map(|probe| MODULE_TO_SUBC_CONTROL_OPS.contains(&probe.op.as_str()))
4728 .unwrap_or(false)
4729}
4730
4731fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
4732 debug!(
4733 op = %op,
4734 connection_id = connection_id.get(),
4735 corr,
4736 "control dispatch"
4737 );
4738}
4739
4740fn log_slow_control_dispatch(
4741 dispatch_started_at: Option<StdInstant>,
4742 op: &'static str,
4743 connection_id: ConnectionId,
4744 corr: u64,
4745) {
4746 let Some(dispatch_started_at) = dispatch_started_at else {
4747 return;
4748 };
4749 let elapsed = dispatch_started_at.elapsed();
4750 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
4751 warn!(
4752 op = %op,
4753 connection_id = connection_id.get(),
4754 corr,
4755 elapsed_ms = elapsed.as_millis() as u64,
4756 "slow control dispatch"
4757 );
4758 }
4759}
4760
4761fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
4762 match request {
4763 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
4764 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
4765 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
4766 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
4767 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
4768 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
4769 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
4770 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
4771 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
4772 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
4773 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
4774 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
4775 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
4776 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
4777 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
4778 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
4779 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
4780 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
4781 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
4782 }
4783}
4784
4785fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
4786 match request {
4787 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
4788 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
4789 }
4790}
4791
4792fn parse_client_control_request(
4793 body: &[u8],
4794) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
4795 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
4796 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
4797 Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
4798 ControlRequestBodyError::InvalidBody
4799 }
4800 Ok(_) => ControlRequestBodyError::UnknownOp,
4801 Err(_) => ControlRequestBodyError::InvalidBody,
4802 };
4803 (err, classification)
4804 })
4805}
4806
4807fn parse_module_control_request_from_module(
4808 body: &[u8],
4809) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
4810 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
4811 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
4812 Ok(probe) if MODULE_TO_SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
4813 ControlRequestBodyError::InvalidBody
4814 }
4815 Ok(_) => ControlRequestBodyError::UnknownOp,
4816 Err(_) => ControlRequestBodyError::InvalidBody,
4817 };
4818 (err, classification)
4819 })
4820}
4821
4822#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
4823enum ProviderRoleKind {
4824 ToolProvider,
4825 PipelineStage,
4826 ManagementSurface,
4827 InternalService,
4828}
4829
4830fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
4831 match role {
4832 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
4833 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
4834 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
4835 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
4836 }
4837}
4838
4839fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
4840 roles.iter().map(provider_role_kind).collect()
4841}
4842
4843fn capability_census_trigger(
4847 old: Option<&CapabilityDeclarations>,
4848 new: Option<&CapabilityDeclarations>,
4849) -> bool {
4850 let old_provides = old
4851 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
4852 .unwrap_or_default();
4853 let old_denies = old
4854 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
4855 .unwrap_or_default();
4856 let new = new.cloned().unwrap_or(CapabilityDeclarations {
4857 provides: Vec::new(),
4858 requires: Vec::new(),
4859 must_never_reach: Vec::new(),
4860 });
4861
4862 new.provides
4863 .iter()
4864 .any(|capability| !old_provides.contains(capability))
4865 || new
4866 .must_never_reach
4867 .iter()
4868 .any(|capability| !old_denies.contains(capability))
4869}
4870
4871fn denied_capability<'a>(
4874 opening_manifest: &'a ModuleManifest,
4875 target_manifest: &ModuleManifest,
4876) -> Option<&'a str> {
4877 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
4878 let target_capabilities = target_manifest.capabilities.as_ref()?;
4879 opening_capabilities
4880 .must_never_reach
4881 .iter()
4882 .find(|denied| {
4883 target_capabilities
4884 .provides
4885 .iter()
4886 .any(|provided| provided == *denied)
4887 })
4888 .map(String::as_str)
4889}
4890
4891fn catalog_update_frozen_field_message(
4892 registered: &ModuleManifest,
4893 provides: &[ProviderRole],
4894) -> Option<String> {
4895 let old_has_provides = !registered.provides.is_empty();
4896 let new_has_provides = !provides.is_empty();
4897 if old_has_provides != new_has_provides {
4898 return Some(format!(
4899 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
4900 registered.module_id
4901 ));
4902 }
4903
4904 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
4905 return Some(format!(
4906 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
4907 registered.module_id
4908 ));
4909 }
4910
4911 let registered_concurrency = manifest_concurrency(registered);
4912 let mut candidate = registered.clone();
4913 candidate.provides = provides.to_vec();
4914 let candidate_concurrency = manifest_concurrency(&candidate);
4915 if candidate_concurrency != registered_concurrency {
4916 return Some(format!(
4917 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
4918 registered.module_id, registered_concurrency, candidate_concurrency
4919 ));
4920 }
4921
4922 None
4925}
4926
4927fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
4928 manifest.provides.iter().any(is_routable_role)
4929}
4930
4931fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
4937 manifest
4938 .provides
4939 .iter()
4940 .find_map(|provider| match provider {
4941 ProviderRole::ToolProvider { concurrency, .. }
4942 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
4943 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
4944 })
4945 .unwrap_or(Concurrency::ModuleManaged)
4946}
4947
4948fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
4955 let has_management_surface = manifest
4956 .provides
4957 .iter()
4958 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
4959 if !has_management_surface {
4960 return false;
4961 }
4962 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
4963 return false;
4964 };
4965 let Some(provides) = raw
4966 .get("manifest")
4967 .and_then(|manifest| manifest.get("provides"))
4968 .and_then(serde_json::Value::as_array)
4969 else {
4970 return false;
4971 };
4972 provides.iter().any(|role| {
4977 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
4978 && role.get("concurrency").is_none()
4979 })
4980}
4981
4982fn negotiate_version(peer_version: u8) -> Result<u8, String> {
4983 if peer_version != PROTOCOL_VERSION {
4984 return Err(format!(
4985 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
4986 ));
4987 }
4988 Ok(PROTOCOL_VERSION)
4989}
4990
4991fn pong(frame: &Frame) -> Result<Frame, RouterError> {
4992 Frame::build_with_version(
4993 response_version(frame),
4994 FrameType::Pong,
4995 frame.header.flags,
4996 0,
4997 0,
4998 frame.header.corr,
4999 Vec::new(),
5000 )
5001 .map_err(RouterError::FrameBuild)
5002}
5003
5004fn control_error_frame(
5005 frame: &Frame,
5006 code: &'static str,
5007 message: impl Into<String>,
5008) -> Result<Frame, RouterError> {
5009 control_error_body_frame(
5010 frame,
5011 ErrorBody {
5012 code: code.to_string(),
5013 message: message.into(),
5014 detail: None,
5015 },
5016 )
5017}
5018
5019fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
5020 let body = serde_json::to_vec(&error).map_err(|err| {
5021 RouterError::backend(
5022 0,
5023 frame.header.corr,
5024 format!("failed to encode control ERROR: {err}"),
5025 )
5026 })?;
5027
5028 Frame::build_with_version(
5029 response_version(frame),
5030 FrameType::Error,
5031 control_flags(),
5032 0,
5033 0,
5034 frame.header.corr,
5035 body,
5036 )
5037 .map_err(RouterError::FrameBuild)
5038}
5039
5040fn control_response_body_frame<T: Serialize>(
5041 frame: &Frame,
5042 reply: &T,
5043 label: &'static str,
5044) -> Result<Frame, RouterError> {
5045 let body = serde_json::to_vec(reply).map_err(|err| {
5046 RouterError::backend(
5047 0,
5048 frame.header.corr,
5049 format!("failed to encode {label}: {err}"),
5050 )
5051 })?;
5052
5053 Frame::build_with_version(
5054 response_version(frame),
5055 FrameType::Response,
5056 control_flags(),
5057 0,
5058 0,
5059 frame.header.corr,
5060 body,
5061 )
5062 .map_err(RouterError::FrameBuild)
5063}
5064
5065fn forwarding_error_code(err: &ForwardingError) -> &'static str {
5095 match err {
5096 ForwardingError::NoModuleConnection => "target_unavailable",
5097 ForwardingError::ModuleReloading { .. } => "module_reloading",
5098 ForwardingError::ClientRouteChannelExhausted { .. }
5099 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
5100 ForwardingError::StaleModuleEndpoint
5101 | ForwardingError::UnknownReservation { .. }
5102 | ForwardingError::ConnectionClosing { .. }
5103 | ForwardingError::ClientEgressClosed { .. } => "target_unavailable",
5104 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
5107 ForwardingError::RelayCorrelationExhausted
5108 | ForwardingError::RouteOpenBuild(_)
5109 | ForwardingError::Poisoned => "forwarding_error",
5110 }
5111}
5112
5113fn response_version(frame: &Frame) -> u8 {
5114 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
5115 frame.header.ver
5116 } else {
5117 PROTOCOL_VERSION
5118 }
5119}
5120
5121fn control_flags() -> Flags {
5122 Flags::new(false, Priority::Passive, false)
5123}
5124
5125fn send_goodbye_target_best_effort(target: &GoodbyeTarget, context: &str) {
5126 let Ok(frame) = Frame::build_with_version(
5127 target.negotiated_ver,
5128 FrameType::Goodbye,
5129 control_flags(),
5130 target.channel,
5131 target.epoch,
5132 0,
5133 Vec::new(),
5134 ) else {
5135 return;
5136 };
5137 if let Err(err) = target.sink.try_send(frame) {
5138 warn!(
5139 route_channel = target.channel,
5140 route_epoch = target.epoch,
5141 error = %err,
5142 %context,
5143 "route GOODBYE dropped under backpressure"
5144 );
5145 }
5146}
5147
5148pub(crate) fn send_route_control_pushes(
5149 forwarding: &ForwardingTable,
5150 routes: Vec<EndpointRoute>,
5151 push: ClientControlPush,
5152) {
5153 let body = match serde_json::to_vec(&push) {
5154 Ok(body) => body,
5155 Err(err) => {
5156 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
5157 return;
5158 }
5159 };
5160 let mut targets = Vec::new();
5161 for route in routes {
5162 let target = route.goodbye_target;
5163 if let Some(existing) = targets
5164 .iter()
5165 .find(|existing: &&GoodbyeTarget| existing.connection_id == target.connection_id)
5166 {
5167 debug_assert_eq!(
5168 existing.negotiated_ver, target.negotiated_ver,
5169 "one connection cannot negotiate multiple frame versions"
5170 );
5171 continue;
5172 }
5173 targets.push(target);
5174 }
5175 for target in targets {
5176 let frame = match Frame::build_with_version(
5177 target.negotiated_ver,
5178 FrameType::Push,
5179 control_flags(),
5180 0,
5181 0,
5182 0,
5183 body.clone(),
5184 ) {
5185 Ok(frame) => frame,
5186 Err(err) => {
5187 warn!(
5188 route_channel = target.channel,
5189 error = %err,
5190 "failed to build route lifecycle control PUSH frame"
5191 );
5192 continue;
5193 }
5194 };
5195 if let Err(err) = target.sink.try_send(frame) {
5196 if target.close_on_delivery_failure() {
5197 warn!(
5198 target_connection_id = target.connection_id.get(),
5199 route_channel = target.channel,
5200 error = %err,
5201 "route lifecycle control PUSH was not delivered to client; closing target connection"
5202 );
5203 let _ = forwarding.escalate_client_delivery_failure(
5204 target.connection_id,
5205 target.channel,
5206 target.epoch,
5207 CloseReason::new(
5208 "route_lifecycle_push_delivery_failed",
5209 format!(
5210 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
5211 target.channel
5212 ),
5213 ),
5214 );
5215 }
5216 }
5217 }
5218}
5219
5220#[cfg(test)]
5221mod tests {
5222 use std::{
5223 collections::BTreeMap,
5224 fmt,
5225 path::PathBuf,
5226 sync::{Arc, Mutex},
5227 time::Duration,
5228 };
5229
5230 use serde_json::{json, Value};
5231 use subc_protocol::{
5232 manifest::{
5233 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
5234 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
5235 },
5236 session::HealthStatus,
5237 FrameType,
5238 };
5239
5240 use super::*;
5241 use crate::{
5242 forwarding::{DataRoute, DataRouteState},
5243 registry::ChannelState,
5244 router::FrameSink,
5245 stderr_tail::DEFAULT_MAX_LINE_BYTES,
5246 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
5247 test_support::TestTempDir,
5248 RouteCtx, Router,
5249 };
5250 use tokio::{
5251 sync::mpsc,
5252 time::{sleep, Instant},
5253 };
5254 use tracing::{
5255 field::{Field, Visit},
5256 Event, Subscriber,
5257 };
5258 use tracing_subscriber::{layer::Context, prelude::*, Layer};
5259
5260 fn fake_aft_stub_path() -> PathBuf {
5279 let mut path = std::env::current_exe().expect("current_exe available in tests");
5280 path.pop(); path.pop(); path.push(if cfg!(windows) {
5283 "fake-aft-stub.exe"
5284 } else {
5285 "fake-aft-stub"
5286 });
5287 assert!(
5288 path.exists(),
5289 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
5290 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
5291 path.display()
5292 );
5293 path
5294 }
5295
5296 fn client_retries(code: &str) -> bool {
5301 subc_protocol::error_codes::is_retryable_route_open(code)
5302 }
5303
5304 #[test]
5315 fn retryability_of_forwarding_codes_matches_the_failure() {
5316 let transient = [
5319 ForwardingError::NoModuleConnection,
5320 ForwardingError::ModuleReloading {
5321 module_id: "m".into(),
5322 },
5323 ForwardingError::StaleModuleEndpoint,
5324 ForwardingError::UnknownReservation {
5325 client_channel: 1,
5326 module_channel: 1,
5327 },
5328 ForwardingError::ConnectionClosing {
5329 connection_id: ConnectionId::new(1),
5330 },
5331 ForwardingError::ClientEgressClosed {
5332 connection_id: ConnectionId::new(1),
5333 },
5334 ];
5335 for err in transient {
5336 let code = forwarding_error_code(&err);
5337 assert!(
5338 client_retries(code),
5339 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
5340 );
5341 }
5342
5343 let permanent = [
5348 ForwardingError::ClientRouteChannelExhausted {
5349 connection_id: ConnectionId::new(1),
5350 },
5351 ForwardingError::ModuleRouteChannelExhausted {
5352 endpoint: ModuleEndpointId {
5353 connection_id: ConnectionId::new(1),
5354 generation: 1,
5355 },
5356 },
5357 ForwardingError::RelayCorrelationExhausted,
5358 ForwardingError::RouteOpenBuild("x".into()),
5359 ForwardingError::Poisoned,
5360 ];
5361 for err in permanent {
5362 let code = forwarding_error_code(&err);
5363 assert!(
5364 !client_retries(code),
5365 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
5366 );
5367 }
5368 }
5369
5370 #[tokio::test]
5398 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
5399 let handler = ControlHandler::default();
5400 let frame =
5401 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
5402
5403 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
5407 assert_eq!(
5408 stamped,
5409 Principal::Direct,
5410 "a caller that proved nothing must not be stamped as a supervised module"
5411 );
5412
5413 let forged = handler
5418 .route_open_principal(
5419 &frame,
5420 Some(ConsumerIdentity {
5421 module_id: "aft".to_string(),
5422 launch_nonce: "not-a-real-nonce".to_string(),
5423 }),
5424 )
5425 .unwrap();
5426 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
5427 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
5428 }
5429
5430 #[test]
5447 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
5448 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
5449 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
5450 let ClientControlRequest::RouteOpen {
5451 consumer_identity, ..
5452 } = parsed
5453 else {
5454 panic!("route.open body must parse as RouteOpen");
5455 };
5456 assert_eq!(
5457 consumer_identity,
5458 Some(ConsumerIdentity {
5459 module_id: "aft".to_string(),
5460 launch_nonce: "n".to_string(),
5461 }),
5462 "the wire field name must reach the value route_open_principal reads"
5463 );
5464 }
5465
5466 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
5467 ModuleManifest::builder(module_id, "0.1.0")
5468 .protocol_ver(protocol_ver)
5469 .provides(vec![ProviderRole::ToolProvider {
5470 tools: vec![Tool {
5471 name: "read".to_string(),
5472 description: None,
5473 execution_mode: ExecutionMode::Pure,
5474 schema: json!({"type": "object"}),
5475 }],
5476 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
5477 concurrency: Concurrency::ModuleManaged,
5478 emits_push: true,
5479 sub_supervises: true,
5480 }])
5481 .build()
5482 }
5483
5484 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
5485 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
5486 }
5487
5488 fn hello_frame_with_control_ops(
5489 module_id: &str,
5490 protocol_ver: u8,
5491 corr: u64,
5492 control_ops: Option<Vec<String>>,
5493 ) -> Frame {
5494 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
5495 }
5496
5497 fn hello_frame_with_nonce(
5498 module_id: &str,
5499 protocol_ver: u8,
5500 corr: u64,
5501 launch_nonce: Option<&str>,
5502 ) -> Frame {
5503 hello_frame_full(
5504 module_id,
5505 protocol_ver,
5506 corr,
5507 None,
5508 launch_nonce.map(ToOwned::to_owned),
5509 )
5510 }
5511
5512 fn hello_frame_full(
5513 module_id: &str,
5514 protocol_ver: u8,
5515 corr: u64,
5516 control_ops: Option<Vec<String>>,
5517 launch_nonce: Option<String>,
5518 ) -> Frame {
5519 let body = serde_json::to_vec(&ModuleHelloBody {
5520 manifest: manifest(module_id, protocol_ver),
5521 protocol_ver,
5522 control_ops,
5523 launch_nonce,
5524 })
5525 .unwrap();
5526 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
5527 }
5528
5529 fn non_routable_hello_frame_with_control_ops(
5530 module_id: &str,
5531 corr: u64,
5532 control_ops: Option<Vec<String>>,
5533 ) -> Frame {
5534 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
5535 manifest.provides.clear();
5536 let body = serde_json::to_vec(&ModuleHelloBody {
5537 manifest,
5538 protocol_ver: PROTOCOL_VERSION,
5539 control_ops,
5540 launch_nonce: None,
5541 })
5542 .unwrap();
5543 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
5544 }
5545
5546 fn capability_grammar_hello_frame(
5547 capabilities: Value,
5548 runtime_computed: Option<Value>,
5549 corr: u64,
5550 ) -> Frame {
5551 let mut body = serde_json::to_value(ModuleHelloBody {
5552 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
5553 protocol_ver: PROTOCOL_VERSION,
5554 control_ops: None,
5555 launch_nonce: None,
5556 })
5557 .expect("HELLO body serializes");
5558 body["manifest"]["capabilities"] = capabilities;
5559 if let Some(runtime_computed) = runtime_computed {
5560 body["runtime_computed"] = runtime_computed;
5561 }
5562 Frame::build(
5563 FrameType::Hello,
5564 control_flags(),
5565 0,
5566 0,
5567 corr,
5568 serde_json::to_vec(&body).expect("HELLO body reserializes"),
5569 )
5570 .expect("HELLO frame builds")
5571 }
5572
5573 fn channel_request(channel: u16, corr: u64) -> Frame {
5574 Frame::build(
5575 FrameType::Request,
5576 Flags::new(true, Priority::Interactive, false),
5577 channel,
5578 0,
5579 corr,
5580 b"opaque".to_vec(),
5581 )
5582 .unwrap()
5583 }
5584
5585 fn route_ctx(
5586 connection_id: ConnectionId,
5587 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
5588 let (tx, rx) = mpsc::channel(8);
5589 (
5590 RouteCtx {
5591 connection_id,
5592 egress: FrameSink::new(tx),
5593 },
5594 rx,
5595 )
5596 }
5597
5598 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
5599 serde_json::from_slice(&frame.body).unwrap()
5600 }
5601
5602 fn parse_error(frame: &Frame) -> Value {
5603 serde_json::from_slice(&frame.body).unwrap()
5604 }
5605
5606 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
5607 serde_json::from_slice(&frame.body).unwrap()
5608 }
5609
5610 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
5611 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
5612 route_channel,
5613 route_epoch: 0,
5614 kind,
5615 })
5616 .unwrap();
5617 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
5618 }
5619
5620 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
5621 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
5622 module_id: module_id.to_string(),
5623 })
5624 .unwrap();
5625 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
5626 }
5627
5628 fn route_open_frame(corr: u64, module_id: &str, project_root: TestTempDir) -> Frame {
5629 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
5630 }
5631
5632 fn route_open_frame_with_consumer_capabilities(
5633 corr: u64,
5634 module_id: &str,
5635 project_root: TestTempDir,
5636 consumer_capabilities: Option<Vec<String>>,
5637 ) -> Frame {
5638 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
5639 target: RouteTarget::ToolProvider {
5640 module_id: module_id.to_string(),
5641 },
5642 identity: BindIdentity::new(
5643 project_root.path().to_path_buf(),
5644 "unit".to_string(),
5645 "session".to_string(),
5646 ),
5647 consumer_identity: None,
5648 consumer_capabilities,
5649 admission_facts: None,
5650 })
5651 .unwrap();
5652 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
5653 }
5654
5655 fn route_open_frame_with_admission_facts(
5656 corr: u64,
5657 module_id: &str,
5658 project_root: TestTempDir,
5659 consumer_identity: Option<subc_control::ConsumerIdentity>,
5660 facts: Option<Value>,
5661 ) -> Frame {
5662 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
5663 target: RouteTarget::ToolProvider {
5664 module_id: module_id.to_string(),
5665 },
5666 identity: BindIdentity::new(
5667 project_root.path().to_path_buf(),
5668 "unit".to_string(),
5669 format!("session-{corr}"),
5670 ),
5671 consumer_identity,
5672 consumer_capabilities: None,
5673 admission_facts: facts,
5674 })
5675 .unwrap();
5676 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
5677 }
5678
5679 #[derive(Clone, Default)]
5680 struct EventCapture {
5681 events: Arc<Mutex<Vec<CapturedEvent>>>,
5682 }
5683
5684 #[derive(Clone, Debug)]
5685 struct CapturedEvent {
5686 target: String,
5687 fields: BTreeMap<String, String>,
5688 }
5689
5690 impl EventCapture {
5691 fn events(&self) -> Vec<CapturedEvent> {
5692 self.events.lock().unwrap().clone()
5693 }
5694 }
5695
5696 impl<S> Layer<S> for EventCapture
5697 where
5698 S: Subscriber,
5699 {
5700 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
5701 let mut visitor = EventFieldVisitor::default();
5702 event.record(&mut visitor);
5703 self.events.lock().unwrap().push(CapturedEvent {
5704 target: event.metadata().target().to_string(),
5705 fields: visitor.fields,
5706 });
5707 }
5708 }
5709
5710 #[derive(Default)]
5711 struct EventFieldVisitor {
5712 fields: BTreeMap<String, String>,
5713 }
5714
5715 impl Visit for EventFieldVisitor {
5716 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
5717 self.fields
5718 .insert(field.name().to_string(), format!("{value:?}"));
5719 }
5720 }
5721
5722 fn health_response(corr: u64, status: HealthStatus) -> Frame {
5723 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
5724 status,
5725 detail: Some("warming".to_string()),
5726 metrics: Some(json!({"queue_depth": 3})),
5727 })
5728 .unwrap();
5729 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
5730 }
5731
5732 fn route_bind_ack(corr: u64) -> Frame {
5733 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
5734 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
5735 }
5736
5737 fn unique_project_root(label: &str) -> TestTempDir {
5738 TestTempDir::new(label)
5739 }
5740
5741 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
5742 match parse_route_poll(frame) {
5743 ClientControlResponse::RoutePoll {
5744 status: None,
5745 live: Some(live),
5746 ..
5747 } => assert_eq!(live, expected_live),
5748 other => panic!("unexpected route.poll response: {other:?}"),
5749 }
5750 }
5751
5752 fn bind_liveness_route(
5753 registry: &Registry,
5754 forwarding: &ForwardingTable,
5755 module_id: &str,
5756 ) -> (RouteCtx, u16, u32) {
5757 let module_connection = ConnectionId::new(101);
5758 let client_connection = ConnectionId::new(202);
5759 let registration = registry
5760 .register_with_control_ops(
5761 manifest(module_id, PROTOCOL_VERSION),
5762 PROTOCOL_VERSION,
5763 module_connection,
5764 module_baseline_control_ops(),
5765 )
5766 .unwrap();
5767 let (module_tx, _module_rx) = mpsc::channel(8);
5768 let endpoint = forwarding
5769 .register_module_connection(
5770 module_connection,
5771 module_id.to_string(),
5772 PROTOCOL_VERSION,
5773 manifest_concurrency(®istration.manifest),
5774 FrameSink::new(module_tx),
5775 )
5776 .unwrap();
5777 let (client_ctx, _client_rx) = route_ctx(client_connection);
5778 let pending = forwarding
5779 .begin_route_bind_relay_for_test(
5780 client_connection,
5781 client_ctx.egress.clone(),
5782 1,
5783 module_id,
5784 )
5785 .unwrap();
5786 assert_eq!(pending.endpoint, endpoint);
5787 let route_channel = pending.client_channel;
5788 let route_epoch = pending.client_epoch;
5789 forwarding
5790 .complete_pending_relay(
5791 module_connection,
5792 pending.corr,
5793 RouteBindRelayOutcome::Accepted,
5794 )
5795 .unwrap();
5796 (client_ctx, route_channel, route_epoch)
5797 }
5798
5799 struct FakeProcessLiveness {
5800 live: Option<bool>,
5801 }
5802
5803 impl ModuleProcessLiveness for FakeProcessLiveness {
5804 fn process_live(&self, _module_id: &str) -> Option<bool> {
5805 self.live
5806 }
5807 }
5808
5809 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5810 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
5811 {
5812 let registry = Arc::new(Registry::default());
5813 let supervisor_handle = SupervisorHandle::new();
5814 let supervisor = Supervisor::new(
5815 Arc::clone(®istry),
5816 RestartPolicy::new(1, Duration::from_millis(10)),
5817 )
5818 .with_handle(supervisor_handle.clone());
5819 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
5820 let module = supervisor
5821 .spawn(ModuleSpec {
5822 module_id: "stderr-tail-wire".to_string(),
5823 program: fake_aft_stub_path(),
5824 args: Vec::new(),
5825 env: vec![
5826 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
5827 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
5828 ],
5829 reserved: false,
5830 reserved_prefixes: Vec::new(),
5831 protocol: ModuleProtocol::Subc,
5832 overlap: Default::default(),
5833 })
5834 .unwrap();
5835
5836 let deadline = Instant::now() + Duration::from_secs(5);
5837 loop {
5838 let tail = module.stderr_tail(None, None);
5839 if tail
5840 .entries
5841 .iter()
5842 .any(|entry| matches!(entry, TailEntry::ProcessStart))
5843 && tail.entries.iter().any(|entry| {
5844 matches!(
5845 entry,
5846 TailEntry::Line {
5847 truncated: true,
5848 ..
5849 }
5850 )
5851 })
5852 {
5853 break;
5854 }
5855 assert!(
5856 Instant::now() < deadline,
5857 "module did not produce a truncated line and restart boundary: {tail:?}"
5858 );
5859 sleep(Duration::from_millis(10)).await;
5860 }
5861
5862 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
5863 let request = ClientControlRequest::SupervisorStderrTail {
5864 module_id: "stderr-tail-wire".to_string(),
5865 max_lines: None,
5866 max_bytes: None,
5867 };
5868 let frame = Frame::build(
5869 FrameType::Request,
5870 control_flags(),
5871 0,
5872 0,
5873 1,
5874 serde_json::to_vec(&request).unwrap(),
5875 )
5876 .unwrap();
5877 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
5878 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
5879 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
5880 serde_json::from_slice(&responses[0].body).unwrap()
5881 else {
5882 panic!("expected supervisor.stderr_tail response");
5883 };
5884
5885 assert!(
5886 tail.entries
5887 .iter()
5888 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
5889 "the control response lost the restart boundary"
5890 );
5891 let Some(StderrTailEntry::Line { text, truncated }) = tail.entries.iter().find(|entry| {
5892 matches!(
5893 entry,
5894 StderrTailEntry::Line {
5895 truncated: true,
5896 ..
5897 }
5898 )
5899 }) else {
5900 panic!("the control response lost the truncated line");
5901 };
5902 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
5903 assert!(*truncated);
5904 }
5905
5906 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5907 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
5908 let registry = Arc::new(Registry::default());
5909 let supervisor_handle = SupervisorHandle::new();
5910 let supervisor =
5911 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
5912 .with_handle(supervisor_handle.clone());
5913 let module = supervisor
5914 .spawn(ModuleSpec {
5915 module_id: "terminal-golden".to_string(),
5916 program: fake_aft_stub_path(),
5917 args: Vec::new(),
5918 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
5919 reserved: false,
5920 reserved_prefixes: Vec::new(),
5921 protocol: ModuleProtocol::Subc,
5922 overlap: Default::default(),
5923 })
5924 .unwrap();
5925
5926 let deadline = Instant::now() + Duration::from_secs(5);
5927 while module.terminal_history().entries.len() != 2 {
5928 assert!(
5929 Instant::now() < deadline,
5930 "module did not retain two terminal exits: {:?}",
5931 module.terminal_history()
5932 );
5933 sleep(Duration::from_millis(10)).await;
5934 }
5935
5936 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
5937 let request = ClientControlRequest::SupervisorTerminals {
5938 module_id: "terminal-golden".to_string(),
5939 };
5940 let frame = Frame::build(
5941 FrameType::Request,
5942 control_flags(),
5943 0,
5944 0,
5945 1,
5946 serde_json::to_vec(&request).unwrap(),
5947 )
5948 .unwrap();
5949 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
5950 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
5951 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
5952 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
5953 panic!("expected supervisor.terminals response");
5954 };
5955 assert_eq!(terminals.entries.len(), 2);
5956 assert_eq!(terminals.dropped, 0);
5957
5958 let mut rendered = serde_json::to_value(response).unwrap();
5959 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
5962 for (index, entry) in rendered["entries"]
5963 .as_array_mut()
5964 .expect("terminal response entries array")
5965 .iter_mut()
5966 .enumerate()
5967 {
5968 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
5969 }
5970
5971 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
5972 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
5973 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
5974 if std::env::var_os("UPDATE_GOLDEN").is_some() {
5975 std::fs::write(&golden_path, &serialized).unwrap();
5976 }
5977 let expected: Value =
5978 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
5979 assert_eq!(rendered, expected);
5980 }
5981
5982 #[test]
5983 fn hello_registers_manifest_and_returns_ack() {
5984 let registry = Arc::new(Registry::default());
5985 let handler = ControlHandler::new(Arc::clone(®istry));
5986 let conn = ConnectionId::new(1);
5987
5988 let responses = handler
5989 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
5990 .unwrap();
5991
5992 assert_eq!(responses.len(), 1);
5993 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
5994 assert_eq!(responses[0].header.channel, 0);
5995 assert_eq!(responses[0].header.corr, 7);
5996 let ack = parse_ack(&responses[0]);
5997 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
5998 assert!(ack
5999 .subc_capabilities
6000 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
6001 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
6002 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
6003 assert!(ack
6004 .subc_ops
6005 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
6006 assert!(ack
6007 .subc_ops
6008 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
6009
6010 let registration = registry.get_module("aft").unwrap().unwrap();
6011 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
6012 assert_eq!(registration.state, ChannelState::Active);
6013 assert_eq!(registration.connection_id, conn);
6014 assert_eq!(registration.control_ops, module_baseline_control_ops());
6015 }
6016
6017 #[test]
6018 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
6019 let invalid_identifiers = [
6020 ("case_change", "credentials-Provider/v1"),
6021 ("leading_zero", "credentials-provider/v01"),
6022 ("trailing_hyphen", "credentials-provider-/v1"),
6023 ("consecutive_hyphens", "credentials--provider/v1"),
6024 ("uppercase", "Credentials-provider/v1"),
6025 ("missing_v", "credentials-provider/1"),
6026 ("whitespace", "credentials provider/v1"),
6027 ("zero_version", "credentials-provider/v0"),
6028 ("out_of_range_version", "credentials-provider/v4294967296"),
6029 (
6030 "overlength_name",
6031 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
6032 ),
6033 ];
6034 let mut cases = invalid_identifiers
6035 .into_iter()
6036 .map(|(name, identifier)| {
6037 (
6038 format!("identifier_{name}"),
6039 "capabilities.provides[0]".to_string(),
6040 identifier.to_string(),
6041 json!({ "provides": [identifier] }),
6042 None,
6043 )
6044 })
6045 .collect::<Vec<_>>();
6046 cases.extend([
6047 (
6048 "unknown_need".to_string(),
6049 "capabilities.requires[0].need".to_string(),
6050 "deferred".to_string(),
6051 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
6052 None,
6053 ),
6054 (
6055 "duplicate_provides".to_string(),
6056 "capabilities.provides[1]".to_string(),
6057 "credentials-provider/v1".to_string(),
6058 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
6059 None,
6060 ),
6061 (
6062 "duplicate_must_never_reach".to_string(),
6063 "capabilities.must_never_reach[1]".to_string(),
6064 "credentials-provider/v1".to_string(),
6065 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
6066 None,
6067 ),
6068 (
6069 "duplicate_requires_same_need".to_string(),
6070 "capabilities.requires[1]".to_string(),
6071 "credentials-provider/v1".to_string(),
6072 json!({ "requires": [
6073 { "capability": "credentials-provider/v1", "need": "required" },
6074 { "capability": "credentials-provider/v1", "need": "required" }
6075 ] }),
6076 None,
6077 ),
6078 (
6079 "duplicate_requires_conflicting_need".to_string(),
6080 "capabilities.requires[1]".to_string(),
6081 "credentials-provider/v1".to_string(),
6082 json!({ "requires": [
6083 { "capability": "credentials-provider/v1", "need": "required" },
6084 { "capability": "credentials-provider/v1", "need": "optional" }
6085 ] }),
6086 None,
6087 ),
6088 (
6089 "capabilities_root_pointer".to_string(),
6090 "runtime_computed[0]".to_string(),
6091 "/capabilities".to_string(),
6092 json!({}),
6093 Some(json!(["/capabilities"])),
6094 ),
6095 (
6096 "capabilities_descendant_pointer".to_string(),
6097 "runtime_computed[0]".to_string(),
6098 "/capabilities/provides".to_string(),
6099 json!({}),
6100 Some(json!(["/capabilities/provides"])),
6101 ),
6102 (
6103 "malformed_pointer_without_leading_slash".to_string(),
6104 "runtime_computed[0]".to_string(),
6105 "capabilities".to_string(),
6106 json!({}),
6107 Some(json!(["capabilities"])),
6108 ),
6109 (
6110 "malformed_pointer_escape".to_string(),
6111 "runtime_computed[0]".to_string(),
6112 "/roles/~2/tools".to_string(),
6113 json!({}),
6114 Some(json!(["/roles/~2/tools"])),
6115 ),
6116 (
6117 "unknown_capabilities_field".to_string(),
6118 "capabilities.future".to_string(),
6119 "<array>".to_string(),
6120 json!({ "future": [] }),
6121 None,
6122 ),
6123 ]);
6124
6125 for (index, (name, field, value, capabilities, runtime_computed)) in
6126 cases.into_iter().enumerate()
6127 {
6128 let registry = Arc::new(Registry::default());
6129 let handler = ControlHandler::new(Arc::clone(®istry));
6130 let response = handler
6131 .handle_control(
6132 ConnectionId::new((index + 1) as u64),
6133 capability_grammar_hello_frame(
6134 capabilities,
6135 runtime_computed,
6136 index as u64 + 1,
6137 ),
6138 )
6139 .expect("invalid HELLO returns a refusal");
6140
6141 assert_eq!(response.len(), 1, "{name} must emit one refusal");
6142 let error = parse_error(&response[0]);
6143 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
6144 let message = error["message"]
6145 .as_str()
6146 .expect("error message is a string");
6147 assert!(
6148 message.contains(&field),
6149 "{name}: field missing from {message}"
6150 );
6151 assert!(
6152 message.contains(&value),
6153 "{name}: value missing from {message}"
6154 );
6155 assert_eq!(
6156 registry
6157 .active_registration_count()
6158 .expect("registry reads"),
6159 0,
6160 "{name}: refused HELLO must not create a catalog entry"
6161 );
6162 }
6163 }
6164
6165 #[test]
6166 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
6167 let registry = Arc::new(Registry::default());
6168 let handler = ControlHandler::new(Arc::clone(®istry));
6169 let capabilities = json!({
6170 "provides": ["credentials-provider/v1"],
6171 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
6172 "must_never_reach": ["federation-transport/v1"]
6173 });
6174 let response = handler
6175 .handle_control(
6176 ConnectionId::new(99),
6177 capability_grammar_hello_frame(
6178 capabilities.clone(),
6179 Some(json!(["/roles/0/tools"])),
6180 99,
6181 ),
6182 )
6183 .expect("valid HELLO registers");
6184 assert_eq!(response[0].header.ty, FrameType::HelloAck);
6185
6186 let request = Frame::build(
6187 FrameType::Request,
6188 control_flags(),
6189 0,
6190 0,
6191 100,
6192 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
6193 .expect("catalog request serializes"),
6194 )
6195 .expect("catalog request frame builds");
6196 let response = handler
6197 .handle_catalog_list(request, None)
6198 .expect("catalog list succeeds");
6199 let ClientControlResponse::CatalogList { modules, .. } =
6200 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
6201 else {
6202 panic!("catalog request must return catalog.list");
6203 };
6204 assert_eq!(modules.len(), 1);
6205 assert_eq!(
6206 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
6207 capabilities
6208 );
6209 }
6210
6211 #[test]
6212 fn catalog_list_mirrors_management_operation_description() {
6213 let registry = Arc::new(Registry::default());
6214 let handler = ControlHandler::new(Arc::clone(®istry));
6215 let description = "List managed records and return their identifiers and metadata.";
6216 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
6217 manifest.provides = vec![ProviderRole::ManagementSurface {
6218 operations: vec![ManagementOperation {
6219 name: "records.list".to_string(),
6220 kind: ManagementOperationKind::Query,
6221 description: Some(description.to_string()),
6222 }],
6223 config_schema: json!({"type": "object"}),
6224 observability: vec![ObservabilitySurface {
6225 name: "records.stats".to_string(),
6226 kind: ObservabilityKind::Snapshot,
6227 }],
6228 identity_scope: vec![IdentityScope::Project],
6229 concurrency: Concurrency::ModuleManaged,
6230 }];
6231 registry
6232 .register_with_control_ops(
6233 manifest,
6234 PROTOCOL_VERSION,
6235 ConnectionId::new(99),
6236 Vec::new(),
6237 )
6238 .expect("described management manifest registers");
6239
6240 let request = Frame::build(
6241 FrameType::Request,
6242 control_flags(),
6243 0,
6244 0,
6245 100,
6246 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
6247 .expect("catalog request serializes"),
6248 )
6249 .expect("catalog request frame builds");
6250 let response = handler
6251 .handle_catalog_list(request, None)
6252 .expect("catalog list succeeds");
6253 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
6254 assert_eq!(
6255 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
6256 "catalog.list must preserve the declared operation description verbatim"
6257 );
6258 }
6259
6260 #[test]
6261 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
6262 let registry = Arc::new(Registry::default());
6263 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
6264 [("vault".to_string(), true), ("squatter".to_string(), true)],
6265 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
6266 );
6267 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
6268 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
6269 provides: vec!["credentials-provider/v1".to_string()],
6270 requires: Vec::new(),
6271 must_never_reach: Vec::new(),
6272 });
6273 let frame = Frame::build(
6274 FrameType::Hello,
6275 control_flags(),
6276 0,
6277 0,
6278 77,
6279 serde_json::to_vec(&ModuleHelloBody {
6280 manifest: squatter,
6281 protocol_ver: PROTOCOL_VERSION,
6282 control_ops: None,
6283 launch_nonce: None,
6284 })
6285 .expect("HELLO serializes"),
6286 )
6287 .expect("HELLO frame builds");
6288 let response = handler
6289 .handle_control(ConnectionId::new(77), frame)
6290 .expect("reserved claim receives a typed refusal");
6291 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
6292 assert_eq!(
6293 registry
6294 .active_registration_count()
6295 .expect("registry reads"),
6296 0,
6297 "a reserved capability refusal must not leave a catalog entry"
6298 );
6299 }
6300
6301 #[test]
6302 fn server_describe_surfaces_required_capability_verdict_fields() {
6303 let registry = Arc::new(Registry::default());
6304 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
6305 [
6306 ("consumer".to_string(), true),
6307 ("provider".to_string(), false),
6308 ],
6309 BTreeMap::new(),
6310 );
6311 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
6312 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
6313 provides: Vec::new(),
6314 requires: vec![subc_protocol::manifest::CapabilityRequirement {
6315 capability: "credentials-provider/v1".to_string(),
6316 need: subc_protocol::manifest::CapabilityNeed::Required,
6317 }],
6318 must_never_reach: Vec::new(),
6319 });
6320 let hello = Frame::build(
6321 FrameType::Hello,
6322 control_flags(),
6323 0,
6324 0,
6325 78,
6326 serde_json::to_vec(&ModuleHelloBody {
6327 manifest: consumer,
6328 protocol_ver: PROTOCOL_VERSION,
6329 control_ops: None,
6330 launch_nonce: None,
6331 })
6332 .expect("HELLO serializes"),
6333 )
6334 .expect("HELLO frame builds");
6335 handler
6336 .handle_control(ConnectionId::new(78), hello)
6337 .expect("consumer registers");
6338 let describe = Frame::build(
6339 FrameType::Request,
6340 control_flags(),
6341 0,
6342 0,
6343 79,
6344 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
6345 .expect("request serializes"),
6346 )
6347 .expect("describe frame builds");
6348 let response = handler
6349 .handle_server_describe(describe)
6350 .expect("server.describe succeeds");
6351 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
6352 let requirement = &rendered["capability_requirements"][0];
6353 assert_eq!(requirement["consumer"], "consumer");
6354 assert_eq!(requirement["verdict"], "never_provided");
6355 assert_eq!(requirement["episode_seq"], 1);
6356 assert_eq!(requirement["config_satisfiable"], false);
6357 assert_eq!(requirement["runtime_available"], false);
6358 assert!(requirement["detail"]
6359 .as_str()
6360 .expect("detail string")
6361 .contains("credentials-provider/v1"));
6362 }
6363
6364 #[test]
6365 fn catalog_list_omits_capabilities_for_legacy_manifest() {
6366 let registry = Arc::new(Registry::default());
6367 let handler = ControlHandler::new(Arc::clone(®istry));
6368 let hello = handler
6369 .handle_control(
6370 ConnectionId::new(101),
6371 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
6372 )
6373 .expect("legacy HELLO registers");
6374 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
6375
6376 let request = Frame::build(
6377 FrameType::Request,
6378 control_flags(),
6379 0,
6380 0,
6381 102,
6382 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
6383 .expect("catalog request serializes"),
6384 )
6385 .expect("catalog request frame builds");
6386 let response = handler
6387 .handle_catalog_list(request, None)
6388 .expect("catalog list succeeds");
6389 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
6390 assert!(
6391 body["modules"][0].get("capabilities").is_none(),
6392 "legacy manifest must retain an absent capabilities field on catalog.list"
6393 );
6394 }
6395
6396 #[test]
6397 fn hello_ack_omits_storage_when_no_storage_config() {
6398 let registry = Arc::new(Registry::default());
6399 let handler = ControlHandler::new(Arc::clone(®istry));
6400 let responses = handler
6401 .handle_control(
6402 ConnectionId::new(1),
6403 hello_frame("aft", PROTOCOL_VERSION, 7),
6404 )
6405 .unwrap();
6406 let ack = parse_ack(&responses[0]);
6407 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
6408 }
6409
6410 #[test]
6411 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
6412 let registry = Arc::new(Registry::default());
6415 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
6416 crate::daemon_config::StorageConfig::Sqlite {
6417 data_home: std::path::PathBuf::from("/data"),
6418 },
6419 ));
6420
6421 let responses = handler
6422 .handle_control(
6423 ConnectionId::new(1),
6424 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
6425 )
6426 .unwrap();
6427 let ack = parse_ack(&responses[0]);
6428 assert_eq!(
6429 ack.storage,
6430 Some(serde_json::json!({
6431 "module_id": "alfonso-routing",
6432 "storage_namespace": "default",
6433 "isolation": { "kind": "module" },
6434 "backend": {
6435 "backend": "sqlite",
6436 "path": "/data/cortexkit/alfonso-routing/store.db"
6437 }
6438 })),
6439 "the delivered descriptor is the module's own sqlite store path"
6440 );
6441 }
6442
6443 #[test]
6444 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
6445 let registry = Arc::new(Registry::default());
6446 let handler = ControlHandler::new(Arc::clone(®istry));
6447 let conn = ConnectionId::new(1);
6448 let responses = handler
6449 .handle_control(
6450 conn,
6451 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
6452 )
6453 .unwrap();
6454 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
6455 let registration = registry.get_module("aft").unwrap().unwrap();
6456 assert_eq!(registration.control_ops, module_baseline_control_ops());
6457
6458 let frame =
6459 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
6460 assert!(handler
6461 .guard_module_control_op(&frame, "aft", "route.bind")
6462 .unwrap()
6463 .is_none());
6464 let error = handler
6465 .guard_module_control_op(&frame, "aft", "test.synthetic")
6466 .unwrap()
6467 .expect("synthetic ungranted op should be rejected");
6468 assert_eq!(error.header.ty, FrameType::Error);
6469 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
6470 }
6471
6472 #[test]
6473 fn hello_control_ops_some_adds_optional_grants() {
6474 let registry = Arc::new(Registry::default());
6475 let handler = ControlHandler::new(Arc::clone(®istry));
6476 handler
6477 .handle_control(
6478 ConnectionId::new(1),
6479 hello_frame_with_control_ops(
6480 "aft",
6481 PROTOCOL_VERSION,
6482 7,
6483 Some(vec![
6484 "future.synthetic".to_string(),
6485 "route.bind".to_string(),
6486 ]),
6487 ),
6488 )
6489 .unwrap();
6490 let registration = registry.get_module("aft").unwrap().unwrap();
6491 assert_eq!(
6492 registration.control_ops,
6493 vec![
6494 "route.bind".to_string(),
6495 "route.status".to_string(),
6496 "future.synthetic".to_string(),
6497 ]
6498 );
6499 let frame =
6500 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
6501 assert!(handler
6502 .guard_module_control_op(&frame, "aft", "future.synthetic")
6503 .unwrap()
6504 .is_none());
6505 }
6506
6507 #[tokio::test]
6508 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
6509 let registry = Arc::new(Registry::default());
6510 let forwarding = Arc::new(ForwardingTable::default());
6511 let handler =
6512 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
6513 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
6514 let responses = handler
6515 .handle_control_frame(
6516 &module_ctx,
6517 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
6518 )
6519 .await
6520 .unwrap();
6521 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
6522
6523 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
6524 let responses = handler
6525 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
6526 .await
6527 .unwrap();
6528 assert_eq!(responses.len(), 1);
6529 assert_eq!(responses[0].header.ty, FrameType::Error);
6530 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
6531 assert!(module_rx.try_recv().is_err());
6532 }
6533
6534 #[tokio::test]
6535 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
6536 let registry = Arc::new(Registry::default());
6537 let forwarding = Arc::new(ForwardingTable::default());
6538 let handler =
6539 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
6540 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
6541 handler
6542 .handle_control_frame(
6543 &module_ctx,
6544 hello_frame_with_control_ops(
6545 "aft",
6546 PROTOCOL_VERSION,
6547 7,
6548 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
6549 ),
6550 )
6551 .await
6552 .unwrap();
6553
6554 let project_root = unique_project_root("demux");
6555 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
6556 let route_handler = handler.clone();
6557 let route_task = tokio::spawn(async move {
6558 route_handler
6559 .handle_control_frame(
6560 &route_client_ctx,
6561 route_open_frame(100, "aft", project_root),
6562 )
6563 .await
6564 .unwrap()
6565 });
6566 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
6567 .await
6568 .unwrap()
6569 .unwrap();
6570 assert!(matches!(
6571 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
6572 ModuleControlRequest::RouteBind { .. }
6573 ));
6574
6575 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
6576 let health_handler = handler.clone();
6577 let health_task = tokio::spawn(async move {
6578 health_handler
6579 .handle_control_frame(
6580 &health_client_ctx,
6581 supervisor_health_probe_frame(101, "aft"),
6582 )
6583 .await
6584 .unwrap()
6585 });
6586 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
6587 .await
6588 .unwrap()
6589 .unwrap();
6590 assert_eq!(
6591 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
6592 ModuleControlRequest::HealthCheck {}
6593 );
6594
6595 handler
6596 .handle_control_frame(
6597 &module_ctx,
6598 health_response(health_frame.header.corr, HealthStatus::Degraded),
6599 )
6600 .await
6601 .unwrap();
6602 let health_response = health_task.await.unwrap();
6603 assert_eq!(health_response.len(), 1);
6604 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
6605 ClientControlResponse::SupervisorHealthProbe {
6606 module_id,
6607 status,
6608 detail,
6609 metrics,
6610 } => {
6611 assert_eq!(module_id, "aft");
6612 assert_eq!(status, HealthStatus::Degraded);
6613 assert_eq!(detail.as_deref(), Some("warming"));
6614 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
6615 }
6616 other => panic!("unexpected health response: {other:?}"),
6617 }
6618
6619 handler
6620 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
6621 .await
6622 .unwrap();
6623 let route_response = route_task.await.unwrap();
6624 assert!(route_response.is_empty());
6625 let published = route_client_rx.recv().await.unwrap();
6626 assert!(matches!(
6627 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
6628 ClientControlResponse::RouteOpen { .. }
6629 ));
6630 }
6631
6632 async fn relay_route_open(
6637 handler: &ControlHandler,
6638 client_connection: ConnectionId,
6639 client_egress: &FrameSink,
6640 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
6641 corr: u64,
6642 module_id: &str,
6643 project_root_label: &str,
6644 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
6645 let ctx = RouteCtx {
6646 connection_id: client_connection,
6647 egress: client_egress.clone(),
6648 };
6649 let handler = handler.clone();
6650 let project_root = unique_project_root(project_root_label);
6651 let module_id = module_id.to_string();
6652 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
6653 let task = tokio::spawn(async move {
6654 let _guard = tracing::dispatcher::set_default(&dispatch);
6655 handler
6656 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
6657 .await
6658 .unwrap()
6659 });
6660 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
6661 .await
6662 .expect("module receives the relayed route.bind")
6663 .expect("module egress is open");
6664 (task, bind.frame)
6665 }
6666
6667 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
6668 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
6669 ModuleControlRequest::RouteBind {
6670 route_channel,
6671 epoch,
6672 ..
6673 } => (route_channel, epoch),
6674 other => panic!("expected a route.bind request, got {other:?}"),
6675 }
6676 }
6677
6678 fn published_route(frame: &Frame) -> (u16, u32) {
6679 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
6680 ClientControlResponse::RouteOpen {
6681 route_channel,
6682 route_epoch,
6683 } => (route_channel, route_epoch),
6684 other => panic!("expected a route.open response, got {other:?}"),
6685 }
6686 }
6687
6688 #[tokio::test]
6708 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
6709 let registry = Arc::new(Registry::default());
6710 let forwarding = Arc::new(ForwardingTable::default());
6711 let handler =
6712 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
6713
6714 let module_connection = ConnectionId::new(30);
6715 let (module_ctx, mut module_rx) = route_ctx(module_connection);
6716 handler
6717 .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
6718 .await
6719 .unwrap();
6720
6721 let dying_client = ConnectionId::new(31);
6722 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
6723
6724 let (first_task, first_bind) = relay_route_open(
6727 &handler,
6728 dying_client,
6729 &dying_ctx.egress,
6730 &mut module_rx,
6731 100,
6732 "aft",
6733 "closing-first",
6734 )
6735 .await;
6736 handler
6737 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
6738 .await
6739 .unwrap();
6740 assert!(first_task.await.unwrap().is_empty());
6741 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
6742
6743 let (second_task, second_bind) = relay_route_open(
6745 &handler,
6746 dying_client,
6747 &dying_ctx.egress,
6748 &mut module_rx,
6749 101,
6750 "aft",
6751 "closing-second",
6752 )
6753 .await;
6754 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
6755
6756 assert!(forwarding
6759 .escalate_client_delivery_failure(
6760 dying_client,
6761 first_channel,
6762 first_epoch,
6763 CloseReason::new(
6764 "module_to_client_delivery_failed",
6765 "client egress refused a module frame",
6766 ),
6767 )
6768 .unwrap());
6769 assert!(!dying_ctx.egress.is_closed());
6770
6771 let ack = handler
6773 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
6774 .await;
6775 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
6776 if module_loop_error.is_some() {
6777 handler.cleanup_connection(module_connection).unwrap();
6781 }
6782 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
6786 .await
6787 .ok()
6788 .flatten();
6789
6790 assert!(
6792 registry
6793 .get_module_by_connection(module_connection)
6794 .unwrap()
6795 .is_some(),
6796 "one client's closing connection ended the shared module connection: \
6797 {module_loop_error:?}"
6798 );
6799 let cotenant = ConnectionId::new(32);
6801 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
6802 let (cotenant_task, cotenant_bind) = relay_route_open(
6803 &handler,
6804 cotenant,
6805 &cotenant_ctx.egress,
6806 &mut module_rx,
6807 102,
6808 "aft",
6809 "closing-cotenant",
6810 )
6811 .await;
6812 handler
6813 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
6814 .await
6815 .unwrap();
6816 assert!(cotenant_task.await.unwrap().is_empty());
6817 let (cotenant_channel, cotenant_epoch) =
6818 published_route(&cotenant_rx.recv().await.unwrap());
6819 assert!(matches!(
6820 forwarding
6821 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
6822 .unwrap(),
6823 DataRoute::Client(DataRouteState::Bound(_))
6824 ));
6825
6826 let goodbye = post_ack_module_frame
6829 .expect("module receives a GOODBYE for the abandoned route channel");
6830 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
6831 assert_eq!(goodbye.header.channel, abandoned_channel);
6832 assert_eq!(goodbye.header.epoch, abandoned_epoch);
6833
6834 assert!(dying_rx.try_recv().is_err());
6838 let second_response = second_task.await.unwrap();
6839 assert_eq!(second_response.len(), 1);
6840 assert_eq!(
6841 parse_error(&second_response[0])["code"],
6842 "target_unavailable"
6843 );
6844 }
6845
6846 #[test]
6853 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
6854 let handler = ControlHandler::default();
6855 let module_connection = ConnectionId::new(30);
6856 let client_connection = ConnectionId::new(31);
6857
6858 handler
6859 .refuse_to_end_module_connection_for_a_client(
6860 module_connection,
6861 77,
6862 ForwardingError::ConnectionClosing {
6863 connection_id: client_connection,
6864 },
6865 )
6866 .expect("a closing client must never end the module connection");
6867
6868 assert!(matches!(
6869 handler.refuse_to_end_module_connection_for_a_client(
6870 module_connection,
6871 78,
6872 ForwardingError::ConnectionClosing {
6873 connection_id: module_connection,
6874 },
6875 ),
6876 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
6877 connection_id
6878 })) if connection_id == module_connection
6879 ));
6880 assert!(matches!(
6881 handler.refuse_to_end_module_connection_for_a_client(
6882 module_connection,
6883 79,
6884 ForwardingError::Poisoned,
6885 ),
6886 Err(RouterError::Forwarding(ForwardingError::Poisoned))
6887 ));
6888 assert!(matches!(
6889 handler.refuse_to_end_module_connection_for_a_client(
6890 module_connection,
6891 80,
6892 ForwardingError::StaleModuleEndpoint,
6893 ),
6894 Err(RouterError::Forwarding(
6895 ForwardingError::StaleModuleEndpoint
6896 ))
6897 ));
6898 }
6899
6900 #[tokio::test]
6913 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
6914 let registry = Arc::new(Registry::default());
6915 let forwarding = Arc::new(ForwardingTable::default());
6916 let supervisor = SupervisorHandle::new();
6917 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
6918 let handler =
6919 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
6920 .with_supervisor(supervisor);
6921
6922 let (target_ctx, _target_rx) = route_ctx(ConnectionId::new(90));
6923 handler
6924 .handle_control_frame(&target_ctx, hello_frame("target", PROTOCOL_VERSION, 1))
6925 .await
6926 .unwrap();
6927
6928 let wrong_nonce = handler
6932 .handle_control_frame(
6933 &route_ctx(ConnectionId::new(91)).0,
6934 route_open_frame_with_admission_facts(
6935 20,
6936 "target",
6937 unique_project_root("admission-facts"),
6938 Some(subc_control::ConsumerIdentity {
6939 module_id: "fed".to_string(),
6940 launch_nonce: "not-the-real-nonce".to_string(),
6941 }),
6942 None,
6943 ),
6944 )
6945 .await
6946 .unwrap();
6947 assert_eq!(
6948 parse_error(&wrong_nonce[0])["code"],
6949 "bad_consumer_identity",
6950 "a mismatched launch nonce must be refused, not stamped Reserved"
6951 );
6952
6953 let never_spawned = handler
6957 .handle_control_frame(
6958 &route_ctx(ConnectionId::new(92)).0,
6959 route_open_frame_with_admission_facts(
6960 21,
6961 "target",
6962 unique_project_root("admission-facts"),
6963 Some(subc_control::ConsumerIdentity {
6964 module_id: "never-spawned".to_string(),
6965 launch_nonce: "any-nonce".to_string(),
6966 }),
6967 None,
6968 ),
6969 )
6970 .await
6971 .unwrap();
6972 assert_eq!(
6973 parse_error(&never_spawned[0])["code"],
6974 "bad_consumer_identity",
6975 "an unspawned module_id must be refused rather than accepted for lack of a record"
6976 );
6977 }
6978
6979 #[tokio::test]
7002 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
7003 let registry = Arc::new(Registry::default());
7004 let forwarding = Arc::new(ForwardingTable::default());
7005 let supervisor = SupervisorHandle::new();
7006 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7007 let handler =
7008 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
7009 .with_supervisor(supervisor);
7010
7011 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
7012 handler
7013 .handle_control_frame(&target_ctx, hello_frame("target", PROTOCOL_VERSION, 1))
7014 .await
7015 .unwrap();
7016
7017 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
7018 let route_handler = handler.clone();
7019 let route_task = tokio::spawn(async move {
7020 route_handler
7021 .handle_control_frame(
7022 &client_ctx,
7023 route_open_frame_with_admission_facts(
7024 30,
7025 "target",
7026 unique_project_root("admission-facts"),
7027 Some(subc_control::ConsumerIdentity {
7028 module_id: "fed".to_string(),
7029 launch_nonce: "fed-nonce".to_string(),
7030 }),
7031 None,
7032 ),
7033 )
7034 .await
7035 .unwrap()
7036 });
7037
7038 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7045 .await
7046 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
7047 .expect("module control channel closed before route.bind");
7048 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
7049 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
7050 panic!("expected route.bind")
7051 };
7052 assert_eq!(
7053 principal,
7054 Some(Principal::Reserved {
7055 module_id: "fed".to_string()
7056 }),
7057 "a correctly attested consumer must be stamped Reserved for its own id"
7058 );
7059
7060 handler
7061 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
7062 .await
7063 .unwrap();
7064 assert!(route_task.await.unwrap().is_empty());
7065 assert!(
7066 matches!(
7067 serde_json::from_slice::<ClientControlResponse>(
7068 &client_rx.recv().await.unwrap().body
7069 )
7070 .unwrap(),
7071 ClientControlResponse::RouteOpen { .. }
7072 ),
7073 "the route must actually open, not merely avoid an error"
7074 );
7075 }
7076
7077 #[tokio::test(start_paused = true)]
7078 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
7079 let registry = Arc::new(Registry::default());
7080 let forwarding = Arc::new(ForwardingTable::default());
7081 let supervisor = SupervisorHandle::new();
7082 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7083 let handler =
7084 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
7085 .with_supervisor(supervisor);
7086
7087 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
7088 handler
7089 .handle_control_frame(&target_ctx, hello_frame("target", PROTOCOL_VERSION, 1))
7090 .await
7091 .unwrap();
7092
7093 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
7094 let direct_handler = handler.clone();
7095 let direct_open = tokio::spawn(async move {
7096 direct_handler
7097 .handle_control_frame(
7098 &direct_ctx,
7099 route_open_frame(2, "target", unique_project_root("route-census-direct")),
7100 )
7101 .await
7102 .unwrap()
7103 });
7104 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7105 .await
7106 .expect("no direct route.bind within 5s")
7107 .expect("target control channel closed before direct route.bind");
7108 handler
7109 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
7110 .await
7111 .unwrap();
7112 assert!(direct_open.await.unwrap().is_empty());
7113 let _ = direct_rx.recv().await.unwrap();
7114
7115 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
7116 let reserved_handler = handler.clone();
7117 let reserved_open = tokio::spawn(async move {
7118 reserved_handler
7119 .handle_control_frame(
7120 &reserved_ctx,
7121 route_open_frame_with_admission_facts(
7122 3,
7123 "target",
7124 unique_project_root("admission-facts"),
7125 Some(ConsumerIdentity {
7126 module_id: "fed".to_string(),
7127 launch_nonce: "fed-nonce".to_string(),
7128 }),
7129 None,
7130 ),
7131 )
7132 .await
7133 .unwrap()
7134 });
7135 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7136 .await
7137 .expect("no reserved route.bind within 5s")
7138 .expect("target control channel closed before reserved route.bind");
7139 handler
7140 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
7141 .await
7142 .unwrap();
7143 assert!(reserved_open.await.unwrap().is_empty());
7144 let _ = reserved_rx.recv().await.unwrap();
7145
7146 forwarding
7147 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
7148 .unwrap();
7149 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
7150 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
7151 module_id: Some("target".to_string()),
7152 })
7153 .unwrap();
7154 let census_frame =
7155 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
7156 let response = handler
7157 .handle_control_frame(&census_ctx, census_frame)
7158 .await
7159 .unwrap()
7160 .pop()
7161 .unwrap();
7162 let actual: Value = serde_json::from_slice(&response.body).unwrap();
7163 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
7164 assert!(matches!(
7165 decoded,
7166 ClientControlResponse::SupervisorRoutes { .. }
7167 ));
7168 let routes = actual["modules"][0]["routes"].as_array().unwrap();
7169 assert_eq!(routes.len(), 2);
7170 assert!(routes.iter().all(|route| route["draining"] == true));
7171 assert!(
7174 routes.iter().all(|route| route["drain_reason"] == "reload"),
7175 "draining routes must name the drain's reason: {routes:?}"
7176 );
7177 assert!(routes.iter().any(|route| {
7178 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
7179 }));
7180 assert!(routes.iter().any(|route| {
7181 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
7182 }));
7183
7184 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
7185 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
7186 if std::env::var_os("UPDATE_GOLDEN").is_some() {
7187 std::fs::write(
7188 &golden_path,
7189 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
7190 )
7191 .unwrap();
7192 }
7193 let expected: Value =
7194 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
7195 assert_eq!(actual, expected);
7196 }
7197
7198 async fn query_live_roots(
7199 handler: &ControlHandler,
7200 module_ctx: &RouteCtx,
7201 ) -> ModuleControlResponseToModule {
7202 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
7203 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
7204 let response = handler
7205 .handle_control_frame(module_ctx, frame)
7206 .await
7207 .unwrap()
7208 .pop()
7209 .unwrap();
7210 serde_json::from_slice(&response.body).unwrap()
7211 }
7212
7213 #[tokio::test(start_paused = true)]
7214 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
7215 let registry = Arc::new(Registry::default());
7216 let forwarding = Arc::new(ForwardingTable::default());
7217 let handler = ControlHandler::with_forwarding(registry, forwarding);
7218 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
7219 handler
7220 .handle_control_frame(&target_ctx, hello_frame("target", PROTOCOL_VERSION, 1))
7221 .await
7222 .unwrap();
7223 let root = unique_project_root("live-roots-known");
7224 let path = ProjectRootId::from_path_allowing_missing(root.path())
7225 .unwrap()
7226 .as_path()
7227 .to_path_buf();
7228 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
7229 let open_handler = handler.clone();
7230 let opened = tokio::spawn(async move {
7231 open_handler
7232 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
7233 .await
7234 .unwrap()
7235 });
7236 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7237 .await
7238 .unwrap()
7239 .unwrap();
7240 handler
7241 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
7242 .await
7243 .unwrap();
7244 assert!(opened.await.unwrap().is_empty());
7245 let _ = client_rx.recv().await.unwrap();
7246
7247 let root = unique_project_root("live-roots-pending");
7248 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
7249 .unwrap()
7250 .as_path()
7251 .to_path_buf();
7252 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
7253 let open_handler = handler.clone();
7254 let pending = tokio::spawn(async move {
7255 open_handler
7256 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
7257 .await
7258 .unwrap()
7259 });
7260 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7261 .await
7262 .unwrap()
7263 .unwrap();
7264 let actual = query_live_roots(&handler, &target_ctx).await;
7265 let ModuleControlResponseToModule::LiveRoots {
7266 roots,
7267 unknown_root_bindings,
7268 total_bindings,
7269 } = actual
7270 else {
7271 panic!("expected live roots")
7272 };
7273 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
7274 assert_eq!(unknown_root_bindings, 0);
7275 assert_eq!(
7276 roots.len(),
7277 2,
7278 "root-known arm must retain each canonical root"
7279 );
7280 assert_eq!(
7281 total_bindings,
7282 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
7283 );
7284 let counts = roots
7285 .iter()
7286 .map(|root| (root.project_root.clone(), root.bound, root.pending))
7287 .collect::<Vec<_>>();
7288 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
7289 expected.sort_by(|a, b| a.0.cmp(&b.0));
7290 assert_eq!(
7291 counts, expected,
7292 "roots must sort by path and count pending separately"
7293 );
7294 handler
7295 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
7296 .await
7297 .unwrap();
7298 assert!(pending.await.unwrap().is_empty());
7299 }
7300
7301 #[tokio::test(start_paused = true)]
7302 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
7303 let forwarding = Arc::new(ForwardingTable::default());
7304 let handler =
7305 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
7306 let (target_ctx, _target_rx) = route_ctx(ConnectionId::new(311));
7307 handler
7308 .handle_control_frame(&target_ctx, hello_frame("target", PROTOCOL_VERSION, 1))
7309 .await
7310 .unwrap();
7311 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
7312 let pending = forwarding
7313 .begin_route_bind_relay_for_test(
7314 client_ctx.connection_id,
7315 client_ctx.egress.clone(),
7316 2,
7317 "target",
7318 )
7319 .unwrap();
7320 forwarding
7321 .complete_pending_relay(
7322 target_ctx.connection_id,
7323 pending.corr,
7324 RouteBindRelayOutcome::Accepted,
7325 )
7326 .unwrap();
7327 let actual = query_live_roots(&handler, &target_ctx).await;
7328 let ModuleControlResponseToModule::LiveRoots {
7329 roots,
7330 unknown_root_bindings,
7331 total_bindings,
7332 } = actual
7333 else {
7334 panic!("expected live roots")
7335 };
7336 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
7337 assert_eq!(
7338 unknown_root_bindings, 1,
7339 "unknown-root arm must not read as no bindings"
7340 );
7341 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
7342 assert_eq!(
7343 total_bindings,
7344 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
7345 );
7346 }
7347
7348 #[tokio::test(start_paused = true)]
7349 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
7350 let handler = ControlHandler::with_forwarding(
7351 Arc::new(Registry::default()),
7352 Arc::new(ForwardingTable::default()),
7353 );
7354 let (first_ctx, _first_rx) = route_ctx(ConnectionId::new(315));
7355 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
7356 handler
7357 .handle_control_frame(&first_ctx, hello_frame("first", PROTOCOL_VERSION, 1))
7358 .await
7359 .unwrap();
7360 handler
7361 .handle_control_frame(&second_ctx, hello_frame("second", PROTOCOL_VERSION, 2))
7362 .await
7363 .unwrap();
7364 let root = unique_project_root("second-only");
7365 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
7366 let cloned = handler.clone();
7367 let open = tokio::spawn(async move {
7368 cloned
7369 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
7370 .await
7371 .unwrap()
7372 });
7373 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
7374 .await
7375 .unwrap()
7376 .unwrap();
7377 let first = query_live_roots(&handler, &first_ctx).await;
7378 let second = query_live_roots(&handler, &second_ctx).await;
7379 assert!(
7380 matches!(
7381 first,
7382 ModuleControlResponseToModule::LiveRoots {
7383 total_bindings: 0,
7384 ..
7385 }
7386 ),
7387 "cross-module scope must not expose another module's roots"
7388 );
7389 assert!(
7390 matches!(
7391 second,
7392 ModuleControlResponseToModule::LiveRoots {
7393 total_bindings: 1,
7394 ..
7395 }
7396 ),
7397 "second module must see its pending route"
7398 );
7399 handler
7400 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
7401 .await
7402 .unwrap();
7403 assert!(open.await.unwrap().is_empty());
7404 }
7405
7406 #[tokio::test(start_paused = true)]
7407 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
7408 let handler = ControlHandler::with_forwarding(
7409 Arc::new(Registry::default()),
7410 Arc::new(ForwardingTable::default()),
7411 );
7412 let (target_ctx, _target_rx) = route_ctx(ConnectionId::new(321));
7413 handler
7414 .handle_control_frame(&target_ctx, hello_frame("target", PROTOCOL_VERSION, 1))
7415 .await
7416 .unwrap();
7417 let actual = query_live_roots(&handler, &target_ctx).await;
7418 let ModuleControlResponseToModule::LiveRoots {
7419 roots,
7420 unknown_root_bindings,
7421 total_bindings,
7422 } = actual
7423 else {
7424 panic!("expected live roots")
7425 };
7426 assert!(roots.is_empty());
7427 assert_eq!(unknown_root_bindings, 0);
7428 assert_eq!(total_bindings, 0);
7429 assert_eq!(
7430 total_bindings,
7431 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
7432 );
7433 }
7434
7435 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
7442 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
7443 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
7444 let text = std::fs::read_to_string(&path)
7445 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
7446 let vectors: Vec<(String, Value)> = text
7447 .lines()
7448 .filter(|line| !line.trim().is_empty())
7449 .map(|line| {
7450 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
7451 let id = entry["corpus_id"]
7452 .as_str()
7453 .expect("every vector carries a corpus_id")
7454 .to_string();
7455 (id, entry["package"].clone())
7456 })
7457 .collect();
7458 assert_eq!(
7462 vectors.len(),
7463 3,
7464 "vendored fed corpus changed size; re-sync from subc-federation"
7465 );
7466
7467 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
7482 let richest = vectors
7483 .iter()
7484 .filter_map(|(_, package)| package.as_object())
7485 .map(|object| {
7486 object
7487 .keys()
7488 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
7489 .count()
7490 })
7491 .max()
7492 .unwrap_or(0);
7493 assert!(
7494 richest >= 2,
7495 "vendored corpus no longer carries a package with unmodelled fields, \
7496 so the relay test can no longer distinguish a verbatim relay from a lossy one"
7497 );
7498
7499 vectors
7500 }
7501
7502 #[tokio::test]
7510 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
7511 for (corpus_id, package) in fed_admission_facts_vectors() {
7512 let registry = Arc::new(Registry::default());
7513 let forwarding = Arc::new(ForwardingTable::default());
7514 let supervisor = SupervisorHandle::new();
7515 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7516 let handler =
7517 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
7518 .with_supervisor(supervisor)
7519 .with_admission_facts_config(
7520 Some("fed".to_string()),
7521 Some(vec!["target".to_string()]),
7522 );
7523
7524 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
7525 handler
7526 .handle_control_frame(&target_ctx, hello_frame("target", PROTOCOL_VERSION, 1))
7527 .await
7528 .unwrap();
7529
7530 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
7531 let route_handler = handler.clone();
7532 let expected = package.clone();
7533 let route_task = tokio::spawn(async move {
7534 route_handler
7535 .handle_control_frame(
7536 &client_ctx,
7537 route_open_frame_with_admission_facts(
7538 20,
7539 "target",
7540 unique_project_root("admission-facts"),
7541 Some(subc_control::ConsumerIdentity {
7542 module_id: "fed".to_string(),
7543 launch_nonce: "fed-nonce".to_string(),
7544 }),
7545 Some(package),
7546 ),
7547 )
7548 .await
7549 .unwrap()
7550 });
7551
7552 let bind_frame = target_rx.recv().await.unwrap();
7553 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
7554 let ModuleControlRequest::RouteBind {
7555 admission_facts, ..
7556 } = bind
7557 else {
7558 panic!("{corpus_id}: expected route.bind")
7559 };
7560 assert_eq!(
7561 admission_facts,
7562 Some(expected),
7563 "{corpus_id}: relay must not add, drop or reshape any field"
7564 );
7565
7566 handler
7567 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
7568 .await
7569 .unwrap();
7570 route_task.await.unwrap();
7571 }
7572 }
7573
7574 #[tokio::test]
7575 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
7576 let registry = Arc::new(Registry::default());
7577 let forwarding = Arc::new(ForwardingTable::default());
7578 let supervisor = SupervisorHandle::new();
7579 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7580 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
7581 let handler =
7582 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
7583 .with_supervisor(supervisor)
7584 .with_admission_facts_config(
7585 Some("fed".to_string()),
7586 Some(vec!["target".to_string()]),
7587 );
7588
7589 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
7590 handler
7591 .handle_control_frame(&target_ctx, hello_frame("target", PROTOCOL_VERSION, 1))
7592 .await
7593 .unwrap();
7594 let (other_ctx, _other_rx) = route_ctx(ConnectionId::new(71));
7595 handler
7596 .handle_control_frame(&other_ctx, hello_frame("other", PROTOCOL_VERSION, 2))
7597 .await
7598 .unwrap();
7599
7600 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
7601 let expected_facts = facts.clone();
7602 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
7603 let route_handler = handler.clone();
7604 let route_task = tokio::spawn(async move {
7605 route_handler
7606 .handle_control_frame(
7607 &client_ctx,
7608 route_open_frame_with_admission_facts(
7609 10,
7610 "target",
7611 unique_project_root("admission-facts"),
7612 Some(subc_control::ConsumerIdentity {
7613 module_id: "fed".to_string(),
7614 launch_nonce: "fed-nonce".to_string(),
7615 }),
7616 Some(facts.clone()),
7617 ),
7618 )
7619 .await
7620 .unwrap()
7621 });
7622 let bind_frame = target_rx.recv().await.unwrap();
7623 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
7624 let ModuleControlRequest::RouteBind {
7625 admission_facts, ..
7626 } = bind
7627 else {
7628 panic!("expected route.bind")
7629 };
7630 assert_eq!(admission_facts, Some(expected_facts));
7631 handler
7632 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
7633 .await
7634 .unwrap();
7635 assert!(route_task.await.unwrap().is_empty());
7636 assert!(matches!(
7637 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
7638 .unwrap(),
7639 ClientControlResponse::RouteOpen { .. }
7640 ));
7641
7642 let direct = handler
7643 .handle_control_frame(
7644 &route_ctx(ConnectionId::new(73)).0,
7645 route_open_frame_with_admission_facts(
7646 11,
7647 "target",
7648 unique_project_root("admission-facts"),
7649 None,
7650 Some(json!({"x": 1})),
7651 ),
7652 )
7653 .await
7654 .unwrap();
7655 assert_eq!(
7656 parse_error(&direct[0])["code"],
7657 "admission_facts_not_permitted"
7658 );
7659
7660 let different_reserved = handler
7661 .handle_control_frame(
7662 &route_ctx(ConnectionId::new(77)).0,
7663 route_open_frame_with_admission_facts(
7664 15,
7665 "target",
7666 unique_project_root("admission-facts"),
7667 Some(subc_control::ConsumerIdentity {
7668 module_id: "other".to_string(),
7669 launch_nonce: "other-nonce".to_string(),
7670 }),
7671 Some(json!({"x": 1})),
7672 ),
7673 )
7674 .await
7675 .unwrap();
7676 assert_eq!(
7677 parse_error(&different_reserved[0])["code"],
7678 "admission_facts_not_permitted"
7679 );
7680
7681 let other_target = handler
7682 .handle_control_frame(
7683 &route_ctx(ConnectionId::new(74)).0,
7684 route_open_frame_with_admission_facts(
7685 12,
7686 "other",
7687 unique_project_root("admission-facts"),
7688 Some(subc_control::ConsumerIdentity {
7689 module_id: "fed".to_string(),
7690 launch_nonce: "fed-nonce".to_string(),
7691 }),
7692 Some(json!({"x": 1})),
7693 ),
7694 )
7695 .await
7696 .unwrap();
7697 assert_eq!(
7698 parse_error(&other_target[0])["code"],
7699 "admission_facts_target_not_allowed"
7700 );
7701
7702 let nonexistent = handler
7703 .handle_control_frame(
7704 &route_ctx(ConnectionId::new(75)).0,
7705 route_open_frame_with_admission_facts(
7706 13,
7707 "missing",
7708 unique_project_root("admission-facts"),
7709 None,
7710 Some(json!({"x": 1})),
7711 ),
7712 )
7713 .await
7714 .unwrap();
7715 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
7716
7717 let described = handler
7718 .handle_control_frame(
7719 &route_ctx(ConnectionId::new(76)).0,
7720 Frame::build(
7721 FrameType::Request,
7722 control_flags(),
7723 0,
7724 0,
7725 14,
7726 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
7727 )
7728 .unwrap(),
7729 )
7730 .await
7731 .unwrap();
7732 let ClientControlResponse::ServerDescribe { capabilities, .. } =
7733 serde_json::from_slice(&described[0].body).unwrap()
7734 else {
7735 panic!("expected server.describe response")
7736 };
7737 assert!(capabilities
7738 .iter()
7739 .any(|cap| cap == "admission_facts_relay_v1"));
7740 }
7741
7742 #[tokio::test]
7743 async fn admission_facts_without_configured_carrier_are_rejected() {
7744 let registry = Arc::new(Registry::default());
7745 let forwarding = Arc::new(ForwardingTable::default());
7746 let handler = ControlHandler::with_forwarding(registry, forwarding);
7747 let (target_ctx, _) = route_ctx(ConnectionId::new(78));
7748 handler
7749 .handle_control_frame(&target_ctx, hello_frame("target", PROTOCOL_VERSION, 1))
7750 .await
7751 .unwrap();
7752
7753 let responses = handler
7754 .handle_control_frame(
7755 &route_ctx(ConnectionId::new(79)).0,
7756 route_open_frame_with_admission_facts(
7757 16,
7758 "target",
7759 unique_project_root("admission-facts"),
7760 None,
7761 Some(json!({"x": 1})),
7762 ),
7763 )
7764 .await
7765 .unwrap();
7766 assert_eq!(
7767 parse_error(&responses[0])["code"],
7768 "admission_facts_not_permitted"
7769 );
7770 }
7771
7772 #[tokio::test]
7773 async fn route_open_relays_consumer_capabilities_verbatim() {
7774 let registry = Arc::new(Registry::default());
7775 let forwarding = Arc::new(ForwardingTable::default());
7776 let handler =
7777 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
7778 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
7779 handler
7780 .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
7781 .await
7782 .unwrap();
7783
7784 let expected = vec!["elicitation".to_string(), "roots".to_string()];
7785 let expected_for_request = expected.clone();
7786 let project_root = unique_project_root("consumer-capabilities-present");
7787 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
7788 let route_handler = handler.clone();
7789 let route_task = tokio::spawn(async move {
7790 route_handler
7791 .handle_control_frame(
7792 &client_ctx,
7793 route_open_frame_with_consumer_capabilities(
7794 401,
7795 "aft",
7796 project_root,
7797 Some(expected_for_request),
7798 ),
7799 )
7800 .await
7801 .unwrap()
7802 });
7803 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7804 .await
7805 .unwrap()
7806 .unwrap();
7807 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
7808 let ModuleControlRequest::RouteBind {
7809 consumer_capabilities,
7810 ..
7811 } = bind
7812 else {
7813 panic!("expected route.bind request, got {bind:?}");
7814 };
7815 assert_eq!(consumer_capabilities, Some(expected.clone()));
7816
7817 handler
7818 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
7819 .await
7820 .unwrap();
7821 let route_response = route_task.await.unwrap();
7822 assert!(route_response.is_empty());
7823 let published = client_rx.recv().await.unwrap();
7824 assert!(matches!(
7825 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
7826 ClientControlResponse::RouteOpen { .. }
7827 ));
7828 }
7829
7830 #[tokio::test]
7831 async fn route_open_without_consumer_capabilities_relays_none() {
7832 let registry = Arc::new(Registry::default());
7833 let forwarding = Arc::new(ForwardingTable::default());
7834 let handler =
7835 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
7836 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
7837 handler
7838 .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
7839 .await
7840 .unwrap();
7841
7842 let project_root = unique_project_root("consumer-capabilities-absent");
7843 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
7844 let route_handler = handler.clone();
7845 let route_task = tokio::spawn(async move {
7846 route_handler
7847 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
7848 .await
7849 .unwrap()
7850 });
7851 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7852 .await
7853 .unwrap()
7854 .unwrap();
7855 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
7856 let ModuleControlRequest::RouteBind {
7857 consumer_capabilities,
7858 ..
7859 } = bind
7860 else {
7861 panic!("expected route.bind request, got {bind:?}");
7862 };
7863 assert_eq!(consumer_capabilities, None);
7864
7865 handler
7866 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
7867 .await
7868 .unwrap();
7869 let route_response = route_task.await.unwrap();
7870 assert!(route_response.is_empty());
7871 let published = client_rx.recv().await.unwrap();
7872 assert!(matches!(
7873 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
7874 ClientControlResponse::RouteOpen { .. }
7875 ));
7876 }
7877
7878 #[tokio::test]
7879 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
7880 let registry = Arc::new(Registry::default());
7881 let forwarding = Arc::new(ForwardingTable::default());
7882 let handler =
7883 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
7884 .with_health_probe_timeout(Duration::from_secs(5));
7885 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
7886 let responses = handler
7887 .handle_control_frame(
7888 &module_ctx,
7889 non_routable_hello_frame_with_control_ops(
7890 "mcp",
7891 300,
7892 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
7893 ),
7894 )
7895 .await
7896 .unwrap();
7897 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
7898 assert!(registry
7899 .get_module("mcp")
7900 .unwrap()
7901 .unwrap()
7902 .manifest
7903 .provides
7904 .is_empty());
7905
7906 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
7907 let route_response = handler
7908 .handle_control_frame(
7909 &route_client_ctx,
7910 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
7911 )
7912 .await
7913 .unwrap();
7914 assert_eq!(route_response[0].header.ty, FrameType::Error);
7915 assert_eq!(
7916 parse_error(&route_response[0])["code"],
7917 "target_unavailable"
7918 );
7919 assert!(parse_error(&route_response[0])["message"]
7920 .as_str()
7921 .unwrap()
7922 .contains("does not provide the requested target"));
7923 assert!(module_rx.try_recv().is_err());
7924
7925 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
7926 let health_handler = handler.clone();
7927 let health_task = tokio::spawn(async move {
7928 health_handler
7929 .handle_control_frame(
7930 &health_client_ctx,
7931 supervisor_health_probe_frame(302, "mcp"),
7932 )
7933 .await
7934 .unwrap()
7935 });
7936 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7937 .await
7938 .unwrap()
7939 .unwrap();
7940 assert_eq!(
7941 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
7942 ModuleControlRequest::HealthCheck {}
7943 );
7944 handler
7945 .handle_control_frame(
7946 &module_ctx,
7947 health_response(health_frame.header.corr, HealthStatus::Ok),
7948 )
7949 .await
7950 .unwrap();
7951 let health_response = health_task.await.unwrap();
7952 assert_eq!(health_response[0].header.ty, FrameType::Response);
7953 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
7954 ClientControlResponse::SupervisorHealthProbe {
7955 module_id, status, ..
7956 } => {
7957 assert_eq!(module_id, "mcp");
7958 assert_eq!(status, HealthStatus::Ok);
7959 }
7960 other => panic!("unexpected health response: {other:?}"),
7961 }
7962
7963 forwarding
7968 .cleanup_connection(module_ctx.connection_id)
7969 .unwrap();
7970 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
7971 let cleanup_response = tokio::time::timeout(
7972 Duration::from_millis(200),
7973 handler.handle_control_frame(
7974 &cleanup_probe_ctx,
7975 supervisor_health_probe_frame(303, "mcp"),
7976 ),
7977 )
7978 .await
7979 .expect("probe should fail immediately when the control lane is gone")
7980 .unwrap();
7981 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
7982 assert_eq!(
7983 parse_error(&cleanup_response[0])["code"],
7984 "target_unavailable"
7985 );
7986 assert!(parse_error(&cleanup_response[0])["message"]
7987 .as_str()
7988 .unwrap()
7989 .contains("no module connection"));
7990
7991 handler
7992 .cleanup_connection(module_ctx.connection_id)
7993 .unwrap();
7994 }
7995
7996 #[tokio::test]
7997 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
7998 let registry = Arc::new(Registry::default());
7999 let supervisor_handle = SupervisorHandle::new();
8000 let supervisor =
8001 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
8002 .with_handle(supervisor_handle.clone())
8003 .with_connection_file_path(
8004 std::env::temp_dir()
8005 .join(format!("subc-route-open-warming-{}", std::process::id())),
8006 );
8007 let module = supervisor
8008 .supervise_configured(
8009 ModuleSpec {
8010 module_id: "warming".to_string(),
8011 program: fake_aft_stub_path(),
8012 args: Vec::new(),
8013 env: Vec::new(),
8014 reserved: false,
8015 reserved_prefixes: Vec::new(),
8016 protocol: ModuleProtocol::Subc,
8017 overlap: Default::default(),
8018 },
8019 true,
8020 )
8021 .unwrap();
8022 assert_eq!(module.state().unwrap(), ModuleState::Running);
8023
8024 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8025 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
8026 let response = handler
8027 .handle_control_frame(
8028 &ctx,
8029 route_open_frame(304, "warming", unique_project_root("warming")),
8030 )
8031 .await
8032 .unwrap();
8033 module.stop().await.unwrap();
8034
8035 assert_eq!(response[0].header.ty, FrameType::Error);
8036 let error = parse_error(&response[0]);
8037 assert_eq!(error["code"], "module_warming");
8038 assert!(error["message"]
8039 .as_str()
8040 .unwrap()
8041 .contains("state=running, enabled=true, live=false"));
8042 }
8043
8044 #[tokio::test]
8049 async fn route_open_refusal_names_the_check_that_refused() {
8050 let handler = ControlHandler::new(Arc::new(Registry::default()));
8051 let capture = EventCapture::default();
8052 let _subscriber =
8053 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8054 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
8055 let response = handler
8056 .handle_control_frame(
8057 &ctx,
8058 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
8059 )
8060 .await
8061 .unwrap();
8062
8063 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
8064 let event = capture
8065 .events()
8066 .into_iter()
8067 .find(|event| {
8068 event.target == "control"
8069 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
8070 })
8071 .expect("route.open refusal event");
8072 assert_eq!(
8073 event.fields.get("reason"),
8074 Some(&"\"not_registered\"".to_string())
8075 );
8076 }
8077
8078 #[tokio::test]
8079 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
8080 let registry = Arc::new(Registry::default());
8081 let supervisor_handle = SupervisorHandle::new();
8082 let supervisor =
8083 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
8084 .with_handle(supervisor_handle.clone())
8085 .with_connection_file_path(std::env::temp_dir().join(format!(
8086 "subc-route-open-refusal-info-{}",
8087 std::process::id()
8088 )));
8089 let module = supervisor
8090 .supervise_configured(
8091 ModuleSpec {
8092 module_id: "warming".to_string(),
8093 program: fake_aft_stub_path(),
8094 args: Vec::new(),
8095 env: Vec::new(),
8096 reserved: false,
8097 reserved_prefixes: Vec::new(),
8098 protocol: ModuleProtocol::Subc,
8099 overlap: Default::default(),
8100 },
8101 true,
8102 )
8103 .unwrap();
8104 assert_eq!(module.state().unwrap(), ModuleState::Running);
8105
8106 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8107 assert!(handler
8108 .counters()
8109 .snapshot()
8110 .get("route_open_refused_by_code")
8111 .is_none());
8112 let capture = EventCapture::default();
8113 let _subscriber =
8114 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8115 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
8116 let response = handler
8117 .handle_control_frame(
8118 &ctx,
8119 route_open_frame(394, "warming", unique_project_root("refusal-info")),
8120 )
8121 .await
8122 .unwrap();
8123 module.stop().await.unwrap();
8124
8125 assert_eq!(parse_error(&response[0])["code"], "module_warming");
8126 let event = capture
8127 .events()
8128 .into_iter()
8129 .find(|event| {
8130 event.target == "control"
8131 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
8132 })
8133 .expect("route.open refusal event");
8134 assert_eq!(
8135 event.fields.get("module_id"),
8136 Some(&"\"warming\"".to_string())
8137 );
8138 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
8139 assert_eq!(
8140 event.fields.get("reason"),
8141 Some(&"\"supervised_not_registered\"".to_string())
8142 );
8143 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
8144 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
8145 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
8146 assert_eq!(
8147 handler.counters().snapshot()["route_open_refused_by_code"],
8148 json!({ "module_warming": 1 })
8149 );
8150 }
8151
8152 #[tokio::test(flavor = "current_thread")]
8153 async fn route_open_unknown_module_escapes_target_module_id() {
8154 let handler = ControlHandler::new(Arc::new(Registry::default()));
8155 let capture = EventCapture::default();
8156 let _subscriber =
8157 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8158 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
8159 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
8160 let response = handler
8161 .handle_control_frame(
8162 &ctx,
8163 route_open_frame(
8164 395,
8165 hostile_module_id,
8166 unique_project_root("hostile-target-module-id"),
8167 ),
8168 )
8169 .await
8170 .unwrap();
8171
8172 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
8173 let event = capture
8174 .events()
8175 .into_iter()
8176 .find(|event| {
8177 event.target == "control"
8178 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
8179 })
8180 .expect("route.open unknown-module refusal event");
8181 let logged = event.fields.get("module_id").expect("module_id field");
8182 assert!(!logged.bytes().any(|byte| byte < 0x20));
8183 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
8184 }
8185
8186 #[tokio::test(flavor = "current_thread")]
8187 async fn route_open_module_rejection_uses_daemon_counter_key() {
8188 let registry = Arc::new(Registry::default());
8189 let forwarding = Arc::new(ForwardingTable::default());
8190 let handler =
8191 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8192 let module_connection = ConnectionId::new(95);
8193 let (module_ctx, mut module_rx) = route_ctx(module_connection);
8194 handler
8195 .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 395))
8196 .await
8197 .unwrap();
8198
8199 let client_connection = ConnectionId::new(96);
8200 let (client_ctx, _client_rx) = route_ctx(client_connection);
8201 let capture = EventCapture::default();
8202 let _subscriber =
8203 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8204 let (route_task, bind) = relay_route_open(
8205 &handler,
8206 client_connection,
8207 &client_ctx.egress,
8208 &mut module_rx,
8209 396,
8210 "aft",
8211 "hostile-module-code",
8212 )
8213 .await;
8214 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
8215 let rejection = Frame::build(
8216 FrameType::Error,
8217 control_flags(),
8218 0,
8219 0,
8220 bind.header.corr,
8221 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
8222 )
8223 .unwrap();
8224 handler
8225 .handle_control_frame(&module_ctx, rejection)
8226 .await
8227 .unwrap();
8228
8229 let response = route_task.await.unwrap();
8230 assert_eq!(parse_error(&response[0])["code"], hostile_code);
8231 let counters = handler.counters().snapshot();
8232 assert_eq!(
8233 counters["route_open_refused_by_code"],
8234 json!({ "module_rejected": 1 })
8235 );
8236 assert!(counters["route_open_refused_by_code"]
8237 .get(hostile_code)
8238 .is_none());
8239
8240 let event = capture
8241 .events()
8242 .into_iter()
8243 .find(|event| {
8244 event.target == "control"
8245 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
8246 })
8247 .expect("route.open module-rejection refusal event");
8248 let logged = event.fields.get("module_code").expect("module_code field");
8249 assert!(!logged.bytes().any(|byte| byte < 0x20));
8250 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
8251 }
8252
8253 #[tokio::test]
8254 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
8255 let registry = Arc::new(Registry::default());
8256 let supervisor_handle = SupervisorHandle::new();
8257 let missing_program = std::env::temp_dir().join(format!(
8258 "subc-route-open-missing-program-{}",
8259 std::process::id()
8260 ));
8261 let supervisor =
8262 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
8263 .with_handle(supervisor_handle.clone());
8264 let module = supervisor
8265 .supervise_configured(
8266 ModuleSpec {
8267 module_id: "failed".to_string(),
8268 program: missing_program,
8269 args: Vec::new(),
8270 env: Vec::new(),
8271 reserved: false,
8272 reserved_prefixes: Vec::new(),
8273 protocol: ModuleProtocol::Subc,
8274 overlap: Default::default(),
8275 },
8276 true,
8277 )
8278 .unwrap();
8279 assert_eq!(module.state().unwrap(), ModuleState::Failed);
8280
8281 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8282 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
8283 let response = handler
8284 .handle_control_frame(
8285 &ctx,
8286 route_open_frame(305, "failed", unique_project_root("failed")),
8287 )
8288 .await
8289 .unwrap();
8290
8291 assert_eq!(response[0].header.ty, FrameType::Error);
8292 let error = parse_error(&response[0]);
8293 assert_eq!(error["code"], "target_unavailable");
8294 assert!(error["message"]
8295 .as_str()
8296 .unwrap()
8297 .contains("state=failed, enabled=true, live=false"));
8298 }
8299
8300 #[tokio::test]
8301 async fn route_open_role_mismatch_remains_target_unavailable() {
8302 let registry = Arc::new(Registry::default());
8303 let handler = ControlHandler::new(Arc::clone(®istry));
8304 handler
8305 .handle_control(
8306 ConnectionId::new(41),
8307 non_routable_hello_frame_with_control_ops("health-only", 306, None),
8308 )
8309 .unwrap();
8310
8311 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
8312 let response = handler
8313 .handle_control_frame(
8314 &ctx,
8315 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
8316 )
8317 .await
8318 .unwrap();
8319
8320 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
8321 assert!(parse_error(&response[0])["message"]
8322 .as_str()
8323 .unwrap()
8324 .contains("does not provide the requested target"));
8325 }
8326
8327 #[tokio::test]
8328 async fn route_open_inactive_registration_remains_target_unavailable() {
8329 let registry = Arc::new(Registry::default());
8330 let handler = ControlHandler::new(Arc::clone(®istry));
8331 handler
8332 .handle_control(
8333 ConnectionId::new(43),
8334 hello_frame("inactive", PROTOCOL_VERSION, 308),
8335 )
8336 .unwrap();
8337 assert!(registry
8338 .set_module_state_for_test("inactive", ChannelState::Closed)
8339 .unwrap());
8340
8341 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
8342 let response = handler
8343 .handle_control_frame(
8344 &ctx,
8345 route_open_frame(309, "inactive", unique_project_root("inactive")),
8346 )
8347 .await
8348 .unwrap();
8349
8350 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
8351 assert!(parse_error(&response[0])["message"]
8352 .as_str()
8353 .unwrap()
8354 .contains("is not active"));
8355 }
8356
8357 #[tokio::test]
8358 async fn late_health_reply_is_recorded_through_the_module_response_path() {
8359 let registry = Arc::new(Registry::default());
8360 let forwarding = Arc::new(ForwardingTable::default());
8361 let supervisor_handle = SupervisorHandle::new();
8362 let supervisor = Supervisor::new(Arc::clone(®istry), crate::RestartPolicy::default())
8363 .with_forwarding(Arc::clone(&forwarding))
8364 .with_handle(supervisor_handle.clone());
8365 let module = supervisor
8366 .supervise_configured(
8367 crate::ModuleSpec {
8368 module_id: "late-health-response".to_string(),
8369 program: PathBuf::from("disabled-module"),
8370 args: Vec::new(),
8371 env: Vec::new(),
8372 reserved: false,
8373 reserved_prefixes: Vec::new(),
8374 protocol: ModuleProtocol::Subc,
8375 overlap: Default::default(),
8376 },
8377 false,
8378 )
8379 .unwrap();
8380 let handler =
8381 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8382 .with_supervisor(supervisor_handle);
8383 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
8384 handler
8385 .handle_control_frame(
8386 &module_ctx,
8387 hello_frame_with_control_ops(
8388 "late-health-response",
8389 PROTOCOL_VERSION,
8390 7,
8391 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
8392 ),
8393 )
8394 .await
8395 .unwrap();
8396 let probe_started_at = Instant::now() - Duration::from_millis(80);
8397 let pending = forwarding
8398 .begin_health_probe_rpc_for(
8399 "late-health-response",
8400 MODULE_CONTROL_OP_HEALTH_CHECK,
8401 probe_started_at,
8402 Instant::now() - Duration::from_millis(1),
8403 )
8404 .unwrap();
8405 assert!(forwarding
8406 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
8407 .unwrap());
8408
8409 let responses = handler
8410 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
8411 .await
8412 .unwrap();
8413
8414 assert!(responses.is_empty());
8415 let health = module.status().unwrap().health;
8416 assert_eq!(health.late_answer_count, 1);
8417 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
8418 }
8419
8420 #[tokio::test]
8421 async fn health_probe_timeout_and_module_death_are_typed() {
8422 let registry = Arc::new(Registry::default());
8423 let forwarding = Arc::new(ForwardingTable::default());
8424 let handler =
8425 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8426 .with_health_probe_timeout(Duration::from_millis(50));
8427 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
8428 handler
8429 .handle_control_frame(
8430 &module_ctx,
8431 hello_frame_with_control_ops(
8432 "aft",
8433 PROTOCOL_VERSION,
8434 7,
8435 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
8436 ),
8437 )
8438 .await
8439 .unwrap();
8440
8441 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
8442 let responses = handler
8443 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
8444 .await
8445 .unwrap();
8446 assert_eq!(responses[0].header.ty, FrameType::Error);
8447 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
8448 let _ = module_rx.try_recv();
8449
8450 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
8451 let health_handler = handler.clone();
8452 let death_task = tokio::spawn(async move {
8453 health_handler
8454 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
8455 .await
8456 .unwrap()
8457 });
8458 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8459 .await
8460 .unwrap()
8461 .unwrap();
8462 handler
8463 .cleanup_connection(module_ctx.connection_id)
8464 .unwrap();
8465 let responses = death_task.await.unwrap();
8466 assert_eq!(responses[0].header.ty, FrameType::Error);
8467 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
8468 }
8469
8470 #[test]
8471 fn hello_requires_exact_protocol_version() {
8472 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
8473 let registry = Arc::new(Registry::default());
8474 let handler = ControlHandler::new(Arc::clone(®istry));
8475 let responses = handler
8476 .handle_control(
8477 ConnectionId::new(connection),
8478 hello_frame("aft", offered, 9),
8479 )
8480 .unwrap();
8481
8482 assert_eq!(responses.len(), 1);
8483 assert_eq!(responses[0].header.ty, FrameType::Error);
8484 let error = parse_error(&responses[0]);
8485 assert_eq!(error["code"], "version_unsupported");
8486 assert!(registry.get_module("aft").unwrap().is_none());
8487 assert_eq!(registry.active_registration_count().unwrap(), 0);
8488 }
8489 }
8490
8491 #[test]
8492 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
8493 let registry = Arc::new(Registry::default());
8494 let forwarding = Arc::new(ForwardingTable::default());
8495 let handler =
8496 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8497 let module_connection = ConnectionId::new(301);
8498 let registration = registry
8499 .register_with_control_ops(
8500 manifest("aft-push", PROTOCOL_VERSION),
8501 PROTOCOL_VERSION,
8502 module_connection,
8503 module_baseline_control_ops(),
8504 )
8505 .unwrap();
8506 let (module_tx, _module_rx) = mpsc::channel(8);
8507 let endpoint = forwarding
8508 .register_module_connection(
8509 module_connection,
8510 "aft-push".to_string(),
8511 PROTOCOL_VERSION,
8512 manifest_concurrency(®istration.manifest),
8513 FrameSink::new(module_tx),
8514 )
8515 .unwrap();
8516
8517 let unknown = Frame::build(
8519 FrameType::Push,
8520 control_flags(),
8521 0,
8522 0,
8523 5,
8524 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
8525 )
8526 .unwrap();
8527 let out = handler.handle_status_update(endpoint, unknown).unwrap();
8528 assert!(
8529 out.is_empty(),
8530 "unknown push op must be ignored, got {out:?}"
8531 );
8532
8533 let malformed = Frame::build(
8535 FrameType::Push,
8536 control_flags(),
8537 0,
8538 0,
8539 6,
8540 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
8541 )
8542 .unwrap();
8543 let out = handler.handle_status_update(endpoint, malformed).unwrap();
8544 assert_eq!(out.len(), 1);
8545 assert_eq!(out[0].header.ty, FrameType::Error);
8546 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
8547 }
8548
8549 #[test]
8550 fn hello_rejected_when_connection_already_owns_client_routes() {
8551 let registry = Arc::new(Registry::default());
8552 let forwarding = Arc::new(ForwardingTable::default());
8553 let handler =
8554 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8555 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
8557 let client_connection = ConnectionId::new(202);
8558
8559 let responses = handler
8562 .handle_control(
8563 client_connection,
8564 hello_frame("aft-second", PROTOCOL_VERSION, 9),
8565 )
8566 .unwrap();
8567 assert_eq!(responses[0].header.ty, FrameType::Error);
8568 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
8569 assert!(registry.get_module("aft-second").unwrap().is_none());
8570 }
8571
8572 #[test]
8573 fn reserved_module_hello_requires_matching_launch_nonce() {
8574 let registry = Arc::new(Registry::default());
8575 let supervisor = SupervisorHandle::new();
8576 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
8579 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
8580
8581 let no_nonce = handler
8583 .handle_control(
8584 ConnectionId::new(1),
8585 hello_frame("vault", PROTOCOL_VERSION, 1),
8586 )
8587 .unwrap();
8588 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
8589 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
8590 assert!(registry.get_module("vault").unwrap().is_none());
8591
8592 let wrong = handler
8594 .handle_control(
8595 ConnectionId::new(2),
8596 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
8597 )
8598 .unwrap();
8599 assert_eq!(wrong[0].header.ty, FrameType::Error);
8600 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
8601 assert!(registry.get_module("vault").unwrap().is_none());
8602
8603 let ok = handler
8605 .handle_control(
8606 ConnectionId::new(3),
8607 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
8608 )
8609 .unwrap();
8610 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
8611 assert!(registry.get_module("vault").unwrap().is_some());
8612 }
8613
8614 #[test]
8615 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
8616 let registry = Arc::new(Registry::default());
8617 let supervisor = SupervisorHandle::new();
8618 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
8619 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
8620 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
8621
8622 let squat = handler
8623 .handle_control(
8624 ConnectionId::new(1),
8625 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
8626 )
8627 .unwrap();
8628 assert_eq!(squat[0].header.ty, FrameType::Error);
8629 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
8630 assert!(parse_error(&squat[0])["message"]
8631 .as_str()
8632 .unwrap()
8633 .contains("fed:"));
8634
8635 let accepted_peer = handler
8636 .handle_control(
8637 ConnectionId::new(2),
8638 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
8639 )
8640 .unwrap();
8641 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
8642
8643 let accepted_short = handler
8644 .handle_control(
8645 ConnectionId::new(3),
8646 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
8647 )
8648 .unwrap();
8649 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
8650
8651 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
8652 let response = handler
8653 .handle_control(
8654 ConnectionId::new(conn),
8655 hello_frame(module_id, PROTOCOL_VERSION, conn),
8656 )
8657 .unwrap();
8658 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
8659 }
8660 }
8661
8662 #[test]
8663 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
8664 let registry = Arc::new(Registry::default());
8665 let supervisor = SupervisorHandle::new();
8666 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
8667 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
8668 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
8669 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
8670
8671 let owner_nonce = handler
8672 .handle_control(
8673 ConnectionId::new(1),
8674 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
8675 )
8676 .unwrap();
8677 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
8678 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
8679 assert!(registry.get_module("fed:special").unwrap().is_none());
8680
8681 let exact_nonce = handler
8682 .handle_control(
8683 ConnectionId::new(2),
8684 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
8685 )
8686 .unwrap();
8687 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
8688 assert!(registry.get_module("fed:special").unwrap().is_some());
8689 }
8690
8691 #[test]
8692 fn non_reserved_module_ignores_launch_nonce() {
8693 let registry = Arc::new(Registry::default());
8694 let handler = ControlHandler::new(Arc::clone(®istry));
8697 let no_nonce = handler
8698 .handle_control(
8699 ConnectionId::new(1),
8700 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
8701 )
8702 .unwrap();
8703 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
8704 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
8705
8706 let echoed_nonce = handler
8707 .handle_control(
8708 ConnectionId::new(2),
8709 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
8710 )
8711 .unwrap();
8712 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
8713 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
8714 }
8715
8716 #[test]
8717 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
8718 let handler = ControlHandler::default();
8719 let conn = ConnectionId::new(1);
8720 let malformed = Frame::build(
8721 FrameType::Hello,
8722 control_flags(),
8723 0,
8724 0,
8725 3,
8726 b"{not json".to_vec(),
8727 )
8728 .unwrap();
8729
8730 let error = handler.handle_control(conn, malformed).unwrap();
8731 assert_eq!(error[0].header.ty, FrameType::Error);
8732 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
8733
8734 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
8735 let pong = handler.handle_control(conn, ping).unwrap();
8736 assert_eq!(pong[0].header.ty, FrameType::Pong);
8737 assert_eq!(pong[0].header.corr, 4);
8738 }
8739
8740 #[test]
8741 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
8742 let registry = Arc::new(Registry::default());
8743 let handler = ControlHandler::new(Arc::clone(®istry));
8744
8745 handler
8746 .handle_control(
8747 ConnectionId::new(1),
8748 hello_frame("aft", PROTOCOL_VERSION, 1),
8749 )
8750 .unwrap();
8751 let duplicate = handler
8752 .handle_control(
8753 ConnectionId::new(2),
8754 hello_frame("aft", PROTOCOL_VERSION, 2),
8755 )
8756 .unwrap();
8757
8758 assert_eq!(duplicate[0].header.ty, FrameType::Error);
8759 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
8760 let registration = registry.get_module("aft").unwrap().unwrap();
8761 assert_eq!(registration.connection_id, ConnectionId::new(1));
8762 }
8763
8764 #[test]
8765 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
8766 let registry = Arc::new(Registry::default());
8767 let forwarding = Arc::new(ForwardingTable::default());
8768 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
8769 let handler =
8770 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8771 .with_process_liveness(process_liveness);
8772 let (ctx, route_channel, route_epoch) =
8773 bind_liveness_route(®istry, &forwarding, "aft-dead");
8774 let responses = handler
8775 .handle_route_poll(
8776 &ctx,
8777 route_poll_frame(41, PollKind::Liveness, route_channel),
8778 route_channel,
8779 route_epoch,
8780 PollKind::Liveness,
8781 )
8782 .unwrap();
8783
8784 assert_eq!(responses.len(), 1);
8785 assert_eq!(responses[0].header.ty, FrameType::Response);
8786 assert_route_poll_liveness(&responses[0], false);
8787 }
8788
8789 #[test]
8790 fn liveness_poll_without_process_source_uses_bound_route() {
8791 let registry = Arc::new(Registry::default());
8792 let forwarding = Arc::new(ForwardingTable::default());
8793 let handler =
8794 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8795 let (ctx, route_channel, route_epoch) =
8796 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
8797 let responses = handler
8798 .handle_route_poll(
8799 &ctx,
8800 route_poll_frame(42, PollKind::Liveness, route_channel),
8801 route_channel,
8802 route_epoch,
8803 PollKind::Liveness,
8804 )
8805 .unwrap();
8806
8807 assert_route_poll_liveness(&responses[0], true);
8808 }
8809
8810 #[test]
8811 fn liveness_poll_untracked_process_source_uses_bound_route() {
8812 let registry = Arc::new(Registry::default());
8813 let forwarding = Arc::new(ForwardingTable::default());
8814 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
8815 let handler =
8816 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8817 .with_process_liveness(process_liveness);
8818 let (ctx, route_channel, route_epoch) =
8819 bind_liveness_route(®istry, &forwarding, "aft-untracked");
8820 let responses = handler
8821 .handle_route_poll(
8822 &ctx,
8823 route_poll_frame(43, PollKind::Liveness, route_channel),
8824 route_channel,
8825 route_epoch,
8826 PollKind::Liveness,
8827 )
8828 .unwrap();
8829
8830 assert_route_poll_liveness(&responses[0], true);
8831 }
8832
8833 #[tokio::test]
8834 async fn unknown_op_returns_unknown_control_op() {
8835 let handler = ControlHandler::default();
8836 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
8837 let request = Frame::build(
8838 FrameType::Request,
8839 control_flags(),
8840 0,
8841 0,
8842 55,
8843 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
8844 )
8845 .unwrap();
8846
8847 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
8848
8849 assert_eq!(response.len(), 1);
8850 assert_eq!(response[0].header.ty, FrameType::Error);
8851 assert_eq!(response[0].header.corr, 55);
8852 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
8853 }
8854
8855 #[tokio::test]
8856 async fn supervisor_provenance_rejects_unknown_exact_module() {
8857 let handler = ControlHandler::default();
8858 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
8859 let request = Frame::build(
8860 FrameType::Request,
8861 control_flags(),
8862 0,
8863 0,
8864 57,
8865 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
8866 )
8867 .unwrap();
8868
8869 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
8870
8871 assert_eq!(response.len(), 1);
8872 assert_eq!(response[0].header.ty, FrameType::Error);
8873 assert_eq!(response[0].header.corr, 57);
8874 let error = parse_error(&response[0]);
8875 assert_eq!(error["code"], "unknown_module");
8876 assert_eq!(error["message"], "module_id 'missing' is not supervised");
8877 }
8878
8879 #[test]
8880 fn provenance_probe_override_keeps_handler_tests_deterministic() {
8881 let expected = subc_control::RunningImageAgreement::Unavailable {
8882 reason: subc_control::RunningImageUnavailableReason::HashFailed,
8883 };
8884 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
8885 assert_eq!(handler.provenance_probe_override, Some(expected));
8886 }
8887
8888 #[tokio::test]
8889 async fn malformed_control_bodies_return_invalid_control_body() {
8890 let handler = ControlHandler::default();
8891 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
8892
8893 for (corr, body) in [
8894 (56, br#"{"route_channel":1}"#.as_slice()),
8895 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
8896 (
8897 58,
8898 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
8899 ),
8900 ] {
8901 let request = Frame::build(
8902 FrameType::Request,
8903 control_flags(),
8904 0,
8905 0,
8906 corr,
8907 body.to_vec(),
8908 )
8909 .unwrap();
8910 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
8911
8912 assert_eq!(response.len(), 1);
8913 assert_eq!(response[0].header.ty, FrameType::Error);
8914 assert_eq!(response[0].header.corr, corr);
8915 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
8916 }
8917 }
8918
8919 #[tokio::test]
8920 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
8921 let registry = Arc::new(Registry::default());
8922 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
8923 let router = Router::with_control_handler(Arc::clone(&control));
8924 let connection = router.begin_connection();
8925 let (ctx, mut rx) = route_ctx(connection.id());
8926
8927 router
8928 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
8929 .await
8930 .unwrap();
8931 let response = rx.recv().await.unwrap();
8932 let ack = parse_ack(&response);
8933 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
8934 let channel = 1;
8935
8936 let goodbye =
8937 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
8938 router.route_for_connection(&ctx, goodbye).await.unwrap();
8939 assert!(rx.try_recv().is_err());
8940 assert!(registry.get_module("aft").unwrap().is_none());
8941
8942 router
8943 .route_for_connection(&ctx, channel_request(channel, 13))
8944 .await
8945 .unwrap();
8946 let error_frame = rx.recv().await.unwrap();
8947 assert_eq!(error_frame.header.ty, FrameType::Error);
8948 assert_eq!(error_frame.header.channel, channel);
8949 }
8950
8951 #[tokio::test]
8952 async fn dropping_router_connection_releases_registration() {
8953 let registry = Arc::new(Registry::default());
8954 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
8955 let router = Router::with_control_handler(control);
8956 let connection = router.begin_connection();
8957 let (ctx, mut rx) = route_ctx(connection.id());
8958
8959 router
8960 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
8961 .await
8962 .unwrap();
8963 let response = rx.recv().await.unwrap();
8964 let ack = parse_ack(&response);
8965 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
8966 assert!(registry.get_module("aft").unwrap().is_some());
8967
8968 drop(connection);
8969
8970 assert!(registry.get_module("aft").unwrap().is_none());
8971 assert_eq!(registry.active_registration_count().unwrap(), 0);
8972 }
8973
8974 fn capability_manifest(
8975 module_id: &str,
8976 provides: &[&str],
8977 must_never_reach: &[&str],
8978 ) -> ModuleManifest {
8979 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
8980 manifest.capabilities = Some(CapabilityDeclarations {
8981 provides: provides
8982 .iter()
8983 .map(|capability| (*capability).to_string())
8984 .collect(),
8985 requires: Vec::new(),
8986 must_never_reach: must_never_reach
8987 .iter()
8988 .map(|capability| (*capability).to_string())
8989 .collect(),
8990 });
8991 manifest
8992 }
8993
8994 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
8995 Frame::build(
8996 FrameType::Hello,
8997 control_flags(),
8998 0,
8999 0,
9000 corr,
9001 serde_json::to_vec(&ModuleHelloBody {
9002 protocol_ver: manifest.protocol_ver,
9003 manifest,
9004 control_ops: None,
9005 launch_nonce: None,
9006 })
9007 .expect("capability test HELLO serializes"),
9008 )
9009 .expect("capability test HELLO frame builds")
9010 }
9011
9012 fn catalog_update_with_capabilities_frame(
9013 corr: u64,
9014 capabilities: CapabilityDeclarations,
9015 ) -> Frame {
9016 Frame::build(
9017 FrameType::Request,
9018 control_flags(),
9019 0,
9020 0,
9021 corr,
9022 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
9023 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
9024 capabilities: Some(capabilities),
9025 ready: None,
9026 })
9027 .expect("capability catalog.update serializes"),
9028 )
9029 .expect("capability catalog.update frame builds")
9030 }
9031
9032 async fn register_capability_manifest(
9033 handler: &ControlHandler,
9034 ctx: &RouteCtx,
9035 manifest: ModuleManifest,
9036 corr: u64,
9037 ) {
9038 let replies = handler
9039 .handle_control_frame(ctx, hello_frame_with_manifest(manifest, corr))
9040 .await
9041 .expect("capability test HELLO succeeds");
9042 assert!(
9043 matches!(replies.as_slice(), [Frame { header, .. }] if header.ty == FrameType::HelloAck),
9044 "capability test HELLO must register"
9045 );
9046 }
9047
9048 async fn open_route_for_capability_test(
9049 handler: &ControlHandler,
9050 target_ctx: &RouteCtx,
9051 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9052 client_connection_id: u64,
9053 corr: u64,
9054 target_module_id: &str,
9055 consumer_identity: Option<ConsumerIdentity>,
9056 ) -> (
9057 mpsc::Receiver<crate::router::OutboundFrame>,
9058 ModuleControlRequest,
9059 ) {
9060 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
9061 let route_handler = handler.clone();
9062 let target_module_id = target_module_id.to_string();
9063 let route_task = tokio::spawn(async move {
9064 route_handler
9065 .handle_control_frame(
9066 &client_ctx,
9067 route_open_frame_with_admission_facts(
9068 corr,
9069 &target_module_id,
9070 unique_project_root("admission-facts"),
9071 consumer_identity,
9072 None,
9073 ),
9074 )
9075 .await
9076 .expect("capability test route.open succeeds")
9077 });
9078 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
9079 .await
9080 .expect("capability test route.open must reach route.bind")
9081 .expect("target control receiver stays open");
9082 let bind_request: ModuleControlRequest =
9083 serde_json::from_slice(&bind.body).expect("route.bind decodes");
9084 handler
9085 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
9086 .await
9087 .expect("capability test route.bind ACK succeeds");
9088 assert!(route_task.await.expect("route.open task joins").is_empty());
9089 let opened = client_rx
9090 .recv()
9091 .await
9092 .expect("successful route.open publishes a response");
9093 assert!(matches!(
9094 serde_json::from_slice::<ClientControlResponse>(&opened.body),
9095 Ok(ClientControlResponse::RouteOpen { .. })
9096 ));
9097 (client_rx, bind_request)
9098 }
9099
9100 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
9101 assert_eq!(frame.header.ty, FrameType::Push);
9102 assert_eq!(frame.header.channel, 0);
9103 assert_eq!(
9104 serde_json::from_slice::<ClientControlPush>(&frame.body)
9105 .expect("route.closed control push decodes"),
9106 ClientControlPush::RouteClosed {
9107 module_id: target_module_id.to_string(),
9108 reason: RouteCloseReason::CapabilityDenied,
9109 drained: false,
9110 abandoned: 0,
9111 excluded_subscriptions: 0,
9112 terminal: Some(false),
9113 }
9114 );
9115 }
9116
9117 #[tokio::test]
9118 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
9119 let registry = Arc::new(Registry::default());
9120 let forwarding = Arc::new(ForwardingTable::default());
9121 let supervisor = SupervisorHandle::new();
9122 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9123 let handler =
9124 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9125 .with_supervisor(supervisor);
9126 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
9127 let (opener_ctx, _opener_rx) = route_ctx(ConnectionId::new(701));
9128 register_capability_manifest(
9129 &handler,
9130 &target_ctx,
9131 capability_manifest("target", &["credentials-provider/v1"], &[]),
9132 1,
9133 )
9134 .await;
9135 register_capability_manifest(
9136 &handler,
9137 &opener_ctx,
9138 capability_manifest("opener", &[], &["credentials-provider/v1"]),
9139 2,
9140 )
9141 .await;
9142
9143 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
9144 let replies = handler
9145 .handle_control_frame(
9146 &client_ctx,
9147 route_open_frame_with_admission_facts(
9148 3,
9149 "target",
9150 unique_project_root("admission-facts"),
9151 Some(ConsumerIdentity {
9152 module_id: "opener".to_string(),
9153 launch_nonce: "opener-nonce".to_string(),
9154 }),
9155 None,
9156 ),
9157 )
9158 .await
9159 .expect("denied route.open returns a typed frame");
9160 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
9161 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
9162 assert!(
9163 target_rx.try_recv().is_err(),
9164 "forbidden route.open must not relay route.bind"
9165 );
9166 }
9167
9168 #[tokio::test]
9169 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
9170 let registry = Arc::new(Registry::default());
9171 let forwarding = Arc::new(ForwardingTable::default());
9172 let supervisor = SupervisorHandle::new();
9173 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9174 let handler =
9175 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9176 .with_supervisor(supervisor);
9177 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
9178 let (old_opener_ctx, _old_opener_rx) = route_ctx(ConnectionId::new(711));
9179 register_capability_manifest(
9180 &handler,
9181 &target_ctx,
9182 capability_manifest("target", &["credentials-provider/v1"], &[]),
9183 1,
9184 )
9185 .await;
9186 register_capability_manifest(
9187 &handler,
9188 &old_opener_ctx,
9189 capability_manifest("opener", &[], &[]),
9190 2,
9191 )
9192 .await;
9193 let (mut client_rx, _) = open_route_for_capability_test(
9194 &handler,
9195 &target_ctx,
9196 &mut target_rx,
9197 712,
9198 3,
9199 "target",
9200 Some(ConsumerIdentity {
9201 module_id: "opener".to_string(),
9202 launch_nonce: "opener-nonce".to_string(),
9203 }),
9204 )
9205 .await;
9206 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
9207
9208 handler
9209 .cleanup_connection(old_opener_ctx.connection_id)
9210 .expect("old opener registration cleans up");
9211 let (new_opener_ctx, _new_opener_rx) = route_ctx(ConnectionId::new(713));
9212 register_capability_manifest(
9213 &handler,
9214 &new_opener_ctx,
9215 capability_manifest("opener", &[], &["credentials-provider/v1"]),
9216 4,
9217 )
9218 .await;
9219
9220 assert_capability_denied_push(
9221 client_rx
9222 .try_recv()
9223 .expect("HELLO deny addition must emit route.closed")
9224 .frame,
9225 "target",
9226 );
9227 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
9228 assert!(matches!(
9229 target_rx.try_recv(),
9230 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
9231 ));
9232 }
9233
9234 #[tokio::test]
9235 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
9236 let registry = Arc::new(Registry::default());
9237 let forwarding = Arc::new(ForwardingTable::default());
9238 let supervisor = SupervisorHandle::new();
9239 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9240 let handler =
9241 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9242 .with_supervisor(supervisor);
9243 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
9244 let (opener_ctx, _opener_rx) = route_ctx(ConnectionId::new(721));
9245 register_capability_manifest(
9246 &handler,
9247 &target_ctx,
9248 capability_manifest("target", &[], &[]),
9249 1,
9250 )
9251 .await;
9252 register_capability_manifest(
9253 &handler,
9254 &opener_ctx,
9255 capability_manifest("opener", &[], &["credentials-provider/v1"]),
9256 2,
9257 )
9258 .await;
9259 let (mut client_rx, _) = open_route_for_capability_test(
9260 &handler,
9261 &target_ctx,
9262 &mut target_rx,
9263 722,
9264 3,
9265 "target",
9266 Some(ConsumerIdentity {
9267 module_id: "opener".to_string(),
9268 launch_nonce: "opener-nonce".to_string(),
9269 }),
9270 )
9271 .await;
9272 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
9273
9274 let replies = handler
9275 .handle_control_frame(
9276 &target_ctx,
9277 catalog_update_with_capabilities_frame(
9278 4,
9279 CapabilityDeclarations {
9280 provides: vec!["credentials-provider/v1".to_string()],
9281 requires: Vec::new(),
9282 must_never_reach: Vec::new(),
9283 },
9284 ),
9285 )
9286 .await
9287 .expect("claim catalog.update succeeds");
9288 assert!(matches!(
9289 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
9290 Ok(ModuleControlResponseToModule::CatalogUpdate {})
9291 ));
9292 assert_capability_denied_push(
9293 client_rx
9294 .try_recv()
9295 .expect("claim addition must emit route.closed")
9296 .frame,
9297 "target",
9298 );
9299 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
9300 assert!(matches!(
9301 target_rx.try_recv(),
9302 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
9303 ));
9304 }
9305
9306 #[tokio::test]
9307 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
9308 let registry = Arc::new(Registry::default());
9309 let forwarding = Arc::new(ForwardingTable::default());
9310 let supervisor = SupervisorHandle::new();
9311 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9312 let handler =
9313 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9314 .with_supervisor(supervisor);
9315 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
9316 let (opener_ctx, _opener_rx) = route_ctx(ConnectionId::new(731));
9317 register_capability_manifest(
9318 &handler,
9319 &target_ctx,
9320 capability_manifest("target", &["credentials-provider/v1"], &[]),
9321 1,
9322 )
9323 .await;
9324 register_capability_manifest(
9325 &handler,
9326 &opener_ctx,
9327 capability_manifest("opener", &[], &[]),
9328 2,
9329 )
9330 .await;
9331 let (mut client_rx, _) = open_route_for_capability_test(
9332 &handler,
9333 &target_ctx,
9334 &mut target_rx,
9335 732,
9336 3,
9337 "target",
9338 Some(ConsumerIdentity {
9339 module_id: "opener".to_string(),
9340 launch_nonce: "opener-nonce".to_string(),
9341 }),
9342 )
9343 .await;
9344 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
9345
9346 handler
9347 .handle_control_frame(
9348 &target_ctx,
9349 catalog_update_with_capabilities_frame(
9350 4,
9351 CapabilityDeclarations {
9352 provides: Vec::new(),
9353 requires: Vec::new(),
9354 must_never_reach: Vec::new(),
9355 },
9356 ),
9357 )
9358 .await
9359 .expect("claim removal catalog.update succeeds");
9360 assert_eq!(
9361 forwarding.active_binding_count().unwrap(),
9362 1,
9363 "removing an attested target claim must leave the route census unchanged"
9364 );
9365 assert!(
9366 client_rx.try_recv().is_err(),
9367 "claim removal must not emit route.closed capability_denied"
9368 );
9369 assert!(
9370 target_rx.try_recv().is_err(),
9371 "claim removal must not send the target a route GOODBYE"
9372 );
9373 }
9374
9375 #[tokio::test]
9378 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
9379 let registry = Arc::new(Registry::default());
9380 let forwarding = Arc::new(ForwardingTable::default());
9381 let supervisor = SupervisorHandle::new();
9382 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9383 let handler =
9384 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9385 .with_supervisor(supervisor);
9386 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
9387 let (opener_ctx, _opener_rx) = route_ctx(ConnectionId::new(741));
9388 register_capability_manifest(
9389 &handler,
9390 &target_ctx,
9391 capability_manifest("target", &["credentials-provider/v1"], &[]),
9392 1,
9393 )
9394 .await;
9395 register_capability_manifest(
9396 &handler,
9397 &opener_ctx,
9398 capability_manifest("opener", &[], &["credentials-provider/v1"]),
9399 2,
9400 )
9401 .await;
9402
9403 let (_client_rx, bind) = open_route_for_capability_test(
9404 &handler,
9405 &target_ctx,
9406 &mut target_rx,
9407 742,
9408 3,
9409 "target",
9410 None,
9411 )
9412 .await;
9413 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
9414 panic!("direct scope-honesty route must bind");
9415 };
9416 assert_eq!(principal, Some(Principal::Direct));
9417 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
9418 }
9419
9420 #[tokio::test]
9423 async fn must_never_reach_self_route_is_capability_forbidden() {
9424 let registry = Arc::new(Registry::default());
9425 let forwarding = Arc::new(ForwardingTable::default());
9426 let supervisor = SupervisorHandle::new();
9427 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
9428 let handler =
9429 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9430 .with_supervisor(supervisor);
9431 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
9432 register_capability_manifest(
9433 &handler,
9434 &self_ctx,
9435 capability_manifest(
9436 "self-provider",
9437 &["credentials-provider/v1"],
9438 &["credentials-provider/v1"],
9439 ),
9440 1,
9441 )
9442 .await;
9443
9444 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
9445 let replies = handler
9446 .handle_control_frame(
9447 &client_ctx,
9448 route_open_frame_with_admission_facts(
9449 2,
9450 "self-provider",
9451 unique_project_root("admission-facts"),
9452 Some(ConsumerIdentity {
9453 module_id: "self-provider".to_string(),
9454 launch_nonce: "self-nonce".to_string(),
9455 }),
9456 None,
9457 ),
9458 )
9459 .await
9460 .expect("self-route refusal returns a typed frame");
9461 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
9462 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
9463 assert!(
9464 self_rx.try_recv().is_err(),
9465 "self denial must not relay route.bind"
9466 );
9467 }
9468
9469 #[test]
9470 fn unsupported_channel_zero_frame_returns_error() {
9471 let handler = ControlHandler::default();
9472 let request = Frame::build(
9473 FrameType::Request,
9474 control_flags(),
9475 0,
9476 0,
9477 21,
9478 b"opaque".to_vec(),
9479 )
9480 .unwrap();
9481
9482 let response = handler
9483 .handle_control(ConnectionId::new(1), request)
9484 .unwrap();
9485
9486 assert_eq!(response[0].header.ty, FrameType::Error);
9487 assert_eq!(
9488 parse_error(&response[0])["code"],
9489 "unsupported_control_frame"
9490 );
9491 }
9492
9493 mod swap {
9498 use super::*;
9499
9500 const INCUMBENT: ConnectionId = ConnectionId::new(30);
9501 const CANDIDATE: ConnectionId = ConnectionId::new(40);
9502
9503 struct Swap {
9504 registry: Arc<Registry>,
9505 forwarding: Arc<ForwardingTable>,
9506 handler: ControlHandler,
9507 incumbent_ctx: RouteCtx,
9508 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
9509 candidate_ctx: RouteCtx,
9510 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
9511 }
9512
9513 async fn swap_with_incumbent() -> Swap {
9514 let registry = Arc::new(Registry::default());
9515 let forwarding = Arc::new(ForwardingTable::default());
9516 let handler =
9517 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9518 let (incumbent_ctx, incumbent_rx) = route_ctx(INCUMBENT);
9519 handler
9520 .handle_control_frame(&incumbent_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
9521 .await
9522 .unwrap();
9523 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
9524 Swap {
9525 registry,
9526 forwarding,
9527 handler,
9528 incumbent_ctx,
9529 incumbent_rx,
9530 candidate_ctx,
9531 candidate_rx,
9532 }
9533 }
9534
9535 fn register_candidate(swap: &Swap, ready: Option<bool>) {
9536 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
9537 candidate_manifest.ready = ready;
9538 let registration = swap
9539 .registry
9540 .register_candidate_with_control_ops(
9541 candidate_manifest,
9542 PROTOCOL_VERSION,
9543 CANDIDATE,
9544 module_baseline_control_ops(),
9545 )
9546 .unwrap();
9547 swap.forwarding
9548 .register_candidate_module_connection(
9549 CANDIDATE,
9550 "aft".to_string(),
9551 PROTOCOL_VERSION,
9552 manifest_concurrency(®istration.manifest),
9553 swap.candidate_ctx.egress.clone(),
9554 )
9555 .unwrap();
9556 }
9557
9558 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
9559 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
9560 swap.registry.promote_candidate("aft").unwrap().unwrap();
9561 cutover.incumbent.unwrap()
9562 }
9563
9564 fn keyed_total(counters: &Value, key: &str) -> u64 {
9565 counters[key]
9566 .as_object()
9567 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
9568 .unwrap_or(0)
9569 }
9570
9571 #[tokio::test]
9577 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
9578 let mut swap = swap_with_incumbent().await;
9579 let handler = swap.handler.clone();
9580
9581 let cotenant = ConnectionId::new(31);
9583 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
9584 let (cotenant_task, cotenant_bind) = relay_route_open(
9585 &handler,
9586 cotenant,
9587 &cotenant_ctx.egress,
9588 &mut swap.incumbent_rx,
9589 100,
9590 "aft",
9591 "swap-cotenant",
9592 )
9593 .await;
9594 handler
9595 .handle_control_frame(
9596 &swap.incumbent_ctx,
9597 route_bind_ack(cotenant_bind.header.corr),
9598 )
9599 .await
9600 .unwrap();
9601 assert!(cotenant_task.await.unwrap().is_empty());
9602 let (cotenant_channel, cotenant_epoch) =
9603 published_route(&cotenant_rx.recv().await.unwrap());
9604
9605 let caller = ConnectionId::new(32);
9607 let (caller_ctx, mut caller_rx) = route_ctx(caller);
9608 let (caller_task, caller_bind) = relay_route_open(
9609 &handler,
9610 caller,
9611 &caller_ctx.egress,
9612 &mut swap.incumbent_rx,
9613 101,
9614 "aft",
9615 "swap-caller",
9616 )
9617 .await;
9618 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
9619
9620 register_candidate(&swap, None);
9621 cutover(&swap);
9622
9623 let ack = handler
9625 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
9626 .await;
9627 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
9628 if module_loop_error.is_some() {
9629 handler.cleanup_connection(INCUMBENT).unwrap();
9632 }
9633
9634 assert!(
9636 cotenant_rx.try_recv().is_err(),
9637 "the co-tenant route on the incumbent was torn down by one late ack: \
9638 {module_loop_error:?}"
9639 );
9640 assert!(matches!(
9641 swap.forwarding
9642 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
9643 .unwrap(),
9644 DataRoute::Client(DataRouteState::Bound(_))
9645 ));
9646 assert_eq!(module_loop_error, None);
9647 assert!(swap
9648 .registry
9649 .get_module_by_connection(INCUMBENT)
9650 .unwrap()
9651 .is_some());
9652
9653 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
9655 .await
9656 .expect("the incumbent is told to drop the abandoned binding")
9657 .unwrap()
9658 .frame;
9659 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
9660 assert_eq!(goodbye.header.channel, abandoned_channel);
9661 assert_eq!(goodbye.header.epoch, abandoned_epoch);
9662 assert!(swap.incumbent_rx.try_recv().is_err());
9663
9664 let response = caller_task.await.unwrap();
9666 assert_eq!(response.len(), 1);
9667 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
9668 assert!(caller_rx.try_recv().is_err());
9669
9670 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
9674 let counters = handler.counters().snapshot();
9675 assert_eq!(
9676 keyed_total(&counters, "route_open_accepted_by_principal"),
9677 1
9678 );
9679 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
9680 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
9681 }
9682
9683 #[tokio::test]
9687 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
9688 let mut swap = swap_with_incumbent().await;
9689 register_candidate(&swap, None);
9690 let incumbent = cutover(&swap);
9691 swap.forwarding
9692 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
9693 .unwrap()
9694 .expect("the incumbent is still registered");
9695
9696 let client = ConnectionId::new(33);
9697 let (client_ctx, mut client_rx) = route_ctx(client);
9698 let route_handler = swap.handler.clone();
9699 let open_ctx = RouteCtx {
9700 connection_id: client,
9701 egress: client_ctx.egress.clone(),
9702 };
9703 let mut route_task = tokio::spawn(async move {
9704 route_handler
9705 .handle_control_frame(
9706 &open_ctx,
9707 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
9708 )
9709 .await
9710 .unwrap()
9711 });
9712 let bind = tokio::select! {
9713 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
9714 response = &mut route_task => {
9715 let response = response.unwrap();
9716 panic!(
9717 "post-cutover route.open was refused instead of relayed to the candidate: {}",
9718 parse_error(&response[0])["code"]
9719 );
9720 }
9721 };
9722 swap.handler
9723 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
9724 .await
9725 .unwrap();
9726 assert!(route_task.await.unwrap().is_empty());
9727 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
9728 match swap
9729 .forwarding
9730 .lookup_data_route(client, channel, epoch)
9731 .unwrap()
9732 {
9733 DataRoute::Client(DataRouteState::Bound(route)) => {
9734 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
9735 }
9736 other => panic!("expected a bound route on the candidate, got {other:?}"),
9737 }
9738 assert!(swap.incumbent_rx.try_recv().is_err());
9739 }
9740
9741 #[tokio::test]
9746 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
9747 let swap = swap_with_incumbent().await;
9748 register_candidate(&swap, Some(false));
9749 let update = Frame::build(
9750 FrameType::Request,
9751 control_flags(),
9752 0,
9753 0,
9754 55,
9755 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
9756 provides: manifest("aft", PROTOCOL_VERSION).provides,
9757 capabilities: None,
9758 ready: Some(true),
9759 })
9760 .unwrap(),
9761 )
9762 .unwrap();
9763
9764 let replies = swap
9765 .handler
9766 .handle_control_frame(&swap.candidate_ctx, update)
9767 .await
9768 .unwrap();
9769
9770 assert_eq!(replies.len(), 1);
9771 assert_eq!(
9772 replies[0].header.ty,
9773 FrameType::Response,
9774 "candidate catalog.update was refused: {:?}",
9775 serde_json::from_slice::<Value>(&replies[0].body).ok()
9776 );
9777 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
9778 assert_eq!(
9779 swap.registry
9780 .get_module("aft")
9781 .unwrap()
9782 .unwrap()
9783 .connection_id,
9784 INCUMBENT
9785 );
9786 }
9787 }
9788
9789 mod swap_admission {
9793 use super::*;
9794
9795 const INCUMBENT_NONCE: &str = "incumbent-nonce";
9796 const CANDIDATE_NONCE: &str = "candidate-nonce";
9797
9798 fn handler_with_incumbent(
9799 module_id: &str,
9800 reserved: bool,
9801 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
9802 let registry = Arc::new(Registry::default());
9803 let supervisor = SupervisorHandle::new();
9804 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
9805 if reserved {
9806 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
9807 }
9808 let handler =
9809 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
9810 let incumbent = handler
9811 .handle_control(
9812 ConnectionId::new(1),
9813 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
9814 )
9815 .unwrap();
9816 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
9817 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
9818 (registry, supervisor, handler)
9819 }
9820
9821 #[test]
9827 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
9828 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
9829
9830 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
9831 let replies = handler
9832 .handle_control(
9833 ConnectionId::new(connection),
9834 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
9835 )
9836 .unwrap();
9837 assert_eq!(replies[0].header.ty, FrameType::Error);
9838 assert_eq!(
9839 parse_error(&replies[0])["code"],
9840 "swap_token_invalid",
9841 "nonce {nonce:?}"
9842 );
9843 }
9844 assert!(registry.get_candidate("aft").unwrap().is_none());
9845 assert_eq!(
9846 registry.get_module("aft").unwrap().unwrap().connection_id,
9847 ConnectionId::new(1)
9848 );
9849
9850 let admitted = handler
9853 .handle_control(
9854 ConnectionId::new(4),
9855 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
9856 )
9857 .unwrap();
9858 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
9859 assert_eq!(
9860 registry
9861 .get_candidate("aft")
9862 .unwrap()
9863 .unwrap()
9864 .connection_id,
9865 ConnectionId::new(4)
9866 );
9867 assert_eq!(
9868 registry.get_module("aft").unwrap().unwrap().connection_id,
9869 ConnectionId::new(1),
9870 "the candidate must not take the active slot"
9871 );
9872 let replayed = handler
9873 .handle_control(
9874 ConnectionId::new(5),
9875 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
9876 )
9877 .unwrap();
9878 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
9879
9880 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
9884 let squatter = handler
9885 .handle_control(
9886 ConnectionId::new(6),
9887 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
9888 )
9889 .unwrap();
9890 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
9891 assert!(
9892 registry.get_module("aft").unwrap().is_none(),
9893 "a squatter took the active slot of an id being swapped"
9894 );
9895 }
9896
9897 #[test]
9902 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
9903 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
9904
9905 let replies = handler
9906 .handle_control(
9907 ConnectionId::new(2),
9908 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
9909 )
9910 .unwrap();
9911
9912 assert_eq!(
9913 replies[0].header.ty,
9914 FrameType::HelloAck,
9915 "reserved candidate refused: {:?}",
9916 serde_json::from_slice::<Value>(&replies[0].body).ok()
9917 );
9918 assert_eq!(
9919 registry
9920 .get_candidate("vault")
9921 .unwrap()
9922 .unwrap()
9923 .connection_id,
9924 ConnectionId::new(2)
9925 );
9926 }
9927
9928 #[test]
9931 fn without_an_open_swap_the_ordinary_gates_decide() {
9932 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
9933 supervisor.close_swap("vault");
9934
9935 let candidate = handler
9936 .handle_control(
9937 ConnectionId::new(2),
9938 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
9939 )
9940 .unwrap();
9941 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
9942 let duplicate = handler
9943 .handle_control(
9944 ConnectionId::new(3),
9945 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
9946 )
9947 .unwrap();
9948 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
9949 assert!(registry.get_candidate("vault").unwrap().is_none());
9950 }
9951 }
9952}
9953
9954#[cfg(test)]
9955mod concurrency_default_exposure_tests {
9956 use super::*;
9957
9958 fn hello_body(role_json: &str) -> Vec<u8> {
9959 format!(
9960 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
9961 )
9962 .into_bytes()
9963 }
9964
9965 fn manifest_from(body: &[u8]) -> ModuleManifest {
9966 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
9967 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
9968 .expect("manifest parses")
9969 }
9970
9971 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
9972
9973 #[test]
9974 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
9975 let body = hello_body(&format!(
9976 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
9977 ));
9978 let manifest = manifest_from(&body);
9979 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
9982 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
9983 }
9984
9985 #[test]
9986 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
9987 let body = hello_body(&format!(
9988 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
9989 ));
9990 let manifest = manifest_from(&body);
9991 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
9992 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
9993 }
9994
9995 #[test]
9996 fn non_management_roles_are_never_reported() {
9997 let body = hello_body(
9998 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
9999 );
10000 let manifest = manifest_from(&body);
10001 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
10002 }
10003}