1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 path::PathBuf,
5 sync::{Arc, Mutex},
6 time::{Duration, Instant as StdInstant},
7};
8
9use serde::{Deserialize, Serialize};
10use subc_control::{
11 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
12 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
13 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PollKind, RouteCloseReason,
14 SpawnCursor, StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance,
15 SupervisorEntry, SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
16 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
17};
18use subc_protocol::{
19 error_codes,
20 manifest::{
21 validate_hello_capability_grammar, validate_hello_self_signal_declarations,
22 CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
23 ProviderRole,
24 },
25 session::{
26 HealthReport, ModuleControlPush, ModuleControlRequest, ModuleControlRequestFromModule,
27 ModuleControlResponse, ModuleControlResponseToModule, MODULE_CONTROL_OP_HEALTH_CHECK,
28 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
29 },
30 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
31 Priority, RouteTarget, PROTOCOL_VERSION,
32};
33use tokio::time::{timeout_at, Instant};
34use tracing::{debug, info, warn};
35
36use crate::{
37 capability_requirements::{
38 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
39 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
40 RuntimeModule,
41 },
42 daemon_config::RestartRequiredSection,
43 forwarding::{
44 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
45 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
46 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
47 },
48 observability::{
49 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
50 },
51 provenance::{
52 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
53 },
54 registry::{ChannelState, ConnectionId, Registry, RegistryError},
55 router::{RouteCtx, RouterError},
56 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
57 stderr_tail::{CaptureState, TailEntry},
58 supervise::{
59 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
60 SupervisorHandle, SwapHelloAdmission,
61 },
62 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
63};
64
65pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
71
72const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
73const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
74const CAP_PING_PONG: &str = "ping_pong_v1";
75const CAP_SESSION_ATTACH: &str = "session_attach_v1";
76const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
77
78const SUBC_CONTROL_OPS: &[&str] = &[
79 ops::SERVER_DESCRIBE,
80 ops::CATALOG_LIST,
81 ops::ROUTE_OPEN,
82 ops::ROUTE_POLL,
83 ops::ROUTE_CLOSING,
84 ops::ROUTE_CLOSED,
85 ops::SUPERVISOR_LIST,
86 ops::SUPERVISOR_RESTART,
87 ops::SUPERVISOR_SWAP,
88 ops::SUPERVISOR_RELOAD,
89 ops::SUPERVISOR_RESCAN,
90 ops::SUPERVISOR_RELEASE_RESERVED,
91 ops::SUPERVISOR_SET_ENABLED,
92 ops::SUPERVISOR_HEALTH_PROBE,
93 ops::SUPERVISOR_HEALTH,
94 ops::SUPERVISOR_STDERR_TAIL,
95 ops::SUPERVISOR_TERMINALS,
96 ops::SUPERVISOR_ROUTES,
97 ops::SUPERVISOR_PROVENANCE,
98 ops::SUPERVISOR_SPAWN_SNAPSHOT,
99 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
100];
101
102const MODULE_TO_SUBC_CONTROL_OPS: &[&str] =
103 &[MODULE_TO_SUBC_OP_CATALOG_UPDATE, "supervisor.live_roots"];
104
105const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
106
107pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
115
116pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
131
132pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
150
151const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
152const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
153
154#[derive(Clone)]
155struct DaemonProvenanceFacts {
156 build: DaemonBuildProvenance,
157 pid: Option<u32>,
158 started_at_ms: Option<u64>,
159 start_clock: Option<crate::clock::StartClock>,
160 executable_path: Option<PathBuf>,
161 executable_identity: Option<SpawnedFileIdentity>,
162 process_start_time: Option<u64>,
163 probe: ExecutableIdentityProbe,
164}
165
166impl Default for DaemonProvenanceFacts {
167 fn default() -> Self {
168 Self {
169 build: DaemonBuildProvenance {
170 build_git_sha: None,
171 build_lock_digest: None,
172 },
173 pid: None,
174 started_at_ms: None,
175 start_clock: None,
176 executable_path: None,
177 executable_identity: None,
178 process_start_time: None,
179 probe: ExecutableIdentityProbe::default(),
180 }
181 }
182}
183
184#[derive(Debug, Clone)]
185struct SupervisorRescanContext {
186 supervisor: Supervisor,
187 config_path: PathBuf,
188 configured_port: Option<u16>,
189 storage_config: Option<crate::daemon_config::StorageConfig>,
190 admission_facts_carrier_module_id: Option<String>,
191 admission_facts_targets: Option<Vec<String>>,
192}
193
194#[derive(Clone)]
196pub struct ControlHandler {
197 registry: Arc<Registry>,
198 forwarding: Arc<ForwardingTable>,
199 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
200 supervisor: SupervisorHandle,
201 subc_capabilities: Arc<[String]>,
202 route_bind_relay_timeout: Duration,
206 route_bind_relay_timeouts: BTreeMap<String, Duration>,
210 route_bind_breakers: RouteBindBreakers,
213 route_bind_concurrency: RouteBindConcurrency,
216 route_bind_breaker_threshold: u32,
218 route_bind_breaker_cooldown: Duration,
220 health_probe_timeout: Duration,
221 storage_config: Option<crate::daemon_config::StorageConfig>,
224 machine_id: Option<crate::machine_id::MachineId>,
228 admission_facts_carrier_module_id: Option<String>,
229 admission_facts_targets: Option<Vec<String>>,
230 rescan: Option<SupervisorRescanContext>,
231 connected_clients: ConnectedClients,
232 counters: DaemonCounters,
233 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
234 daemon_provenance: DaemonProvenanceFacts,
235 #[cfg(test)]
236 control_dispatch_delay: Option<Duration>,
237 #[cfg(test)]
238 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
239}
240
241impl fmt::Debug for ControlHandler {
242 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
243 f.debug_struct("ControlHandler")
244 .field("registry", &self.registry)
245 .field("forwarding", &self.forwarding)
246 .field("process_liveness", &self.process_liveness.is_some())
247 .field("supervisor", &self.supervisor)
248 .field("subc_capabilities", &self.subc_capabilities)
249 .finish()
250 }
251}
252
253struct RouteOpenRequest {
254 target: RouteTarget,
255 identity: BindIdentity,
256 consumer_identity: Option<ConsumerIdentity>,
257 consumer_capabilities: Option<Vec<String>>,
258 admission_facts: Option<serde_json::Value>,
259}
260
261struct RouteBindReservationGuard {
262 forwarding: Arc<ForwardingTable>,
263 endpoint: ModuleEndpointId,
264 relay_corr: u64,
265 armed: bool,
266}
267
268struct ModuleControlRpcGuard {
269 forwarding: Arc<ForwardingTable>,
270 endpoint: ModuleEndpointId,
271 corr: u64,
272 armed: bool,
273}
274
275impl ModuleControlRpcGuard {
276 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
277 Self {
278 forwarding,
279 endpoint,
280 corr,
281 armed: true,
282 }
283 }
284
285 fn disarm(&mut self) {
286 self.armed = false;
287 }
288}
289
290impl Drop for ModuleControlRpcGuard {
291 fn drop(&mut self) {
292 if self.armed {
293 let _ = self
294 .forwarding
295 .cancel_module_control_rpc(self.endpoint, self.corr);
296 }
297 }
298}
299
300impl RouteBindReservationGuard {
301 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
302 Self {
303 forwarding,
304 endpoint,
305 relay_corr,
306 armed: true,
307 }
308 }
309
310 fn release_and_disarm(&mut self) {
311 if !self.armed {
312 return;
313 }
314 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
315 self.endpoint,
316 self.relay_corr,
317 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
318 ) {
319 send_goodbye_target_best_effort(
320 &self.forwarding.counters(),
321 &target,
322 "canceled route.bind",
323 );
324 }
325 self.armed = false;
326 }
327
328 fn disarm(&mut self) {
329 self.armed = false;
330 }
331}
332
333impl Drop for RouteBindReservationGuard {
334 fn drop(&mut self) {
335 self.release_and_disarm();
336 }
337}
338
339#[derive(Debug, Clone, Default)]
369pub(crate) struct RouteBindBreakers {
370 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
371}
372
373#[derive(Debug, Clone, Default)]
374pub(crate) struct RouteBindConcurrency {
375 modules: Arc<Mutex<HashMap<String, usize>>>,
376}
377
378struct RouteBindConcurrencyGuard {
379 concurrency: RouteBindConcurrency,
380 module_id: String,
381}
382
383impl RouteBindConcurrency {
384 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
387 let mut modules = self
388 .modules
389 .lock()
390 .expect("route.bind concurrency mutex poisoned");
391 let in_flight = modules.entry(module_id.to_string()).or_default();
392 if *in_flight >= limit {
393 return Err(*in_flight);
394 }
395 *in_flight += 1;
396 Ok(RouteBindConcurrencyGuard {
397 concurrency: self.clone(),
398 module_id: module_id.to_string(),
399 })
400 }
401}
402
403impl Drop for RouteBindConcurrencyGuard {
404 fn drop(&mut self) {
405 let mut modules = self
406 .concurrency
407 .modules
408 .lock()
409 .expect("route.bind concurrency mutex poisoned");
410 let remove = {
411 let in_flight = modules
412 .get_mut(&self.module_id)
413 .expect("admitted route.bind has a concurrency entry");
414 *in_flight -= 1;
415 *in_flight == 0
416 };
417 if remove {
418 modules.remove(&self.module_id);
419 }
420 }
421}
422
423#[derive(Debug, Default)]
424struct ModuleBreakerState {
425 consecutive_timeouts: u32,
427 cooldown_until: Option<Instant>,
430 probe_in_flight: bool,
435}
436
437enum RouteBindAdmission<'a> {
439 Admitted {
440 guard: RouteBindBreakerGuard<'a>,
441 probe: bool,
444 },
445 Refused {
446 consecutive_timeouts: u32,
447 retry_in: Duration,
451 probe_in_flight: bool,
452 },
453}
454
455struct RouteBindBreakerGuard<'a> {
462 breakers: RouteBindBreakers,
463 module_id: &'a str,
464 settled: bool,
465}
466
467impl RouteBindBreakerGuard<'_> {
468 fn record_accepted(&mut self) -> bool {
472 self.settled = true;
473 self.breakers.record_accepted(self.module_id)
474 }
475
476 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
479 self.settled = true;
480 self.breakers
481 .record_timeout(self.module_id, threshold, cooldown)
482 }
483
484 fn record_inconclusive(&mut self) {
493 self.settled = true;
494 self.breakers.record_inconclusive(self.module_id);
495 }
496}
497
498impl Drop for RouteBindBreakerGuard<'_> {
499 fn drop(&mut self) {
500 if !self.settled {
501 self.breakers.record_inconclusive(self.module_id);
502 }
503 }
504}
505
506struct BreakerOpened {
510 consecutive_timeouts: u32,
511 reopened_after_probe: bool,
514}
515
516impl RouteBindBreakers {
517 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
518 self.modules
519 .lock()
520 .expect("route.bind breaker mutex poisoned")
521 }
522
523 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
526 let admitted = |probe| RouteBindAdmission::Admitted {
527 guard: RouteBindBreakerGuard {
528 breakers: self.clone(),
529 module_id,
530 settled: false,
531 },
532 probe,
533 };
534
535 let mut modules = self.lock();
536 let Some(state) = modules.get_mut(module_id) else {
537 return admitted(false);
538 };
539 let Some(cooldown_until) = state.cooldown_until else {
540 return admitted(false);
541 };
542 if state.probe_in_flight {
543 return RouteBindAdmission::Refused {
544 consecutive_timeouts: state.consecutive_timeouts,
545 retry_in: Duration::ZERO,
546 probe_in_flight: true,
547 };
548 }
549 let now = Instant::now();
550 if now < cooldown_until {
551 return RouteBindAdmission::Refused {
552 consecutive_timeouts: state.consecutive_timeouts,
553 retry_in: cooldown_until - now,
554 probe_in_flight: false,
555 };
556 }
557 state.probe_in_flight = true;
558 admitted(true)
559 }
560
561 fn record_accepted(&self, module_id: &str) -> bool {
562 self.lock()
563 .remove(module_id)
564 .is_some_and(|state| state.cooldown_until.is_some())
565 }
566
567 fn record_timeout(
568 &self,
569 module_id: &str,
570 threshold: u32,
571 cooldown: Duration,
572 ) -> Option<BreakerOpened> {
573 let mut modules = self.lock();
574 let state = modules.entry(module_id.to_string()).or_default();
575 let was_open = state.cooldown_until.is_some();
576 let was_probe = state.probe_in_flight;
577 state.probe_in_flight = false;
578 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
579 if state.consecutive_timeouts < threshold {
580 return None;
581 }
582 state.cooldown_until = Some(Instant::now() + cooldown);
583 Some(BreakerOpened {
584 consecutive_timeouts: state.consecutive_timeouts,
585 reopened_after_probe: was_open && was_probe,
586 })
587 }
588
589 fn record_inconclusive(&self, module_id: &str) {
590 if let Some(state) = self.lock().get_mut(module_id) {
591 state.probe_in_flight = false;
592 }
593 }
594
595 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
610 self.lock()
611 .remove(module_id)
612 .map(|state| state.consecutive_timeouts)
613 .filter(|discarded| *discarded > 0)
614 }
615
616 fn open_snapshot(&self) -> Option<serde_json::Value> {
623 let now = Instant::now();
624 let modules = self.lock();
625 let open = modules
626 .iter()
627 .filter_map(|(module_id, state)| {
628 let cooldown_until = state.cooldown_until?;
629 Some((
630 module_id.clone(),
631 serde_json::json!({
632 "consecutive_timeouts": state.consecutive_timeouts,
633 "cooldown_remaining_ms":
634 cooldown_until.saturating_duration_since(now).as_millis() as u64,
635 "probe_in_flight": state.probe_in_flight,
636 }),
637 ))
638 })
639 .collect::<serde_json::Map<String, serde_json::Value>>();
640 (!open.is_empty()).then_some(serde_json::Value::Object(open))
641 }
642}
643
644impl ControlHandler {
645 pub fn new(registry: Arc<Registry>) -> Self {
646 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
647 }
648
649 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
650 let counters = forwarding.counters();
651 let route_bind_breakers = forwarding.route_bind_breakers();
655 let route_bind_concurrency = forwarding.route_bind_concurrency();
656 Self {
657 registry,
658 forwarding,
659 process_liveness: None,
660 supervisor: SupervisorHandle::new(),
661 subc_capabilities: Arc::from([
662 CAP_MANIFEST_REGISTRATION.to_string(),
663 CAP_CHANNEL_LIFECYCLE.to_string(),
664 CAP_PING_PONG.to_string(),
665 CAP_SESSION_ATTACH.to_string(),
666 CAP_ADMISSION_FACTS_RELAY.to_string(),
667 ]),
668 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
669 route_bind_relay_timeouts: BTreeMap::new(),
670 route_bind_breakers,
671 route_bind_concurrency,
672 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
673 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
674 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
675 storage_config: None,
676 machine_id: None,
677 admission_facts_carrier_module_id: None,
678 admission_facts_targets: None,
679 rescan: None,
680 connected_clients: ConnectedClients::new(),
681 counters,
682 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
683 daemon_provenance: DaemonProvenanceFacts::default(),
684 #[cfg(test)]
685 control_dispatch_delay: None,
686 #[cfg(test)]
687 provenance_probe_override: None,
688 }
689 }
690
691 pub fn with_storage_config(
694 mut self,
695 storage_config: Option<crate::daemon_config::StorageConfig>,
696 ) -> Self {
697 self.storage_config = storage_config;
698 self
699 }
700
701 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
704 self.machine_id = machine_id;
705 self
706 }
707
708 pub fn with_admission_facts_config(
712 mut self,
713 carrier_module_id: Option<String>,
714 targets: Option<Vec<String>>,
715 ) -> Self {
716 self.admission_facts_carrier_module_id = carrier_module_id;
717 self.admission_facts_targets = targets;
718 self
719 }
720
721 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
724 self.route_bind_relay_timeout = timeout;
725 self
726 }
727
728 pub fn with_route_bind_relay_timeouts(
734 mut self,
735 timeouts: impl IntoIterator<Item = (String, Duration)>,
736 ) -> Self {
737 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
738 self
739 }
740
741 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
747 self.route_bind_relay_timeouts
748 .get(module_id)
749 .copied()
750 .unwrap_or(self.route_bind_relay_timeout)
751 }
752
753 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
761 self.route_bind_breaker_threshold = threshold.max(1);
762 self.route_bind_breaker_cooldown = cooldown;
763 self
764 }
765
766 #[cfg(test)]
767 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
768 self.health_probe_timeout = timeout;
769 self
770 }
771
772 #[cfg(test)]
773 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
774 self.control_dispatch_delay = Some(delay);
775 self
776 }
777
778 pub fn with_process_liveness(
779 mut self,
780 process_liveness: Arc<dyn ModuleProcessLiveness>,
781 ) -> Self {
782 self.process_liveness = Some(process_liveness);
783 self
784 }
785
786 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
787 self.supervisor = supervisor;
788 self
789 }
790
791 pub fn with_daemon_provenance(
792 mut self,
793 pid: u32,
794 started_at_ms: u64,
795 executable_path: Option<PathBuf>,
796 build_git_sha: Option<String>,
797 build_lock_digest: Option<String>,
798 ) -> Self {
799 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
800 let process_start_time = process_start_time(pid);
801 self.daemon_provenance = DaemonProvenanceFacts {
802 build: DaemonBuildProvenance {
803 build_git_sha,
804 build_lock_digest,
805 },
806 pid: Some(pid),
807 started_at_ms: Some(started_at_ms),
808 start_clock: None,
809 executable_path,
810 executable_identity,
811 process_start_time,
812 probe: ExecutableIdentityProbe::default(),
813 };
814 self
815 }
816
817 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
818 self.daemon_provenance.start_clock = Some(clock);
819 self
820 }
821
822 #[cfg(test)]
823 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
824 self.provenance_probe_override = Some(result);
825 self
826 }
827
828 pub fn with_capability_config(
832 self,
833 modules: impl IntoIterator<Item = (String, bool)>,
834 reserved_capabilities: BTreeMap<String, String>,
835 ) -> Self {
836 self.capability_evaluator
837 .configure(modules, reserved_capabilities);
838 self
839 }
840
841 pub fn with_supervisor_rescan(
842 mut self,
843 supervisor: Supervisor,
844 config_path: impl Into<PathBuf>,
845 configured_port: Option<u16>,
846 ) -> Self {
847 self.rescan = Some(SupervisorRescanContext {
848 supervisor,
849 config_path: config_path.into(),
850 configured_port,
851 storage_config: self.storage_config.clone(),
852 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
853 admission_facts_targets: self.admission_facts_targets.clone(),
854 });
855 self
856 }
857
858 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
859 self.connected_clients = connected_clients;
860 self
861 }
862
863 pub fn forwarding(&self) -> Arc<ForwardingTable> {
864 Arc::clone(&self.forwarding)
865 }
866
867 pub(crate) fn counters(&self) -> DaemonCounters {
868 self.counters.clone()
869 }
870
871 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
874 tokio::spawn(async move {
875 loop {
876 self.capability_evaluator
877 .wait_for_change_or_deadline()
878 .await;
879 self.refresh_capability_requirements();
880 }
881 });
882 }
883
884 fn runtime_capability_snapshot(
885 &self,
886 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
887 let runtime = self
888 .supervisor
889 .list()
890 .into_iter()
891 .map(|module| {
892 let status = module.status().map_err(|err| {
893 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
894 })?;
895 Ok(RuntimeModule {
896 module_id: status.module_id,
897 state: status.state,
898 enabled: status.enabled,
899 })
900 })
901 .collect::<Result<Vec<_>, RouterError>>()?;
902 let (_, registrations) = self.registry.list_modules().map_err(|err| {
903 RouterError::backend(
904 0,
905 0,
906 format!("failed to list capability registrations: {err}"),
907 )
908 })?;
909 let registrations = registrations
910 .into_iter()
911 .map(|registration| RegisteredModule {
912 module_id: registration.manifest.module_id,
913 module_version: registration.manifest.module_version,
914 capabilities: registration.manifest.capabilities,
915 })
916 .collect();
917 Ok((runtime, registrations))
918 }
919
920 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
927 let cached_registration = RegisteredModule {
928 module_id: registration.manifest.module_id.clone(),
929 module_version: registration.manifest.module_version.clone(),
930 capabilities: registration.manifest.capabilities.clone(),
931 };
932 if self.capability_evaluator.record_hello(&cached_registration) {
933 warn!(
934 module_id = %cached_registration.module_id,
935 "capability claims drifted from the cached manifest"
936 );
937 }
938 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
939 self.enforce_capability_denies();
940 }
941 self.refresh_capability_requirements();
942 }
943
944 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
948 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
949 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
950 self.supervisor.set_swap_promotion_observer(observer);
951 }
952
953 pub fn refresh_capability_requirements(&self) {
954 match self.runtime_capability_snapshot() {
955 Ok((runtime, registrations)) => {
956 log_requirement_events(
957 self.capability_evaluator
958 .evaluate_now(&runtime, ®istrations),
959 );
960 }
961 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
962 }
963 }
964
965 fn enforce_capability_denies(&self) {
969 let (_, registrations) = match self.registry.list_modules() {
970 Ok(snapshot) => snapshot,
971 Err(err) => {
972 warn!(error = %err, "failed to read registrations for capability deny census");
973 return;
974 }
975 };
976 let manifests = registrations
977 .into_iter()
978 .map(|registration| {
979 (
980 registration.manifest.module_id.clone(),
981 registration.manifest,
982 )
983 })
984 .collect::<BTreeMap<_, _>>();
985 let census = match self.forwarding.route_census(None) {
986 Ok(census) => census,
987 Err(err) => {
988 warn!(error = %err, "failed to read route census for capability deny enforcement");
989 return;
990 }
991 };
992
993 for (target_module_id, routes) in census {
994 let Some(target_manifest) = manifests.get(&target_module_id) else {
995 continue;
996 };
997 let mut closed_routes = Vec::new();
998 let mut module_goodbyes = Vec::new();
999 for route in routes {
1000 let Principal::Reserved {
1001 module_id: opening_module_id,
1002 } = &route.principal
1003 else {
1004 continue;
1005 };
1006 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1007 continue;
1008 };
1009 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1010 continue;
1011 };
1012
1013 match self.forwarding.release_client_route(
1014 route.goodbye_target.connection_id,
1015 route.goodbye_target.channel,
1016 route.goodbye_target.epoch,
1017 ) {
1018 Ok(RouteRelease::Removed(module_goodbye)) => {
1019 warn!(
1020 opening_module_id,
1021 target_module_id,
1022 capability,
1023 "force-closing route because an attested capability deny edge now matches"
1024 );
1025 closed_routes.push(route);
1026 module_goodbyes.push(module_goodbye);
1027 }
1028 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1029 Err(err) => warn!(
1030 opening_module_id,
1031 target_module_id,
1032 capability,
1033 error = %err,
1034 "failed to force-close capability-denied route"
1035 ),
1036 }
1037 }
1038
1039 if closed_routes.is_empty() {
1040 continue;
1041 }
1042 send_route_control_pushes(
1043 &self.forwarding,
1044 closed_routes,
1045 ClientControlPush::RouteClosed {
1046 module_id: target_module_id,
1047 reason: RouteCloseReason::CapabilityDenied,
1048 drained: false,
1049 abandoned: 0,
1050 excluded_subscriptions: 0,
1051 terminal: Some(false),
1052 },
1053 );
1054 self.emit_route_goodbyes(module_goodbyes);
1055 }
1056 }
1057
1058 fn not_ready_reason(
1064 &self,
1065 registration: &crate::registry::ModuleRegistration,
1066 ) -> Option<NotReadyReason> {
1067 if !registration.ready {
1068 return Some(NotReadyReason {
1069 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1070 capability: None,
1071 });
1072 }
1073 self.first_unprovided_required_capability(registration)
1074 .map(|capability| NotReadyReason {
1075 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1076 capability: Some(capability),
1077 })
1078 }
1079
1080 fn first_unprovided_required_capability(
1098 &self,
1099 registration: &crate::registry::ModuleRegistration,
1100 ) -> Option<String> {
1101 let required = registration
1102 .manifest
1103 .capabilities
1104 .iter()
1105 .flat_map(|declarations| declarations.requires.iter())
1106 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1107 .map(|requirement| requirement.capability.as_str())
1108 .collect::<BTreeSet<_>>();
1109 if required.is_empty() {
1110 return None;
1111 }
1112 let module_id = registration.manifest.module_id.as_str();
1113 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1114 if required
1115 .iter()
1116 .any(|capability| verdict(capability).is_none())
1117 {
1118 self.refresh_capability_requirements();
1119 }
1120 required
1121 .into_iter()
1122 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1123 .map(str::to_string)
1124 }
1125
1126 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1127 self.capability_evaluator
1128 .statuses()
1129 .into_iter()
1130 .map(capability_requirement_status)
1131 .collect()
1132 }
1133
1134 fn deregister_connection(
1141 &self,
1142 connection_id: ConnectionId,
1143 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1144 self.registry.deregister_connection(connection_id)
1145 }
1146
1147 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1148 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1149 return None;
1150 }
1151 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1152 parse_client_control_request(&frame.body)
1153 else {
1154 return None;
1155 };
1156 Some(target_module_id(&target).to_string())
1157 }
1158
1159 pub(crate) fn route_open_capacity_refusal(
1160 &self,
1161 ctx: &RouteCtx,
1162 frame: &Frame,
1163 target_module_id: &str,
1164 limit: usize,
1165 ) -> Result<Frame, RouterError> {
1166 self.route_open_admission_refusal_frame(
1167 ctx,
1168 frame,
1169 target_module_id,
1170 format!(
1171 "connection already has {limit} route.open binds in flight; retry after one settles"
1172 ),
1173 )
1174 }
1175
1176 fn route_open_admission_refusal_frame(
1184 &self,
1185 ctx: &RouteCtx,
1186 frame: &Frame,
1187 target_module_id: &str,
1188 message: impl Into<String>,
1189 ) -> Result<Frame, RouterError> {
1190 self.route_open_refusal_frame(
1191 ctx,
1192 frame,
1193 target_module_id,
1194 "open_admission_full",
1195 error_codes::TARGET_UNAVAILABLE,
1196 message,
1197 )
1198 }
1199
1200 #[cfg(test)]
1206 pub fn handle_control(
1207 &self,
1208 connection_id: ConnectionId,
1209 frame: Frame,
1210 ) -> Result<Vec<Frame>, RouterError> {
1211 match frame.header.ty {
1212 FrameType::Ping => Ok(vec![pong(&frame)?]),
1213 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1214 FrameType::Goodbye => self.handle_goodbye(connection_id),
1215 ty => Ok(vec![control_error_frame(
1216 &frame,
1217 "unsupported_control_frame",
1218 format!("unsupported channel-0 frame {ty:?}"),
1219 )?]),
1220 }
1221 }
1222
1223 pub async fn handle_control_frame(
1224 &self,
1225 ctx: &RouteCtx,
1226 frame: Frame,
1227 ) -> Result<Vec<Frame>, RouterError> {
1228 self.handle_control_frame_timed(ctx, frame, None).await
1229 }
1230
1231 pub(crate) async fn handle_control_frame_timed(
1232 &self,
1233 ctx: &RouteCtx,
1234 frame: Frame,
1235 dispatch_started_at: Option<StdInstant>,
1236 ) -> Result<Vec<Frame>, RouterError> {
1237 match frame.header.ty {
1238 FrameType::Ping => Ok(vec![pong(&frame)?]),
1239 FrameType::Hello => {
1240 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1241 }
1242 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1243 FrameType::Cancel => {
1244 if self
1245 .supervisor
1246 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1247 {
1248 Ok(Vec::new())
1249 } else {
1250 Ok(vec![control_error_frame(
1251 &frame,
1252 "unknown_subscription",
1253 "no supervisor spawn subscription has this correlation id",
1254 )?])
1255 }
1256 }
1257 FrameType::Request => {
1258 if self
1259 .forwarding
1260 .module_endpoint_for_connection(ctx.connection_id)
1261 .map_err(RouterError::Forwarding)?
1262 .is_some()
1263 {
1264 if !is_known_module_request_op(&frame.body) {
1265 return Ok(vec![control_error_frame(
1266 &frame,
1267 "unsupported_control_frame",
1268 "module-originated channel-0 REQUEST is not supported",
1269 )?]);
1270 }
1271 let request = match parse_module_control_request_from_module(&frame.body) {
1272 Ok(request) => request,
1273 Err((err, ControlRequestBodyError::UnknownOp)) => {
1274 return Ok(vec![control_error_frame(
1275 &frame,
1276 "unsupported_control_frame",
1277 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1278 )?])
1279 }
1280 Err((err, ControlRequestBodyError::InvalidBody)) => {
1281 return Ok(vec![control_error_frame(
1282 &frame,
1283 "invalid_control_body",
1284 format!("malformed module control body: {err}"),
1285 )?])
1286 }
1287 };
1288 let op = module_control_request_op(&request);
1289 let corr = frame.header.corr;
1290 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1291 let result =
1292 self.handle_module_control_request(ctx.connection_id, frame, request);
1293 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1294 return result;
1295 }
1296
1297 if is_known_module_request_op(&frame.body) {
1298 return Ok(vec![control_error_frame(
1299 &frame,
1300 "not_registered",
1301 "catalog.update requires an active module registration owned by this connection",
1302 )?]);
1303 }
1304
1305 let request = match parse_client_control_request(&frame.body) {
1306 Ok(request) => request,
1307 Err((err, ControlRequestBodyError::UnknownOp)) => {
1308 return Ok(vec![control_error_frame(
1309 &frame,
1310 "unknown_control_op",
1311 format!("unknown client control op: {err}"),
1312 )?])
1313 }
1314 Err((err, ControlRequestBodyError::InvalidBody)) => {
1315 return Ok(vec![control_error_frame(
1316 &frame,
1317 "invalid_control_body",
1318 format!("malformed client control body: {err}"),
1319 )?])
1320 }
1321 };
1322 let op = client_control_request_op(&request);
1323 let corr = frame.header.corr;
1324 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1325 #[cfg(test)]
1326 if let Some(delay) = self.control_dispatch_delay {
1327 tokio::time::sleep(delay).await;
1328 }
1329 let result = self
1330 .handle_client_control_request(ctx, frame, request)
1331 .await;
1332 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1333 result
1334 }
1335 FrameType::Push => {
1336 let Some(endpoint) = self
1337 .forwarding
1338 .module_endpoint_for_connection(ctx.connection_id)
1339 .map_err(RouterError::Forwarding)?
1340 else {
1341 return Ok(vec![control_error_frame(
1342 &frame,
1343 "unsupported_control_frame",
1344 "client-originated channel-0 PUSH is not supported",
1345 )?]);
1346 };
1347 self.handle_status_update(endpoint, frame)
1348 }
1349 FrameType::Response | FrameType::Error
1350 if self
1351 .forwarding
1352 .module_endpoint_for_connection(ctx.connection_id)
1353 .map_err(RouterError::Forwarding)?
1354 .is_some() =>
1355 {
1356 self.handle_module_relay_response(ctx.connection_id, frame)
1357 }
1358 ty => Ok(vec![control_error_frame(
1359 &frame,
1360 "unsupported_control_frame",
1361 format!("unsupported channel-0 frame {ty:?}"),
1362 )?]),
1363 }
1364 }
1365
1366 pub fn cleanup_connection(
1367 &self,
1368 connection_id: ConnectionId,
1369 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1370 let crash_closed = self
1371 .registry
1372 .get_module_by_connection(connection_id)?
1373 .and_then(|registration| {
1374 self.forwarding
1375 .module_endpoint_for_connection(connection_id)
1376 .ok()
1377 .flatten()
1378 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1379 .map(|routes| (registration.manifest.module_id, routes))
1380 });
1381 if let Some((module_id, routes)) = crash_closed {
1382 let terminal = match self.supervisor.get(&module_id) {
1383 None => false,
1384 Some(module) => match module.will_recover_after_connection_loss() {
1385 Ok(will_recover) => !will_recover,
1386 Err(err) => {
1387 warn!(
1388 %module_id,
1389 error = %err,
1390 "failed to read crash recovery verdict; reporting non-terminal conservatively"
1391 );
1392 false
1393 }
1394 },
1395 };
1396 send_route_control_pushes(
1397 &self.forwarding,
1398 routes,
1399 ClientControlPush::RouteClosed {
1400 module_id,
1401 reason: RouteCloseReason::Crash,
1402 drained: false,
1403 abandoned: 0,
1404 excluded_subscriptions: 0,
1405 terminal: Some(terminal),
1406 },
1407 );
1408 }
1409 let registrations = self.deregister_connection(connection_id);
1410 if let Ok(released_routes) = self.forwarding.cleanup_connection(connection_id) {
1411 self.emit_route_goodbyes(released_routes);
1412 }
1413 if matches!(®istrations, Ok(r) if !r.is_empty()) {
1417 crate::supervise::notify_registration_release();
1418 self.capability_evaluator.wake_deadline_loop();
1419 self.refresh_capability_requirements();
1420 }
1421 self.supervisor.remove_spawn_subscribers(connection_id);
1422 registrations
1423 }
1424
1425 pub(crate) fn handle_route_goodbye(
1426 &self,
1427 connection_id: ConnectionId,
1428 route_channel: u16,
1429 route_epoch: u32,
1430 ) -> Result<bool, RouterError> {
1431 debug!(
1432 connection_id = connection_id.get(),
1433 route_channel, route_epoch, "handling route GOODBYE"
1434 );
1435 let RouteRelease::Removed(released_route) = self
1436 .forwarding
1437 .release_client_route(connection_id, route_channel, route_epoch)
1438 .map_err(RouterError::Forwarding)?
1439 else {
1440 return Ok(false);
1441 };
1442 self.emit_route_goodbyes(vec![released_route]);
1443 Ok(true)
1444 }
1445
1446 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
1447 for released in released_routes {
1448 let frame = match Frame::build_with_version(
1449 released.negotiated_ver,
1450 FrameType::Goodbye,
1451 control_flags(),
1452 released.channel,
1453 released.epoch,
1454 0,
1455 Vec::new(),
1456 ) {
1457 Ok(frame) => frame,
1458 Err(err) => {
1459 warn!(
1460 route_channel = released.channel,
1461 error = %err,
1462 "failed to build route GOODBYE frame"
1463 );
1464 continue;
1465 }
1466 };
1467 if !released.close_on_delivery_failure() {
1468 crate::forwarding::send_module_route_goodbye(
1469 &self.counters,
1470 &released.sink,
1471 frame,
1472 released.module_id.as_deref(),
1473 "client route released",
1474 );
1475 continue;
1476 }
1477 if let Err(err) = released.sink.try_send(frame) {
1478 warn!(
1479 target_connection_id = released.connection_id.get(),
1480 route_channel = released.channel,
1481 error = %err,
1482 "route GOODBYE was not delivered to client; closing target connection"
1483 );
1484 if self
1485 .forwarding
1486 .escalate_client_delivery_failure(
1487 released.connection_id,
1488 released.channel,
1489 released.epoch,
1490 CloseReason::new(
1491 "route_goodbye_delivery_failed",
1492 format!(
1493 "failed to enqueue route GOODBYE for channel {}: {err}",
1494 released.channel
1495 ),
1496 ),
1497 crate::forwarding::UndeliveredFrame {
1498 module_id: released.module_id.as_deref(),
1499 sink: &released.sink,
1500 },
1501 )
1502 .unwrap_or(false)
1503 {
1504 self.counters.increment_goodbye_relay_client_failed();
1505 }
1506 }
1507 }
1508 }
1509
1510 fn send_abandoned_route_bind_goodbye(
1522 &self,
1523 module_sink: &crate::FrameSink,
1524 negotiated_ver: u8,
1525 module_channel: u16,
1526 module_epoch: u32,
1527 ) {
1528 let frame = match Frame::build_with_version(
1529 negotiated_ver,
1530 FrameType::Goodbye,
1531 control_flags(),
1532 module_channel,
1533 module_epoch,
1534 0,
1535 Vec::new(),
1536 ) {
1537 Ok(frame) => frame,
1538 Err(err) => {
1539 warn!(
1540 route_channel = module_channel,
1541 error = %err,
1542 "failed to build GOODBYE for abandoned route.bind"
1543 );
1544 return;
1545 }
1546 };
1547 crate::forwarding::send_module_route_goodbye(
1548 &self.counters,
1549 module_sink,
1550 frame,
1551 None,
1552 "abandoned route.bind",
1553 );
1554 }
1555
1556 fn handle_hello(
1557 &self,
1558 connection_id: ConnectionId,
1559 sink: Option<crate::FrameSink>,
1560 frame: Frame,
1561 ) -> Result<Vec<Frame>, RouterError> {
1562 debug!(
1563 connection_id = connection_id.get(),
1564 corr = frame.header.corr,
1565 "handling HELLO"
1566 );
1567 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
1568 Ok(value) => value,
1569 Err(err) => {
1570 return Ok(vec![control_error_frame(
1571 &frame,
1572 "invalid_hello",
1573 format!("malformed HELLO body: {err}"),
1574 )?])
1575 }
1576 };
1577 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
1578 return Ok(vec![control_error_frame(
1579 &frame,
1580 "invalid_capability_grammar",
1581 err.to_string(),
1582 )?]);
1583 }
1584 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
1585 return Ok(vec![control_error_frame(
1586 &frame,
1587 "invalid_manifest",
1588 err.to_string(),
1589 )?]);
1590 }
1591 if let Some(provenance) = hello_value
1592 .get("manifest")
1593 .and_then(|manifest| manifest.get("provenance"))
1594 {
1595 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
1596 return Ok(vec![control_error_frame(
1597 &frame,
1598 "invalid_manifest",
1599 format!("malformed manifest provenance: {err}"),
1600 )?]);
1601 }
1602 }
1603 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
1604 Ok(hello) => hello,
1605 Err(err) => {
1606 return Ok(vec![control_error_frame(
1607 &frame,
1608 "invalid_hello",
1609 format!("malformed HELLO body: {err}"),
1610 )?])
1611 }
1612 };
1613
1614 if hello.protocol_ver != hello.manifest.protocol_ver {
1615 return Ok(vec![control_error_frame(
1616 &frame,
1617 "invalid_manifest",
1618 format!(
1619 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
1620 hello.protocol_ver, hello.manifest.protocol_ver
1621 ),
1622 )?]);
1623 }
1624
1625 if hello.manifest.module_id.trim().is_empty() {
1626 return Ok(vec![control_error_frame(
1627 &frame,
1628 "invalid_manifest",
1629 "manifest module_id must not be empty",
1630 )?]);
1631 }
1632
1633 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
1634 Ok(negotiated_ver) => negotiated_ver,
1635 Err(message) => {
1636 return Ok(vec![control_error_frame(
1637 &frame,
1638 "version_unsupported",
1639 message,
1640 )?])
1641 }
1642 };
1643
1644 let swap_admission = self
1652 .supervisor
1653 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
1654 if swap_admission == SwapHelloAdmission::Refused {
1655 warn!(
1656 module_id = %hello.manifest.module_id,
1657 connection_id = connection_id.get(),
1658 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
1659 );
1660 return Ok(vec![control_error_frame(
1661 &frame,
1662 "swap_token_invalid",
1663 format!(
1664 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
1665 hello.manifest.module_id
1666 ),
1667 )?]);
1668 }
1669 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
1670
1671 if let Some(rejection) = (!swap_candidate)
1679 .then(|| {
1680 self.supervisor.reserved_hello_rejection(
1681 &hello.manifest.module_id,
1682 hello.launch_nonce.as_deref(),
1683 )
1684 })
1685 .flatten()
1686 {
1687 let message = match rejection {
1688 ReservedHelloRejection::Exact { module_id } => format!(
1689 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
1690 ),
1691 ReservedHelloRejection::Prefix {
1692 prefix,
1693 owner_module_id,
1694 } => format!(
1695 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
1696 hello.manifest.module_id
1697 ),
1698 };
1699 return Ok(vec![control_error_frame(
1700 &frame,
1701 "reserved_module",
1702 message,
1703 )?]);
1704 }
1705
1706 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
1707 &hello.manifest.module_id,
1708 hello.manifest.capabilities.as_ref(),
1709 );
1710 if let Some(refusal) = reserved_capability_refusals.first() {
1711 let capability = refusal.capability.clone();
1712 let bound_module = refusal.claimants[0].clone();
1713 log_duplicate_claim_events(reserved_capability_refusals);
1714 return Ok(vec![control_error_frame(
1715 &frame,
1716 "reserved_capability",
1717 format!(
1718 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
1719 capability, bound_module, hello.manifest.module_id
1720 ),
1721 )?]);
1722 }
1723
1724 if self
1727 .forwarding
1728 .connection_has_client_routes(connection_id)
1729 .map_err(RouterError::Forwarding)?
1730 {
1731 return Ok(vec![control_error_frame(
1732 &frame,
1733 "invalid_hello",
1734 "connection has open client routes and cannot also register as a module",
1735 )?]);
1736 }
1737
1738 let control_ops = effective_module_control_ops(hello.control_ops);
1739 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
1742 if swap_candidate {
1743 return self.register_swap_candidate(
1744 connection_id,
1745 sink,
1746 &frame,
1747 hello.manifest,
1748 negotiated_ver,
1749 control_ops,
1750 hello_ack,
1751 );
1752 }
1753 let registration = match self.registry.register_with_control_ops(
1754 hello.manifest,
1755 negotiated_ver,
1756 connection_id,
1757 control_ops,
1758 ) {
1759 Ok(registration) => registration,
1760 Err(RegistryError::DuplicateModuleId { module_id }) => {
1761 return Ok(vec![control_error_frame(
1762 &frame,
1763 "duplicate_module_id",
1764 format!(
1765 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
1766 ),
1767 )?])
1768 }
1769 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
1770 return Ok(vec![control_error_frame(
1771 &frame,
1772 "invalid_module_id",
1773 err.to_string(),
1774 )?])
1775 }
1776 Err(err) => {
1777 return Ok(vec![control_error_frame(
1778 &frame,
1779 "registry_error",
1780 err.to_string(),
1781 )?])
1782 }
1783 };
1784
1785 let reply = if let Some(sink) = sink {
1786 let concurrency = manifest_concurrency(®istration.manifest);
1802 if let Err(err) = self.forwarding.register_module_connection_acked(
1803 connection_id,
1804 registration.manifest.module_id.clone(),
1805 negotiated_ver,
1806 concurrency,
1807 sink,
1808 hello_ack,
1809 ) {
1810 if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
1814 crate::supervise::notify_registration_release();
1815 }
1816 return Ok(vec![control_error_frame(
1817 &frame,
1818 forwarding_error_code(&err),
1819 err.to_string(),
1820 )?]);
1821 }
1822 Vec::new()
1823 } else {
1824 vec![hello_ack]
1827 };
1828
1829 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
1838 info!(
1839 module_id = %registration.manifest.module_id,
1840 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
1841 );
1842 }
1843
1844 self.apply_registration_capabilities(®istration);
1845
1846 info!(
1847 module_id = %registration.manifest.module_id,
1848 module_version = %registration.manifest.module_version,
1849 negotiated_ver,
1850 routable_provider = manifest_provides_routable_role(®istration.manifest),
1851 connection_id = connection_id.get(),
1852 "module registered"
1853 );
1854
1855 Ok(reply)
1856 }
1857
1858 #[allow(clippy::too_many_arguments)]
1868 fn register_swap_candidate(
1869 &self,
1870 connection_id: ConnectionId,
1871 sink: Option<crate::FrameSink>,
1872 frame: &Frame,
1873 manifest: ModuleManifest,
1874 negotiated_ver: u8,
1875 control_ops: Vec<String>,
1876 hello_ack: Frame,
1877 ) -> Result<Vec<Frame>, RouterError> {
1878 let module_id = manifest.module_id.clone();
1879 let registration = match self.registry.register_candidate_with_control_ops(
1880 manifest,
1881 negotiated_ver,
1882 connection_id,
1883 control_ops,
1884 ) {
1885 Ok(registration) => registration,
1886 Err(RegistryError::DuplicateModuleId { module_id }) => {
1887 return Ok(vec![control_error_frame(
1888 frame,
1889 "duplicate_module_id",
1890 format!(
1891 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
1892 ),
1893 )?])
1894 }
1895 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
1896 return Ok(vec![control_error_frame(
1897 frame,
1898 "invalid_module_id",
1899 err.to_string(),
1900 )?])
1901 }
1902 Err(err) => {
1903 return Ok(vec![control_error_frame(
1904 frame,
1905 "registry_error",
1906 err.to_string(),
1907 )?])
1908 }
1909 };
1910 let reply = if let Some(sink) = sink {
1911 let concurrency = manifest_concurrency(®istration.manifest);
1915 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
1916 connection_id,
1917 module_id.clone(),
1918 negotiated_ver,
1919 concurrency,
1920 sink,
1921 hello_ack,
1922 ) {
1923 if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
1924 crate::supervise::notify_registration_release();
1925 }
1926 return Ok(vec![control_error_frame(
1927 frame,
1928 forwarding_error_code(&err),
1929 err.to_string(),
1930 )?]);
1931 }
1932 Vec::new()
1933 } else {
1934 vec![hello_ack]
1935 };
1936 self.supervisor.mark_swap_candidate_admitted(&module_id);
1937 info!(
1938 module_id = %module_id,
1939 module_version = %registration.manifest.module_version,
1940 negotiated_ver,
1941 ready = registration.ready,
1942 connection_id = connection_id.get(),
1943 "swap candidate registered; not routable until cutover"
1944 );
1945 Ok(reply)
1946 }
1947
1948 fn build_hello_ack(
1949 &self,
1950 frame: &Frame,
1951 negotiated_ver: u8,
1952 module_id: &str,
1953 ) -> Result<Frame, RouterError> {
1954 let ack = ModuleHelloAckBody {
1955 negotiated_ver,
1956 subc_ops: module_subc_ops(),
1957 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
1958 storage: self
1959 .storage_config
1960 .as_ref()
1961 .map(|cfg| cfg.descriptor_for(module_id)),
1962 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
1963 };
1964 let body = serde_json::to_vec(&ack).map_err(|err| {
1965 RouterError::backend(
1966 0,
1967 frame.header.corr,
1968 format!("failed to encode HELLO_ACK: {err}"),
1969 )
1970 })?;
1971
1972 Frame::build_with_version(
1973 negotiated_ver,
1974 FrameType::HelloAck,
1975 control_flags(),
1976 0,
1977 0,
1978 frame.header.corr,
1979 body,
1980 )
1981 .map_err(RouterError::FrameBuild)
1982 }
1983
1984 async fn handle_client_control_request(
1985 &self,
1986 ctx: &RouteCtx,
1987 frame: Frame,
1988 request: ClientControlRequest,
1989 ) -> Result<Vec<Frame>, RouterError> {
1990 match request {
1991 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
1992 ClientControlRequest::CatalogList { module_id } => {
1993 self.handle_catalog_list(frame, module_id)
1994 }
1995 ClientControlRequest::RouteOpen {
1996 target,
1997 identity,
1998 consumer_identity,
1999 consumer_capabilities,
2000 admission_facts,
2001 } => {
2002 self.handle_route_open(
2003 ctx,
2004 frame,
2005 RouteOpenRequest {
2006 target,
2007 identity,
2008 consumer_identity,
2009 consumer_capabilities,
2010 admission_facts,
2011 },
2012 )
2013 .await
2014 }
2015 ClientControlRequest::RoutePoll {
2016 route_channel,
2017 route_epoch,
2018 kind,
2019 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2020 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame),
2021 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2022 self.handle_supervisor_spawn_snapshot(frame)
2023 }
2024 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2025 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2026 }
2027 ClientControlRequest::SupervisorRestart {
2028 module_id,
2029 drain_timeout_ms,
2030 } => {
2031 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms)
2032 .await
2033 }
2034 ClientControlRequest::SupervisorSwap {
2035 module_id,
2036 ready_timeout_ms,
2037 } => {
2038 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms)
2039 .await
2040 }
2041 ClientControlRequest::SupervisorReload { module_id } => {
2042 self.handle_supervisor_reload(frame, module_id).await
2043 }
2044 ClientControlRequest::SupervisorRescan { preview } => {
2045 self.handle_supervisor_rescan(frame, preview).await
2046 }
2047 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2048 self.handle_supervisor_release_reserved(frame, module_id)
2049 .await
2050 }
2051 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2052 self.handle_supervisor_set_enabled(frame, module_id, enabled)
2053 .await
2054 }
2055 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2056 self.handle_supervisor_health_probe(frame, module_id).await
2057 }
2058 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2059 ClientControlRequest::SupervisorRoutes { module_id } => {
2060 self.handle_supervisor_routes(frame, module_id)
2061 }
2062 ClientControlRequest::SupervisorProvenance { module_id } => {
2063 self.handle_supervisor_provenance(frame, module_id).await
2064 }
2065 ClientControlRequest::SupervisorStderrTail {
2066 module_id,
2067 max_lines,
2068 max_bytes,
2069 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2070 ClientControlRequest::SupervisorTerminals { module_id } => {
2071 self.handle_supervisor_terminals(frame, module_id).await
2072 }
2073 }
2074 }
2075
2076 fn handle_module_control_request(
2077 &self,
2078 connection_id: ConnectionId,
2079 frame: Frame,
2080 request: ModuleControlRequestFromModule,
2081 ) -> Result<Vec<Frame>, RouterError> {
2082 match request {
2083 ModuleControlRequestFromModule::CatalogUpdate {
2084 provides,
2085 capabilities,
2086 ready,
2087 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2088 ModuleControlRequestFromModule::LiveRoots {} => {
2089 let registered = self
2090 .registry
2091 .get_module_by_connection(connection_id)
2092 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2093 let Some(registration) = registered else {
2094 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2095 };
2096 let response = self
2097 .forwarding
2098 .live_roots(®istration.manifest.module_id)
2099 .map_err(RouterError::Forwarding)?;
2100 Ok(vec![control_response_body_frame(
2101 &frame,
2102 &response,
2103 "ModuleControlResponseToModule::LiveRoots",
2104 )?])
2105 }
2106 }
2107 }
2108
2109 fn handle_catalog_update(
2110 &self,
2111 connection_id: ConnectionId,
2112 frame: Frame,
2113 provides: Vec<ProviderRole>,
2114 capabilities: Option<CapabilityDeclarations>,
2115 ready: Option<bool>,
2116 ) -> Result<Vec<Frame>, RouterError> {
2117 self.refresh_capability_requirements();
2118 let Some(registration) = self
2119 .registry
2120 .get_module_by_connection(connection_id)
2121 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2122 else {
2123 return Ok(vec![control_error_frame(
2124 &frame,
2125 "not_registered",
2126 "catalog.update requires an active module registration owned by this connection",
2127 )?]);
2128 };
2129
2130 if let Some(message) =
2131 catalog_update_frozen_field_message(®istration.manifest, &provides)
2132 {
2133 return Ok(vec![control_error_frame(
2134 &frame,
2135 "catalog_update_frozen_field",
2136 message,
2137 )?]);
2138 }
2139
2140 let mut candidate = registration.manifest.clone();
2141 candidate.provides = provides.clone();
2142 candidate.capabilities = capabilities
2143 .clone()
2144 .or_else(|| registration.manifest.capabilities.clone());
2145 if let Err(err) = candidate.validate_capability_grammar() {
2146 return Ok(vec![control_error_frame(
2147 &frame,
2148 "invalid_capability_grammar",
2149 err.to_string(),
2150 )?]);
2151 }
2152
2153 let updated = self
2154 .registry
2155 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
2156 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2157 if updated.is_none() {
2158 return Ok(vec![control_error_frame(
2159 &frame,
2160 "not_registered",
2161 "catalog.update requires an active module registration owned by this connection",
2162 )?]);
2163 }
2164 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
2165 log_duplicate_claim_events(
2166 self.capability_evaluator
2167 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
2168 );
2169 }
2170 if capability_census_trigger(
2171 registration.manifest.capabilities.as_ref(),
2172 updated
2173 .as_ref()
2174 .and_then(|entry| entry.manifest.capabilities.as_ref()),
2175 ) {
2176 self.enforce_capability_denies();
2177 }
2178 self.refresh_capability_requirements();
2179
2180 let response = ModuleControlResponseToModule::CatalogUpdate {};
2181 control_response_body_frame(
2182 &frame,
2183 &response,
2184 "ModuleControlResponseToModule::CatalogUpdate",
2185 )
2186 .map(|frame| vec![frame])
2187 }
2188
2189 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
2190 self.refresh_capability_requirements();
2191 let mut counters = self.counters.snapshot();
2197 if let (Ok((connections_with_routes, max)), Some(obj)) = (
2198 self.forwarding.client_route_concentration(),
2199 counters.as_object_mut(),
2200 ) {
2201 obj.insert(
2202 "client_connections_with_routes".into(),
2203 connections_with_routes.into(),
2204 );
2205 obj.insert("max_routes_on_one_connection".into(), max.into());
2206 }
2207 if let (Some(open_breakers), Some(obj)) = (
2213 self.route_bind_breakers.open_snapshot(),
2214 counters.as_object_mut(),
2215 ) {
2216 obj.insert("route_bind_breakers_open".into(), open_breakers);
2217 }
2218 let response = ClientControlResponse::ServerDescribe {
2219 protocol_ver: PROTOCOL_VERSION,
2220 subc_ops: subc_ops(),
2221 capabilities: self.subc_capabilities.as_ref().to_vec(),
2222 connected_clients: self.connected_clients.count(),
2223 counters: Some(counters),
2224 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
2225 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
2226 capability_requirements: self.capability_requirement_statuses(),
2227 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2228 };
2229 Ok(vec![control_response_body_frame(
2230 &frame,
2231 &response,
2232 "ClientControlResponse::ServerDescribe",
2233 )?])
2234 }
2235
2236 fn handle_catalog_list(
2237 &self,
2238 frame: Frame,
2239 module_id: Option<String>,
2240 ) -> Result<Vec<Frame>, RouterError> {
2241 let (generation, modules) = self.registry.list_modules().map_err(|err| {
2242 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
2243 })?;
2244 let entries = modules
2245 .into_iter()
2246 .filter(|registration| {
2247 module_id
2248 .as_deref()
2249 .map(|wanted| registration.manifest.module_id == wanted)
2250 .unwrap_or(true)
2251 })
2252 .map(|registration| {
2253 let not_ready = self.not_ready_reason(®istration);
2254 let roles = registration.manifest.provides;
2255 CatalogEntry {
2256 module_id: registration.manifest.module_id,
2257 ready: not_ready.is_none(),
2258 not_ready,
2259 module_version: Some(registration.manifest.module_version),
2260 roles,
2261 control_ops: registration.control_ops,
2262 capabilities: registration.manifest.capabilities,
2263 self_signals: registration.manifest.self_signals,
2264 }
2265 })
2266 .collect();
2267 let response = ClientControlResponse::CatalogList {
2268 generation,
2269 modules: entries,
2270 subc_ops: subc_ops(),
2271 };
2272 Ok(vec![control_response_body_frame(
2273 &frame,
2274 &response,
2275 "ClientControlResponse::CatalogList",
2276 )?])
2277 }
2278
2279 fn route_open_principal(
2280 &self,
2281 frame: &Frame,
2282 consumer_identity: Option<ConsumerIdentity>,
2283 ) -> Result<Result<Principal, Frame>, RouterError> {
2284 let Some(consumer_identity) = consumer_identity else {
2285 return Ok(Ok(Principal::Direct));
2286 };
2287
2288 if self.supervisor.spawned_consumer_authorized(
2289 &consumer_identity.module_id,
2290 &consumer_identity.launch_nonce,
2291 ) {
2292 return Ok(Ok(Principal::Reserved {
2293 module_id: consumer_identity.module_id,
2294 }));
2295 }
2296
2297 Ok(Err(control_error_frame(
2298 frame,
2299 "bad_consumer_identity",
2300 format!(
2301 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
2302 consumer_identity.module_id
2303 ),
2304 )?))
2305 }
2306
2307 fn route_open_refusal_frame(
2313 &self,
2314 ctx: &RouteCtx,
2315 frame: &Frame,
2316 module_id: &str,
2317 reason: &'static str,
2318 code: &'static str,
2319 message: impl Into<String>,
2320 ) -> Result<Frame, RouterError> {
2321 self.observe_route_open_refusal(ctx, module_id, reason, code);
2322 control_error_frame(frame, code, message.into())
2323 }
2324
2325 fn route_open_breaker_refusal_frame(
2343 &self,
2344 ctx: &RouteCtx,
2345 frame: &Frame,
2346 module_id: &str,
2347 consecutive_timeouts: u32,
2348 retry_in: Duration,
2349 probe_in_flight: bool,
2350 ) -> Result<Frame, RouterError> {
2351 self.counters
2352 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
2353 debug!(
2354 target: "control",
2355 code = "module_timeout",
2356 module_id = ?module_id,
2357 connection_id = ctx.connection_id.get(),
2358 consecutive_timeouts,
2359 retry_in_ms = retry_in.as_millis() as u64,
2360 probe_in_flight,
2361 "route.open refused by open bind-relay breaker"
2362 );
2363 let detail = if probe_in_flight {
2364 "one probe bind is already in flight; retry once it settles".to_string()
2365 } else {
2366 format!("not relaying for another {retry_in:?}")
2367 };
2368 control_error_frame(
2369 frame,
2370 "module_timeout",
2371 format!(
2372 "module_id '{module_id}' failed {consecutive_timeouts} consecutive route.bind \
2373 relays; {detail}"
2374 ),
2375 )
2376 }
2377
2378 fn observe_route_open_refusal(
2391 &self,
2392 ctx: &RouteCtx,
2393 module_id: &str,
2394 reason: &'static str,
2395 code: &'static str,
2396 ) {
2397 self.counters.increment_route_open_refused(code);
2398 info!(
2399 target: "control",
2400 code,
2401 reason,
2402 module_id = ?module_id,
2403 connection_id = ctx.connection_id.get(),
2404 "route.open refused"
2405 );
2406 }
2407
2408 fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
2453 self.counters.increment_route_open_accepted(principal);
2454 info!(
2455 target: "control",
2456 principal,
2457 module_id,
2458 connection_id = ctx.connection_id.get(),
2459 "route.open accepted"
2460 );
2461 }
2462
2463 fn supervised_absent_route_open_refusal_frame(
2464 &self,
2465 ctx: &RouteCtx,
2466 frame: &Frame,
2467 module_id: &str,
2468 code: &'static str,
2469 status: &crate::supervise::ModuleStatus,
2470 ) -> Result<Frame, RouterError> {
2471 self.counters.increment_route_open_refused(code);
2472 info!(
2473 target: "control",
2474 code,
2475 reason = "supervised_not_registered",
2476 module_id = ?module_id,
2477 connection_id = ctx.connection_id.get(),
2478 state = %status.state,
2479 enabled = status.enabled,
2480 live = status.live,
2481 "route.open refused"
2482 );
2483 control_error_frame(
2484 frame,
2485 code,
2486 format!(
2487 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
2488 status.state, status.enabled, status.live
2489 ),
2490 )
2491 }
2492
2493 async fn handle_route_open(
2494 &self,
2495 ctx: &RouteCtx,
2496 frame: Frame,
2497 request: RouteOpenRequest,
2498 ) -> Result<Vec<Frame>, RouterError> {
2499 let RouteOpenRequest {
2500 target,
2501 mut identity,
2502 consumer_identity,
2503 consumer_capabilities,
2504 admission_facts,
2505 } = request;
2506 let target_module_id = target_module_id(&target).to_string();
2507 debug!(
2508 connection_id = ctx.connection_id.get(),
2509 corr = frame.header.corr,
2510 module_id = %target_module_id,
2511 "handling route.open"
2512 );
2513
2514 let Some(registration) = self
2533 .registry
2534 .get_module(&target_module_id)
2535 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2536 else {
2537 if let Some((status, warming)) =
2538 self.supervisor_status(&target_module_id, frame.header.corr)?
2539 {
2540 if status.protocol == ModuleProtocol::None {
2547 return Ok(vec![self.route_open_refusal_frame(
2548 ctx,
2549 &frame,
2550 &target_module_id,
2551 "protocol_none",
2552 error_codes::MODULE_NO_PROTOCOL,
2553 format!(
2554 "module_id '{target_module_id}' is declared protocol: none; \
2555 it speaks no subc wire and serves no routes"
2556 ),
2557 )?]);
2558 }
2559 let code = if warming {
2560 "module_warming"
2561 } else {
2562 "target_unavailable"
2563 };
2564 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
2565 ctx,
2566 &frame,
2567 &target_module_id,
2568 code,
2569 &status,
2570 )?]);
2571 }
2572 if let Some(removed_ago_ms) =
2573 self.supervisor.removal_tombstone_age_ms(&target_module_id)
2574 {
2575 return Ok(vec![self.route_open_refusal_frame(
2576 ctx,
2577 &frame,
2578 &target_module_id,
2579 "removed",
2580 error_codes::MODULE_REMOVED,
2581 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
2582 )?]);
2583 }
2584 return Ok(vec![self.route_open_refusal_frame(
2585 ctx,
2586 &frame,
2587 &target_module_id,
2588 "not_registered",
2589 error_codes::UNKNOWN_MODULE,
2590 format!("module_id '{target_module_id}' is not registered"),
2591 )?]);
2592 };
2593
2594 if !registration.ready {
2598 self.counters
2599 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
2600 info!(
2601 target: "control",
2602 code = error_codes::MODULE_WARMING,
2603 module_id = ?target_module_id,
2604 connection_id = ctx.connection_id.get(),
2605 reason = "declared_not_ready",
2606 "route.open refused"
2607 );
2608 return Ok(vec![control_error_body_frame(
2609 &frame,
2610 ErrorBody {
2611 code: error_codes::MODULE_WARMING.to_string(),
2612 message: format!(
2613 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
2614 ),
2615 detail: Some(serde_json::json!({
2616 "reason": "declared_not_ready"
2617 })),
2618 },
2619 )?]);
2620 }
2621
2622 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
2641 self.counters
2642 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
2643 info!(
2644 target: "control",
2645 code = error_codes::MODULE_WARMING,
2646 module_id = ?target_module_id,
2647 connection_id = ctx.connection_id.get(),
2648 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
2649 capability = %capability,
2650 "route.open refused"
2651 );
2652 return Ok(vec![control_error_body_frame(
2653 &frame,
2654 ErrorBody {
2655 code: error_codes::MODULE_WARMING.to_string(),
2656 message: format!(
2657 "module_id '{target_module_id}' requires capability '{capability}', \
2658 which no registered module provides; retry"
2659 ),
2660 detail: Some(serde_json::json!({
2661 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
2662 "capability": capability,
2663 })),
2664 },
2665 )?]);
2666 }
2667
2668 if !target_has_required_role(&target, ®istration.manifest.provides) {
2669 return Ok(vec![self.route_open_refusal_frame(
2670 ctx,
2671 &frame,
2672 &target_module_id,
2673 "role_not_provided",
2674 "target_unavailable",
2675 format!("module_id '{target_module_id}' does not provide the requested target"),
2676 )?]);
2677 }
2678
2679 if registration.state != ChannelState::Active {
2680 return Ok(vec![self.route_open_refusal_frame(
2681 ctx,
2682 &frame,
2683 &target_module_id,
2684 "registration_not_active",
2685 "target_unavailable",
2686 format!("module_id '{target_module_id}' is not active"),
2687 )?]);
2688 }
2689
2690 if self
2691 .forwarding
2692 .module_is_draining(&target_module_id)
2693 .map_err(RouterError::Forwarding)?
2694 {
2695 return Ok(vec![self.route_open_refusal_frame(
2696 ctx,
2697 &frame,
2698 &target_module_id,
2699 "reloading",
2700 "module_reloading",
2701 format!("module_id '{target_module_id}' is reloading"),
2702 )?]);
2703 }
2704
2705 if self
2706 .process_liveness
2707 .as_ref()
2708 .and_then(|process_liveness| process_liveness.process_live(&target_module_id))
2709 == Some(false)
2710 {
2711 return Ok(vec![self.route_open_refusal_frame(
2712 ctx,
2713 &frame,
2714 &target_module_id,
2715 "supervisor_not_live",
2716 "target_unavailable",
2717 format!("module_id '{target_module_id}' is not live"),
2718 )?]);
2719 }
2720
2721 if !self
2722 .forwarding
2723 .has_live_module_connection(&target_module_id)
2724 .map_err(RouterError::Forwarding)?
2725 {
2726 return Ok(vec![self.route_open_refusal_frame(
2727 ctx,
2728 &frame,
2729 &target_module_id,
2730 "no_forwarding_connection",
2731 "target_unavailable",
2732 format!("module_id '{target_module_id}' has no live forwarding connection"),
2733 )?]);
2734 }
2735
2736 if let Some(error) =
2737 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
2738 {
2739 self.observe_route_open_refusal(
2740 ctx,
2741 &target_module_id,
2742 "op_not_allowed",
2743 "op_not_allowed",
2744 );
2745 return Ok(vec![error]);
2746 }
2747
2748 let principal = match self.route_open_principal(&frame, consumer_identity)? {
2749 Ok(principal) => principal,
2750 Err(error) => {
2751 self.observe_route_open_refusal(
2752 ctx,
2753 &target_module_id,
2754 "bad_consumer_identity",
2755 "bad_consumer_identity",
2756 );
2757 return Ok(vec![error]);
2758 }
2759 };
2760
2761 if let Principal::Reserved {
2765 module_id: opening_module_id,
2766 } = &principal
2767 {
2768 if let Some(opening_registration) = self
2769 .registry
2770 .get_module(opening_module_id)
2771 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2772 {
2773 if let Some(capability) =
2774 denied_capability(&opening_registration.manifest, ®istration.manifest)
2775 {
2776 warn!(
2777 opening_module_id,
2778 target_module_id,
2779 capability,
2780 "refusing route.open because an attested capability deny edge matches"
2781 );
2782 return Ok(vec![self.route_open_refusal_frame(
2783 ctx,
2784 &frame,
2785 &target_module_id,
2786 "capability_deny_edge",
2787 "capability_forbidden",
2788 format!(
2789 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
2790 ),
2791 )?]);
2792 }
2793 }
2794 }
2795
2796 if admission_facts.is_some() {
2797 let carrier_matches = matches!(
2798 &principal,
2799 Principal::Reserved { module_id }
2800 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
2801 );
2802 if !carrier_matches {
2803 return Ok(vec![self.route_open_refusal_frame(
2804 ctx,
2805 &frame,
2806 &target_module_id,
2807 "admission_facts_carrier_not_permitted",
2808 "admission_facts_not_permitted",
2809 "admission facts may only be carried by the configured reserved module",
2810 )?]);
2811 }
2812
2813 let target_allowed = self
2814 .admission_facts_targets
2815 .as_ref()
2816 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
2817 if !target_allowed {
2818 return Ok(vec![self.route_open_refusal_frame(
2819 ctx,
2820 &frame,
2821 &target_module_id,
2822 "admission_facts_target_not_listed",
2823 "admission_facts_target_not_allowed",
2824 format!(
2825 "admission facts are not permitted for target module_id '{target_module_id}'"
2826 ),
2827 )?]);
2828 }
2829
2830 }
2834
2835 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
2857 Ok(project_root) => project_root,
2858 Err(err) => {
2859 return Ok(vec![control_error_frame(
2860 &frame,
2861 "invalid_project_root",
2862 err.to_string(),
2863 )?])
2864 }
2865 };
2866 identity.project_root = project_root.as_path().to_path_buf();
2867
2868 let _concurrency_guard = match self
2879 .route_bind_concurrency
2880 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
2881 {
2882 Ok(guard) => guard,
2883 Err(in_flight) => {
2884 return Ok(vec![self.route_open_admission_refusal_frame(
2885 ctx,
2886 &frame,
2887 &target_module_id,
2888 format!(
2889 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
2890 ),
2891 )?]);
2892 }
2893 };
2894
2895 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
2898 RouteBindAdmission::Admitted { guard, probe } => {
2899 if probe {
2900 info!(
2901 module_id = %target_module_id,
2902 connection_id = ctx.connection_id.get(),
2903 "route.bind breaker half-open: admitting one probe"
2904 );
2905 }
2906 guard
2907 }
2908 RouteBindAdmission::Refused {
2909 consecutive_timeouts,
2910 retry_in,
2911 probe_in_flight,
2912 } => {
2913 return Ok(vec![self.route_open_breaker_refusal_frame(
2914 ctx,
2915 &frame,
2916 &target_module_id,
2917 consecutive_timeouts,
2918 retry_in,
2919 probe_in_flight,
2920 )?]);
2921 }
2922 };
2923
2924 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
2929 let relay_deadline = Instant::now() + route_bind_relay_timeout;
2930 let pending = match self
2931 .forwarding
2932 .begin_route_bind_relay_for(
2933 ctx.connection_id,
2934 ctx.egress.clone(),
2935 response_version(&frame),
2936 frame.header.corr,
2937 &target_module_id,
2938 principal.clone(),
2939 Some(project_root),
2940 relay_deadline,
2941 )
2942 .await
2943 {
2944 Ok(pending) => pending,
2945 Err(err) => {
2946 return Ok(vec![self.route_open_refusal_frame(
2947 ctx,
2948 &frame,
2949 &target_module_id,
2950 "relay_reservation_failed",
2951 forwarding_error_code(&err),
2952 err.to_string(),
2953 )?])
2954 }
2955 };
2956 let crate::forwarding::PendingRouteBindRelay {
2957 endpoint,
2958 module_sink,
2959 negotiated_ver,
2960 client_channel,
2961 client_epoch,
2962 module_channel,
2963 module_epoch,
2964 corr: relay_corr,
2965 receiver,
2966 } = pending;
2967 let mut reservation =
2968 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
2969
2970 debug!(
2971 connection_id = ctx.connection_id.get(),
2972 client_channel,
2973 client_epoch,
2974 module_channel,
2975 module_epoch,
2976 "reserved route handle pair"
2977 );
2978 let principal_label = match &principal {
2981 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
2982 Principal::Direct => "direct".to_string(),
2983 other => format!("{other:?}"),
2984 };
2985 let relay = ModuleControlRequest::RouteBind {
2986 route_channel: module_channel,
2987 epoch: module_epoch,
2988 target,
2989 identity,
2990 principal: Some(principal),
2991 consumer_capabilities,
2992 admission_facts,
2993 };
2994 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
2995 RouterError::backend(
2996 0,
2997 frame.header.corr,
2998 format!("failed to encode route.bind request: {err}"),
2999 )
3000 })?;
3001 let relay_frame = Frame::build_with_version(
3002 negotiated_ver,
3003 FrameType::Request,
3004 control_flags(),
3005 0,
3006 0,
3007 relay_corr,
3008 relay_body,
3009 )
3010 .map_err(RouterError::FrameBuild)?;
3011
3012 if let Err(err) = module_sink.send(relay_frame).await {
3013 reservation.release_and_disarm();
3014 return Ok(vec![self.route_open_refusal_frame(
3015 ctx,
3016 &frame,
3017 &target_module_id,
3018 "relay_send_failed",
3019 "target_unavailable",
3020 err.to_string(),
3021 )?]);
3022 }
3023
3024 if !self
3025 .forwarding
3026 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
3027 .map_err(RouterError::Forwarding)?
3028 {
3029 self.send_abandoned_route_bind_goodbye(
3030 &module_sink,
3031 negotiated_ver,
3032 module_channel,
3033 module_epoch,
3034 );
3035 }
3036
3037 match timeout_at(relay_deadline, receiver).await {
3038 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
3039 reservation.disarm();
3040 if breaker.record_accepted() {
3041 info!(
3042 module_id = %target_module_id,
3043 "route.bind breaker closed: the probe was accepted"
3044 );
3045 }
3046 self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
3047 Ok(Vec::new())
3048 }
3049 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
3050 reservation.release_and_disarm();
3051 breaker.record_inconclusive();
3055 self.counters
3056 .increment_route_open_refused("module_rejected");
3057 info!(
3058 target: "control",
3059 code = "module_rejected",
3060 module_code = ?body.code,
3061 module_id = ?target_module_id,
3062 connection_id = ctx.connection_id.get(),
3063 "route.open refused"
3064 );
3065 Ok(vec![control_error_body_frame(&frame, body)?])
3066 }
3067 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
3068 reservation.release_and_disarm();
3069 breaker.record_inconclusive();
3070 tracing::warn!(
3077 module_id = %target_module_id,
3078 "route.bind relay abandoned: {message}"
3079 );
3080 Ok(vec![self.route_open_refusal_frame(
3081 ctx,
3082 &frame,
3083 &target_module_id,
3084 "relay_abandoned",
3085 "target_unavailable",
3086 message,
3087 )?])
3088 }
3089 Ok(Err(_)) => {
3090 reservation.release_and_disarm();
3091 breaker.record_inconclusive();
3092 Ok(vec![self.route_open_refusal_frame(
3093 ctx,
3094 &frame,
3095 &target_module_id,
3096 "relay_waiter_canceled",
3097 "target_unavailable",
3098 "route.bind relay waiter was canceled before the module responded",
3099 )?])
3100 }
3101 Err(_) => {
3102 reservation.release_and_disarm();
3103 if let Some(opened) = breaker.record_timeout(
3107 self.route_bind_breaker_threshold,
3108 self.route_bind_breaker_cooldown,
3109 ) {
3110 warn!(
3111 module_id = %target_module_id,
3112 consecutive_timeouts = opened.consecutive_timeouts,
3113 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
3114 reopened_after_probe = opened.reopened_after_probe,
3115 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
3116 );
3117 }
3118 tracing::warn!(
3126 module_id = %target_module_id,
3127 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
3128 "route.bind relay timed out: module did not ack within budget"
3129 );
3130 Ok(vec![self.route_open_refusal_frame(
3131 ctx,
3132 &frame,
3133 &target_module_id,
3134 "relay_timed_out",
3135 "module_timeout",
3136 format!(
3137 "module_id '{target_module_id}' did not answer route.bind within {:?}",
3138 route_bind_relay_timeout
3139 ),
3140 )?])
3141 }
3142 }
3143 }
3144
3145 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3146 let response = ClientControlResponse::SupervisorSpawnSnapshot {
3147 snapshot: self.supervisor.spawn_snapshot(),
3148 };
3149 Ok(vec![control_response_body_frame(
3150 &frame,
3151 &response,
3152 "ClientControlResponse::SupervisorSpawnSnapshot",
3153 )?])
3154 }
3155
3156 fn handle_supervisor_spawn_subscribe(
3157 &self,
3158 ctx: &RouteCtx,
3159 frame: Frame,
3160 since: Option<SpawnCursor>,
3161 ) -> Result<Vec<Frame>, RouterError> {
3162 match self.supervisor.subscribe_spawns(
3163 ctx.connection_id,
3164 frame.header.corr,
3165 response_version(&frame),
3166 since,
3167 ctx.egress.clone(),
3168 ) {
3169 Ok(()) => Ok(Vec::new()),
3170 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
3171 Ok(vec![control_error_body_frame(
3172 &frame,
3173 ErrorBody {
3174 code: "spawn_cursor_incarnation_mismatch".to_string(),
3175 message: "spawn cursor belongs to a different daemon incarnation"
3176 .to_string(),
3177 detail: Some(serde_json::json!({
3178 "current_daemon_incarnation": current
3179 })),
3180 },
3181 )?])
3182 }
3183 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
3184 &frame,
3185 ErrorBody {
3186 code: "spawn_cursor_too_old".to_string(),
3187 message: "spawn cursor predates the retained event ring".to_string(),
3188 detail: Some(serde_json::json!({
3189 "oldest_retained_cursor": oldest
3190 })),
3191 },
3192 )?]),
3193 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
3194 0,
3195 frame.header.corr,
3196 format!("failed to open supervisor spawn subscription: {error}"),
3197 )),
3198 }
3199 }
3200
3201 fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3202 let generation = self
3203 .registry
3204 .generation()
3205 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3206 let modules = self
3207 .supervisor
3208 .list()
3209 .into_iter()
3210 .map(|module| {
3211 let status = module.status_for_control("list").map_err(|err| {
3212 RouterError::backend(
3213 0,
3214 frame.header.corr,
3215 format!("failed to read supervisor status: {err}"),
3216 )
3217 })?;
3218 Ok(SupervisorEntry {
3219 module_id: status.module_id,
3220 state: status.state.to_string(),
3221 enabled: status.enabled,
3222 live: status.live,
3223 protocol: status.protocol,
3224 health: status.health.status,
3225 last_probe_ms: status.health.last_probe_ms,
3226 last_exit_code: status.last_exit.as_ref().and_then(|e| e.code),
3227 last_exit_signal: status.last_exit.as_ref().and_then(|e| e.signal),
3228 last_exit_ms: status.last_exit.as_ref().map(|e| e.at_ms),
3229 last_exit_kind: status.last_exit.as_ref().map(|e| e.kind.into()),
3230 restart_count: Some(status.restart_count),
3231 max_restarts: Some(status.max_restarts),
3232 lifetime_restarts: Some(status.lifetime_restarts),
3233 spawn_generation: Some(status.spawn_generation),
3234 restart_window_secs: Some(status.restart_window.as_secs()),
3235 drain_timeout_ms: Some(status.drain_timeout.as_millis() as u64),
3236 restart_backoff_ms: Some(status.restart_backoff.as_millis() as u64),
3237 restart_max_backoff_ms: Some(status.restart_max_backoff.as_millis() as u64),
3238 })
3239 })
3240 .collect::<Result<Vec<_>, RouterError>>()?;
3241 let response = ClientControlResponse::SupervisorList {
3242 generation,
3243 modules,
3244 };
3245 Ok(vec![control_response_body_frame(
3246 &frame,
3247 &response,
3248 "ClientControlResponse::SupervisorList",
3249 )?])
3250 }
3251
3252 fn handle_supervisor_stderr_tail(
3253 &self,
3254 frame: Frame,
3255 module_id: String,
3256 max_lines: Option<u32>,
3257 max_bytes: Option<u32>,
3258 ) -> Result<Vec<Frame>, RouterError> {
3259 let Some(module) = self.supervisor.get(&module_id) else {
3260 return Ok(vec![control_error_frame(
3261 &frame,
3262 "unknown_module",
3263 format!("module_id '{module_id}' is not supervised"),
3264 )?]);
3265 };
3266
3267 let snapshot = module.stderr_tail(
3268 max_lines.map(|value| value as usize),
3269 max_bytes.map(|value| value as usize),
3270 );
3271
3272 let response = ClientControlResponse::SupervisorStderrTail {
3273 module_id,
3274 tail: StderrTail {
3275 capture: match snapshot.capture {
3276 CaptureState::Captured => StderrCaptureState::Captured,
3277 CaptureState::Incomplete { reason } => {
3278 StderrCaptureState::Incomplete { reason }
3279 }
3280 CaptureState::NotCaptured { reason } => {
3281 StderrCaptureState::NotCaptured { reason }
3282 }
3283 },
3284 entries: snapshot
3285 .entries
3286 .into_iter()
3287 .map(|entry| match entry {
3288 TailEntry::Line { text, truncated } => {
3289 StderrTailEntry::Line { text, truncated }
3290 }
3291 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
3292 })
3293 .collect(),
3294 dropped_lines: snapshot.dropped_lines,
3295 },
3296 };
3297 Ok(vec![control_response_body_frame(
3298 &frame,
3299 &response,
3300 "ClientControlResponse::SupervisorStderrTail",
3301 )?])
3302 }
3303
3304 async fn handle_supervisor_terminals(
3305 &self,
3306 frame: Frame,
3307 module_id: String,
3308 ) -> Result<Vec<Frame>, RouterError> {
3309 let Some(module) = self.supervisor.get(&module_id) else {
3310 return Ok(vec![control_error_frame(
3311 &frame,
3312 "unknown_module",
3313 format!("module_id '{module_id}' is not supervised"),
3314 )?]);
3315 };
3316
3317 let terminals = module
3320 .read_durable_terminal_history()
3321 .await
3322 .map_err(|error| {
3323 RouterError::backend(
3324 0,
3325 frame.header.corr,
3326 format!("failed to read terminal history: {error}"),
3327 )
3328 })?;
3329 let response = ClientControlResponse::SupervisorTerminals {
3330 module_id,
3331 terminals,
3332 };
3333 Ok(vec![control_response_body_frame(
3334 &frame,
3335 &response,
3336 "ClientControlResponse::SupervisorTerminals",
3337 )?])
3338 }
3339
3340 fn handle_supervisor_routes(
3341 &self,
3342 frame: Frame,
3343 module_id: Option<String>,
3344 ) -> Result<Vec<Frame>, RouterError> {
3345 let modules = self
3346 .forwarding
3347 .route_census(module_id.as_deref())
3348 .map_err(RouterError::Forwarding)?
3349 .into_iter()
3350 .map(|(module_id, routes)| SupervisorRouteModule {
3351 module_id,
3352 routes: routes
3353 .into_iter()
3354 .map(|route| SupervisorRoute {
3355 consumer: match route.principal {
3356 Principal::Reserved { module_id } => {
3357 SupervisorRouteConsumer::Reserved { module_id }
3358 }
3359 Principal::Direct | Principal::Unverified => {
3360 SupervisorRouteConsumer::Direct {
3361 connection_id: route.goodbye_target.connection_id.get(),
3362 }
3363 }
3364 },
3365 age_ms: Instant::now()
3366 .saturating_duration_since(route.bound_at)
3367 .as_millis()
3368 .try_into()
3369 .unwrap_or(u64::MAX),
3370 draining: route.draining,
3371 drain_reason: route.drain_reason,
3372 })
3373 .collect(),
3374 })
3375 .collect();
3376 let response = ClientControlResponse::SupervisorRoutes { modules };
3377 Ok(vec![control_response_body_frame(
3378 &frame,
3379 &response,
3380 "ClientControlResponse::SupervisorRoutes",
3381 )?])
3382 }
3383
3384 async fn handle_supervisor_provenance(
3385 &self,
3386 frame: Frame,
3387 module_id: Option<String>,
3388 ) -> Result<Vec<Frame>, RouterError> {
3389 let mut selected = if let Some(module_id) = module_id {
3390 let Some(module) = self.supervisor.get(&module_id) else {
3391 return Ok(vec![control_error_frame(
3392 &frame,
3393 "unknown_module",
3394 format!("module_id '{module_id}' is not supervised"),
3395 )?]);
3396 };
3397 vec![module]
3398 } else {
3399 self.supervisor.list()
3400 };
3401
3402 let mut modules = Vec::with_capacity(selected.len());
3403 for module in selected.drain(..) {
3404 let status = module.status().map_err(|err| {
3405 RouterError::backend(
3406 0,
3407 frame.header.corr,
3408 format!("failed to read supervisor status: {err}"),
3409 )
3410 })?;
3411 let module_declared = self
3412 .registry
3413 .get_module(&status.module_id)
3414 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3415 .and_then(|registration| registration.manifest.provenance)
3416 .map(|build| ModuleDeclaredProvenance::Reported { build })
3417 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
3418 #[cfg(test)]
3419 let running_image = match &self.provenance_probe_override {
3420 Some(result) => result.clone(),
3421 None => module.running_image_agreement().await,
3422 };
3423 #[cfg(not(test))]
3424 let running_image = module.running_image_agreement().await;
3425 modules.push(SupervisorModuleProvenance {
3426 module_id: status.module_id,
3427 module_declared,
3428 daemon_observed: SupervisorObservedProcess {
3429 pid: status.pid,
3430 spawned_at_ms: status.spawned_at_ms,
3431 spawned_from: status.spawned_from,
3432 running_image,
3433 },
3434 });
3435 }
3436 let daemon = SupervisorDaemonProvenance {
3437 daemon_build: self.daemon_provenance.build.clone(),
3438 daemon_observed: DaemonObservedProcess {
3439 pid: self.daemon_provenance.pid,
3440 started_at_ms: self
3441 .daemon_provenance
3442 .start_clock
3443 .map(|clock| clock.started_at_ms())
3444 .or(self.daemon_provenance.started_at_ms),
3445 running_image: self
3446 .daemon_provenance
3447 .probe
3448 .observe(
3449 self.daemon_provenance.pid,
3450 self.daemon_provenance.executable_path.as_deref(),
3451 self.daemon_provenance.executable_identity,
3452 self.daemon_provenance.process_start_time,
3453 )
3454 .await,
3455 },
3456 };
3457 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
3458 Ok(vec![control_response_body_frame(
3459 &frame,
3460 &response,
3461 "ClientControlResponse::SupervisorProvenance",
3462 )?])
3463 }
3464
3465 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3466 self.refresh_capability_requirements();
3467 let generation = self
3468 .registry
3469 .generation()
3470 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3471 let modules = self
3472 .supervisor
3473 .list()
3474 .into_iter()
3475 .map(|module| {
3476 let status = module.status_for_control("health").map_err(|err| {
3477 RouterError::backend(
3478 0,
3479 frame.header.corr,
3480 format!("failed to read supervisor health: {err}"),
3481 )
3482 })?;
3483 let module_id = status.module_id;
3484 let capability_detail = self
3485 .capability_evaluator
3486 .required_problem_detail(&module_id);
3487 Ok(SupervisorHealthEntry {
3488 module_id,
3489 status: status.health.status,
3490 detail: append_capability_problem_detail(
3491 status.health.detail,
3492 capability_detail,
3493 ),
3494 metrics: status.health.metrics,
3495 consecutive_failures: status.health.consecutive_failures,
3496 late_answer_count: status.health.late_answer_count,
3497 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
3498 last_action: status.health.last_action,
3499 last_action_ms: status.health.last_action_ms,
3500 last_probe_ms: status.health.last_probe_ms,
3501 })
3502 })
3503 .collect::<Result<Vec<_>, RouterError>>()?;
3504 let response = ClientControlResponse::SupervisorHealth {
3505 generation,
3506 modules,
3507 };
3508 Ok(vec![control_response_body_frame(
3509 &frame,
3510 &response,
3511 "ClientControlResponse::SupervisorHealth",
3512 )?])
3513 }
3514
3515 async fn handle_supervisor_restart(
3516 &self,
3517 frame: Frame,
3518 module_id: String,
3519 drain_timeout_ms: Option<u64>,
3520 ) -> Result<Vec<Frame>, RouterError> {
3521 let operation_lock = self.supervisor.operation_lock();
3522 let _operation_guard = operation_lock.lock().await;
3523 let Some(module) = self.supervisor.get(&module_id) else {
3524 return Ok(vec![control_error_frame(
3525 &frame,
3526 "unknown_module",
3527 format!("module_id '{module_id}' is not supervised"),
3528 )?]);
3529 };
3530
3531 if let Err(err) = module.restart(drain_timeout_ms).await {
3532 let (code, message) = match err {
3533 crate::supervise::SuperviseError::Disabled { .. } => {
3534 ("module_disabled", err.to_string())
3535 }
3536 crate::supervise::SuperviseError::SwapInProgress { .. } => {
3537 ("swap_in_progress", err.to_string())
3538 }
3539 _ => (
3540 "target_unavailable",
3541 format!("failed to restart module_id '{module_id}': {err}"),
3542 ),
3543 };
3544 return Ok(vec![control_error_frame(&frame, code, message)?]);
3545 }
3546
3547 let response = ClientControlResponse::SupervisorAck {
3548 module_id,
3549 applied: true,
3550 };
3551 Ok(vec![control_response_body_frame(
3552 &frame,
3553 &response,
3554 "ClientControlResponse::SupervisorAck",
3555 )?])
3556 }
3557
3558 async fn handle_supervisor_swap(
3562 &self,
3563 frame: Frame,
3564 module_id: String,
3565 ready_timeout_ms: Option<u64>,
3566 ) -> Result<Vec<Frame>, RouterError> {
3567 let module = {
3574 let operation_lock = self.supervisor.operation_lock();
3575 let _operation_guard = operation_lock.lock().await;
3576 self.supervisor.get(&module_id)
3577 };
3578 let Some(module) = module else {
3579 return Ok(vec![control_error_frame(
3580 &frame,
3581 "unknown_module",
3582 format!("module_id '{module_id}' is not supervised"),
3583 )?]);
3584 };
3585
3586 if let Err(err) = module
3587 .swap(ready_timeout_ms.map(Duration::from_millis))
3588 .await
3589 {
3590 use crate::supervise::SuperviseError;
3591 let message = err.to_string();
3592 let error = match err {
3593 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
3594 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
3595 code: "swap_refused".to_string(),
3596 message,
3597 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
3598 },
3599 SuperviseError::SwapFailed {
3600 arm,
3601 candidate_exit,
3602 ..
3603 } => ErrorBody {
3604 code: "swap_failed".to_string(),
3605 message,
3606 detail: Some(serde_json::json!({
3607 "arm": arm.as_str(),
3608 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
3609 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
3610 })),
3611 },
3612 _ => ErrorBody::new(
3613 "target_unavailable",
3614 format!("failed to swap module_id '{module_id}': {message}"),
3615 ),
3616 };
3617 return Ok(vec![control_error_body_frame(&frame, error)?]);
3618 }
3619
3620 let response = ClientControlResponse::SupervisorAck {
3621 module_id,
3622 applied: true,
3623 };
3624 Ok(vec![control_response_body_frame(
3625 &frame,
3626 &response,
3627 "ClientControlResponse::SupervisorAck",
3628 )?])
3629 }
3630
3631 async fn handle_supervisor_reload(
3632 &self,
3633 frame: Frame,
3634 module_id: String,
3635 ) -> Result<Vec<Frame>, RouterError> {
3636 let operation_lock = self.supervisor.operation_lock();
3637 let _operation_guard = operation_lock.lock().await;
3638 let Some(module) = self.supervisor.get(&module_id) else {
3639 return Ok(vec![control_error_frame(
3640 &frame,
3641 "unknown_module",
3642 format!("module_id '{module_id}' is not supervised"),
3643 )?]);
3644 };
3645
3646 if let Err(err) = module.reload().await {
3647 let (code, message) = match err {
3648 crate::supervise::SuperviseError::Disabled { .. } => {
3649 ("module_disabled", err.to_string())
3650 }
3651 crate::supervise::SuperviseError::SwapInProgress { .. } => {
3652 ("swap_in_progress", err.to_string())
3653 }
3654 _ => (
3655 "reload_failed",
3656 format!("failed to reload module_id '{module_id}': {err}"),
3657 ),
3658 };
3659 return Ok(vec![control_error_frame(&frame, code, message)?]);
3660 }
3661
3662 let response = ClientControlResponse::SupervisorAck {
3663 module_id,
3664 applied: true,
3665 };
3666 Ok(vec![control_response_body_frame(
3667 &frame,
3668 &response,
3669 "ClientControlResponse::SupervisorAck",
3670 )?])
3671 }
3672
3673 async fn handle_supervisor_rescan(
3674 &self,
3675 frame: Frame,
3676 preview: bool,
3677 ) -> Result<Vec<Frame>, RouterError> {
3678 let Some(context) = self.rescan.clone() else {
3679 return Ok(vec![control_error_frame(
3680 &frame,
3681 "rescan_unavailable",
3682 "the daemon was not started with a reloadable config path".to_string(),
3683 )?]);
3684 };
3685
3686 let operation_lock = self.supervisor.operation_lock();
3687 let _operation_guard = operation_lock.lock().await;
3688 let loaded = match crate::daemon_config::load(&context.config_path) {
3689 Ok(config) => config,
3690 Err(err) => {
3691 return Ok(vec![control_error_frame(
3692 &frame,
3693 "invalid_daemon_config",
3694 format!("supervisor rescan rejected daemon config: {err}"),
3695 )?])
3696 }
3697 };
3698 let Some(config) = loaded else {
3706 return Ok(vec![control_error_frame(
3707 &frame,
3708 "invalid_daemon_config",
3709 format!(
3710 "daemon config not found at {}; refusing to rescan (an absent config would \
3711 retire every supervised module)",
3712 context.config_path.display()
3713 ),
3714 )?]);
3715 };
3716 let (
3717 configured_port,
3718 storage_config,
3719 admission_facts_carrier_module_id,
3720 admission_facts_targets,
3721 modules,
3722 reserved_capabilities,
3723 ) = (
3724 config.port,
3725 config.storage,
3726 config.admission_facts_carrier_module_id,
3727 config.admission_facts_targets,
3728 config.modules,
3729 config.reserved_capabilities,
3730 );
3731
3732 let mut restart_required = Vec::new();
3741 for section in RestartRequiredSection::ALL {
3742 let changed = match section {
3743 RestartRequiredSection::Port => configured_port != context.configured_port,
3744 RestartRequiredSection::Storage => storage_config != context.storage_config,
3745 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
3746 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
3747 }
3748 RestartRequiredSection::AdmissionFactsTargets => {
3749 admission_facts_targets != context.admission_facts_targets
3750 }
3751 };
3752 if changed {
3753 restart_required.push(section.label().to_string());
3754 }
3755 }
3756 if !restart_required.is_empty() {
3757 warn!(
3758 config_path = %context.config_path.display(),
3759 sections = %restart_required.join(", "),
3760 "daemon config changed outside the modules section; restart the daemon to apply those changes"
3761 );
3762 }
3763
3764 for configured in &modules {
3765 if let Err(err) = validate_spec(&configured.module_spec()) {
3766 return Ok(vec![control_error_frame(
3767 &frame,
3768 "invalid_daemon_config",
3769 format!("supervisor rescan rejected daemon config: {err}"),
3770 )?]);
3771 }
3772 }
3773
3774 let configured_capabilities = modules
3775 .iter()
3776 .map(|module| (module.module_id.clone(), module.enabled))
3777 .collect::<Vec<_>>();
3778 let preview_capability_warnings = if preview {
3779 let (_, registrations) = self.runtime_capability_snapshot()?;
3780 let current_modules = self
3781 .supervisor
3782 .list()
3783 .into_iter()
3784 .map(|module| module.module_id().to_string())
3785 .collect::<BTreeSet<_>>();
3786 let resulting_modules = configured_capabilities.clone();
3787 let removed = current_modules
3788 .into_iter()
3789 .filter(|module_id| {
3790 !resulting_modules
3791 .iter()
3792 .any(|(configured_id, _)| configured_id == module_id)
3793 })
3794 .collect::<Vec<_>>();
3795 self.capability_evaluator.preview_removal_warnings(
3796 resulting_modules,
3797 &removed,
3798 ®istrations,
3799 )
3800 } else {
3801 Vec::new()
3802 };
3803 let result = match self
3804 .reconcile_supervised_modules(&context.supervisor, modules, preview)
3805 .await
3806 {
3807 Ok(result) => result,
3808 Err(message) => {
3809 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
3810 }
3811 };
3812 if !preview {
3813 self.capability_evaluator
3814 .configure(configured_capabilities, reserved_capabilities);
3815 self.capability_evaluator.wake_deadline_loop();
3816 self.refresh_capability_requirements();
3817 }
3818 let mut result = result;
3819 result.restart_required = restart_required;
3820 result.capability_warnings = preview_capability_warnings;
3821 let response = ClientControlResponse::SupervisorRescan { result };
3822 Ok(vec![control_response_body_frame(
3823 &frame,
3824 &response,
3825 "ClientControlResponse::SupervisorRescan",
3826 )?])
3827 }
3828
3829 async fn handle_supervisor_release_reserved(
3830 &self,
3831 frame: Frame,
3832 module_id: String,
3833 ) -> Result<Vec<Frame>, RouterError> {
3834 let Some(context) = self.rescan.clone() else {
3835 return Ok(vec![control_error_frame(
3836 &frame,
3837 "release_unavailable",
3838 "reserved-id release requires a daemon started with a reloadable config path",
3839 )?]);
3840 };
3841 let operation_lock = self.supervisor.operation_lock();
3842 let _operation_guard = operation_lock.lock().await;
3843 let loaded = match crate::daemon_config::load(&context.config_path) {
3844 Ok(Some(config)) => config,
3845 Ok(None) => {
3846 return Ok(vec![control_error_frame(
3847 &frame,
3848 "invalid_daemon_config",
3849 format!(
3850 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
3851 context.config_path.display()
3852 ),
3853 )?])
3854 }
3855 Err(err) => {
3856 return Ok(vec![control_error_frame(
3857 &frame,
3858 "invalid_daemon_config",
3859 format!("unable to verify reserved-id release against daemon config: {err}"),
3860 )?])
3861 }
3862 };
3863 if loaded
3864 .modules
3865 .iter()
3866 .any(|configured| configured.module_id == module_id)
3867 {
3868 return Ok(vec![control_error_frame(
3869 &frame,
3870 "reserved_module_configured",
3871 format!(
3872 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
3873 ),
3874 )?]);
3875 }
3876 if !self.supervisor.release_retained_reserved_gate(&module_id) {
3877 return Ok(vec![control_error_frame(
3878 &frame,
3879 "reserved_gate_not_retained",
3880 format!(
3881 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
3882 ),
3883 )?]);
3884 }
3885
3886 let response = ClientControlResponse::SupervisorAck {
3887 module_id,
3888 applied: true,
3889 };
3890 Ok(vec![control_response_body_frame(
3891 &frame,
3892 &response,
3893 "ClientControlResponse::SupervisorAck",
3894 )?])
3895 }
3896
3897 async fn reconcile_supervised_modules(
3906 &self,
3907 supervisor: &Supervisor,
3908 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
3909 preview: bool,
3910 ) -> Result<SupervisorRescanResult, String> {
3911 let mut current = BTreeMap::new();
3912 for module in self.supervisor.list() {
3913 let (spec, health) = module.configuration().map_err(|err| {
3914 format!(
3915 "failed to read configuration for module_id '{}': {err}",
3916 module.module_id()
3917 )
3918 })?;
3919 let enabled = module
3920 .status()
3921 .map_err(|err| {
3922 format!(
3923 "failed to read status for module_id '{}': {err}",
3924 module.module_id()
3925 )
3926 })?
3927 .enabled;
3928 current.insert(
3929 module.module_id().to_string(),
3930 (module, spec, health, enabled),
3931 );
3932 }
3933 let configured = configured_modules
3934 .into_iter()
3935 .map(|module| (module.module_id.clone(), module))
3936 .collect::<BTreeMap<_, _>>();
3937
3938 let added = configured
3939 .keys()
3940 .filter(|module_id| !current.contains_key(*module_id))
3941 .cloned()
3942 .collect::<Vec<_>>();
3943 let removed = current
3944 .keys()
3945 .filter(|module_id| !configured.contains_key(*module_id))
3946 .cloned()
3947 .collect::<Vec<_>>();
3948 let mut changed_pending_reload = Vec::new();
3949 let mut configuration_changes = BTreeSet::new();
3950 let mut enabled_changes = BTreeSet::new();
3951 let mut unchanged = 0_u32;
3952
3953 for (module_id, configured_module) in &configured {
3954 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
3955 else {
3956 continue;
3957 };
3958 let configuration_changed = *current_spec != configured_module.module_spec()
3959 || *current_health != configured_module.health;
3960 let enabled_changed = *current_enabled != configured_module.enabled;
3961 if configuration_changed {
3962 configuration_changes.insert(module_id.clone());
3963 changed_pending_reload.push(module_id.clone());
3964 }
3965 if enabled_changed {
3966 enabled_changes.insert(module_id.clone());
3967 }
3968 if !configuration_changed && !enabled_changed {
3969 unchanged = unchanged.saturating_add(1);
3970 }
3971 }
3972
3973 if preview {
3979 return Ok(SupervisorRescanResult {
3980 added,
3981 removed,
3982 changed_pending_reload,
3983 enabled_changes: enabled_changes.iter().cloned().collect(),
3984 unchanged,
3985 preview: true,
3986 restart_required: Vec::new(),
3990 capability_warnings: Vec::new(),
3991 });
3992 }
3993
3994 for module_id in &removed {
3995 let module = ¤t
3996 .get(module_id)
3997 .expect("removed module came from current supervisor state")
3998 .0;
3999 module.retire().await.map_err(|err| {
4000 format!("failed to retire module_id '{module_id}' during rescan: {err}")
4001 })?;
4002 self.supervisor.record_rescan_removal(module_id);
4036 self.supervisor.retire(module_id);
4037 }
4038
4039 for module_id in configured.keys() {
4040 let Some((module, _, _, _)) = current.get(module_id) else {
4041 continue;
4042 };
4043 let configured_module = configured
4044 .get(module_id)
4045 .expect("configured module id came from configured map");
4046 if configuration_changes.contains(module_id) {
4047 module
4048 .update_configuration(
4049 configured_module.module_spec(),
4050 configured_module.health,
4051 configured_module.drain_timeout_ms,
4052 )
4053 .await
4054 .map_err(|err| {
4055 format!(
4056 "failed to update module_id '{module_id}' configuration during rescan: {err}"
4057 )
4058 })?;
4059 }
4060 if enabled_changes.contains(module_id) {
4061 module
4062 .set_enabled(configured_module.enabled)
4063 .await
4064 .map_err(|err| {
4065 format!(
4066 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
4067 configured_module.enabled
4068 )
4069 })?;
4070 }
4071 }
4072
4073 for module_id in &added {
4074 let configured_module = configured
4075 .get(module_id)
4076 .expect("added module id came from configured map");
4077 supervisor
4078 .supervise_configured_with_health(
4079 configured_module.module_spec(),
4080 configured_module.enabled,
4081 configured_module.health,
4082 configured_module.drain_timeout_ms,
4083 configured_module.restart,
4084 )
4085 .map_err(|err| {
4086 format!("failed to add module_id '{module_id}' during rescan: {err}")
4087 })?;
4088 }
4089
4090 Ok(SupervisorRescanResult {
4091 added,
4092 removed,
4093 changed_pending_reload,
4094 enabled_changes: enabled_changes.iter().cloned().collect(),
4095 unchanged,
4096 preview: false,
4097 restart_required: Vec::new(),
4100 capability_warnings: Vec::new(),
4101 })
4102 }
4103
4104 async fn handle_supervisor_set_enabled(
4105 &self,
4106 frame: Frame,
4107 module_id: String,
4108 enabled: bool,
4109 ) -> Result<Vec<Frame>, RouterError> {
4110 let operation_lock = self.supervisor.operation_lock();
4111 let _operation_guard = operation_lock.lock().await;
4112 let Some(module) = self.supervisor.get(&module_id) else {
4113 return Ok(vec![control_error_frame(
4114 &frame,
4115 "unknown_module",
4116 format!("module_id '{module_id}' is not supervised"),
4117 )?]);
4118 };
4119
4120 let applied = match module.set_enabled(enabled).await {
4121 Ok(applied) => applied,
4122 Err(err) => {
4123 return Ok(vec![control_error_frame(
4124 &frame,
4125 "target_unavailable",
4126 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
4127 )?])
4128 }
4129 };
4130
4131 self.capability_evaluator.wake_deadline_loop();
4132 self.refresh_capability_requirements();
4133 let response = ClientControlResponse::SupervisorAck { module_id, applied };
4134 Ok(vec![control_response_body_frame(
4135 &frame,
4136 &response,
4137 "ClientControlResponse::SupervisorAck",
4138 )?])
4139 }
4140
4141 async fn handle_supervisor_health_probe(
4142 &self,
4143 frame: Frame,
4144 module_id: String,
4145 ) -> Result<Vec<Frame>, RouterError> {
4146 self.refresh_capability_requirements();
4147 let Some(registration) = self
4148 .registry
4149 .get_module(&module_id)
4150 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4151 else {
4152 return Ok(vec![control_error_frame(
4153 &frame,
4154 "unknown_module",
4155 format!("module_id '{module_id}' is not registered"),
4156 )?]);
4157 };
4158
4159 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
4172 {
4173 return Ok(vec![control_error_frame(
4174 &frame,
4175 "health_not_advertised",
4176 format!("module_id '{module_id}' did not advertise health.check"),
4177 )?]);
4178 }
4179
4180 let deadline = Instant::now() + self.health_probe_timeout;
4181 let pending = match self.forwarding.begin_module_control_rpc_for(
4182 &module_id,
4183 MODULE_CONTROL_OP_HEALTH_CHECK,
4184 deadline,
4185 ) {
4186 Ok(pending) => pending,
4187 Err(err) => {
4188 return Ok(vec![control_error_frame(
4189 &frame,
4190 forwarding_error_code(&err),
4191 err.to_string(),
4192 )?])
4193 }
4194 };
4195
4196 let PendingModuleControlRpc {
4197 endpoint,
4198 module_sink,
4199 negotiated_ver,
4200 corr: probe_corr,
4201 receiver,
4202 } = pending;
4203 let mut guard =
4204 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
4205 let probe_body =
4206 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
4207 RouterError::backend(
4208 0,
4209 frame.header.corr,
4210 format!("failed to encode health.check request: {err}"),
4211 )
4212 })?;
4213 let probe_frame = Frame::build_with_version(
4214 negotiated_ver,
4215 FrameType::Request,
4216 control_flags(),
4217 0,
4218 0,
4219 probe_corr,
4220 probe_body,
4221 )
4222 .map_err(RouterError::FrameBuild)?;
4223
4224 if let Err(err) = module_sink.send(probe_frame).await {
4225 return Ok(vec![control_error_frame(
4226 &frame,
4227 "target_unavailable",
4228 err.to_string(),
4229 )?]);
4230 }
4231
4232 match timeout_at(deadline, receiver).await {
4233 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
4234 guard.disarm();
4235 let Some(report) = response.health_report() else {
4236 return Ok(vec![control_error_frame(
4237 &frame,
4238 "invalid_control_body",
4239 "health.check RPC returned a non-health response",
4240 )?]);
4241 };
4242 let HealthReport {
4247 status,
4248 detail,
4249 metrics,
4250 } = report;
4251 let capability_detail = self
4252 .capability_evaluator
4253 .required_problem_detail(&module_id);
4254 let response = ClientControlResponse::SupervisorHealthProbe {
4255 module_id,
4256 status,
4257 detail: append_capability_problem_detail(detail, capability_detail),
4258 metrics,
4259 };
4260 Ok(vec![control_response_body_frame(
4261 &frame,
4262 &response,
4263 "ClientControlResponse::SupervisorHealthProbe",
4264 )?])
4265 }
4266 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
4267 guard.disarm();
4268 Ok(vec![control_error_body_frame(&frame, body)?])
4269 }
4270 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
4271 guard.disarm();
4272 Ok(vec![control_error_frame(
4273 &frame,
4274 "target_unavailable",
4275 message,
4276 )?])
4277 }
4278 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
4279 guard.disarm();
4280 Ok(vec![control_error_frame(
4281 &frame,
4282 "invalid_control_body",
4283 message,
4284 )?])
4285 }
4286 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
4287 guard.disarm();
4288 Ok(vec![control_error_frame(
4289 &frame,
4290 "invalid_control_body",
4291 format!("expected module-control op '{expected}', got '{actual}'"),
4292 )?])
4293 }
4294 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
4295 guard.disarm();
4296 Ok(vec![control_error_frame(
4297 &frame,
4298 "module_timeout",
4299 format!(
4300 "module_id '{module_id}' answered health.check after {:?}",
4301 self.health_probe_timeout
4302 ),
4303 )?])
4304 }
4305 Ok(Err(_)) => Ok(vec![control_error_frame(
4306 &frame,
4307 "target_unavailable",
4308 "health.check waiter was canceled before the module responded",
4309 )?]),
4310 Err(_) => Ok(vec![control_error_frame(
4311 &frame,
4312 "module_timeout",
4313 format!(
4314 "module_id '{module_id}' did not answer health.check within {:?}",
4315 self.health_probe_timeout
4316 ),
4317 )?]),
4318 }
4319 }
4320
4321 fn supervisor_status(
4322 &self,
4323 module_id: &str,
4324 corr: u64,
4325 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
4326 self.supervisor
4327 .get(module_id)
4328 .map(|module| {
4329 let warming = module.is_warming_for_control("status").map_err(|err| {
4330 RouterError::backend(
4331 0,
4332 corr,
4333 format!(
4334 "failed to read supervisor warming state for module_id '{module_id}': {err}"
4335 ),
4336 )
4337 })?;
4338 module.status_for_control("status").map_err(|err| {
4339 RouterError::backend(
4340 0,
4341 corr,
4342 format!(
4343 "failed to read supervisor status for module_id '{module_id}': {err}"
4344 ),
4345 )
4346 }).map(|status| (status, warming))
4347 })
4348 .transpose()
4349 }
4350
4351 fn guard_module_control_op(
4352 &self,
4353 frame: &Frame,
4354 module_id: &str,
4355 op: &str,
4356 ) -> Result<Option<Frame>, RouterError> {
4357 if self.module_grants_op(module_id, op, frame.header.corr)? {
4358 return Ok(None);
4359 }
4360
4361 Ok(Some(control_error_frame(
4362 frame,
4363 "op_not_allowed",
4364 format!("module_id '{module_id}' did not grant control op '{op}'"),
4365 )?))
4366 }
4367
4368 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
4369 let Some(registration) = self
4370 .registry
4371 .get_module(module_id)
4372 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
4373 else {
4374 return Ok(false);
4375 };
4376 Ok(module_registration_grants_op(®istration.control_ops, op))
4377 }
4378
4379 fn handle_status_update(
4380 &self,
4381 endpoint: ModuleEndpointId,
4382 frame: Frame,
4383 ) -> Result<Vec<Frame>, RouterError> {
4384 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
4385 Ok(update) => update,
4386 Err(err) => {
4387 if is_known_module_push_op(&frame.body) {
4392 return Ok(vec![control_error_frame(
4393 &frame,
4394 "invalid_control_body",
4395 format!("malformed module control push body: {err}"),
4396 )?]);
4397 }
4398 return Ok(Vec::new());
4399 }
4400 };
4401
4402 match update {
4403 ModuleControlPush::RouteStatus {
4404 route_channel,
4405 route_epoch,
4406 status,
4407 } => {
4408 self.forwarding
4409 .cache_status(endpoint, route_channel, route_epoch, status)
4410 .map_err(RouterError::Forwarding)?;
4411 }
4412 }
4413 Ok(Vec::new())
4414 }
4415
4416 fn handle_route_poll(
4417 &self,
4418 ctx: &RouteCtx,
4419 frame: Frame,
4420 route_channel: u16,
4421 route_epoch: u32,
4422 kind: PollKind,
4423 ) -> Result<Vec<Frame>, RouterError> {
4424 let snapshot = self
4425 .forwarding
4426 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
4427 .map_err(RouterError::Forwarding)?;
4428 let response = match (kind, snapshot) {
4429 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
4430 ClientControlResponse::RoutePoll {
4431 route_channel,
4432 route_epoch,
4433 status,
4434 live: None,
4435 }
4436 }
4437 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
4438 route_channel,
4439 route_epoch,
4440 status: None,
4441 live: None,
4442 },
4443 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
4444 let live = self
4470 .process_liveness
4471 .as_ref()
4472 .and_then(|source| source.process_live(&module_id))
4473 .unwrap_or(true);
4474 ClientControlResponse::RoutePoll {
4475 route_channel,
4476 route_epoch,
4477 status: None,
4478 live: Some(live),
4479 }
4480 }
4481 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
4482 route_channel,
4483 route_epoch,
4484 status: None,
4485 live: Some(false),
4486 },
4487 };
4488
4489 Ok(vec![control_response_body_frame(
4490 &frame,
4491 &response,
4492 "ClientControlResponse::RoutePoll",
4493 )?])
4494 }
4495
4496 pub(crate) fn observe_module_control_completion(
4497 &self,
4498 completion: ModuleControlRpcCompletion,
4499 ) -> bool {
4500 match completion {
4501 ModuleControlRpcCompletion::Unknown => false,
4502 ModuleControlRpcCompletion::Settled => true,
4503 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
4504 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
4505 info!(
4506 module_id = %module_id,
4507 latency_ms,
4508 "late health.check answer proves the module is alive"
4509 );
4510 match self
4511 .supervisor
4512 .record_late_health_answer(&module_id, latency_ms)
4513 {
4514 Ok(true) => {}
4515 Ok(false) => debug!(
4516 module_id = %module_id,
4517 latency_ms,
4518 "late health.check answer has no active supervisor snapshot"
4519 ),
4520 Err(err) => warn!(
4521 module_id = %module_id,
4522 latency_ms,
4523 error = %err,
4524 "failed to record late health.check answer"
4525 ),
4526 }
4527 true
4528 }
4529 }
4530 }
4531
4532 fn refuse_to_end_module_connection_for_a_client(
4553 &self,
4554 module_connection_id: ConnectionId,
4555 corr: u64,
4556 err: ForwardingError,
4557 ) -> Result<(), RouterError> {
4558 if let ForwardingError::ConnectionClosing { connection_id } = err {
4559 if connection_id != module_connection_id {
4560 warn!(
4561 module_connection_id = module_connection_id.get(),
4562 client_connection_id = connection_id.get(),
4563 corr,
4564 "dropping a route.bind response for a closing client; the module connection keeps serving"
4565 );
4566 return Ok(());
4567 }
4568 }
4569 Err(RouterError::Forwarding(err))
4570 }
4571
4572 fn handle_module_relay_response(
4573 &self,
4574 connection_id: ConnectionId,
4575 frame: Frame,
4576 ) -> Result<Vec<Frame>, RouterError> {
4577 let mut secondary_error = None;
4578 let outcome = match frame.header.ty {
4579 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
4580 Ok(probe) if probe.op == "route.bind" => {
4581 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
4582 Ok(ModuleControlResponse::RouteBindAck {}) => {
4583 RouteBindRelayOutcome::Accepted
4584 }
4585 Ok(other) => {
4586 let message =
4587 format!("route.bind response carried unexpected body: {other:?}");
4588 secondary_error = Some(control_error_frame(
4589 &frame,
4590 "invalid_control_body",
4591 message.clone(),
4592 )?);
4593 RouteBindRelayOutcome::ModuleGone(message)
4594 }
4595 Err(err) => {
4596 let message = format!("malformed route.bind response body: {err}");
4597 secondary_error = Some(control_error_frame(
4598 &frame,
4599 "invalid_control_body",
4600 message.clone(),
4601 )?);
4602 RouteBindRelayOutcome::ModuleGone(message)
4603 }
4604 }
4605 }
4606 Ok(probe) => {
4607 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
4608 {
4609 Ok(response) => ModuleControlRpcOutcome::Response(response),
4610 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
4611 "malformed {} response body: {err}",
4612 probe.op
4613 )),
4614 };
4615 let completion = self
4616 .forwarding
4617 .complete_module_control_rpc(
4618 connection_id,
4619 frame.header.corr,
4620 Some(&probe.op),
4621 outcome,
4622 )
4623 .map_err(RouterError::Forwarding)?;
4624 if !self.observe_module_control_completion(completion) {
4625 debug!(
4626 connection_id = connection_id.get(),
4627 corr = frame.header.corr,
4628 op = %probe.op,
4629 "dropping late or unknown module-control RPC response"
4630 );
4631 }
4632 return Ok(Vec::new());
4633 }
4634 Err(err) => {
4635 if let Some(expected_op) = self
4636 .forwarding
4637 .pending_module_control_op(connection_id, frame.header.corr)
4638 .map_err(RouterError::Forwarding)?
4639 {
4640 let completion = self
4641 .forwarding
4642 .complete_module_control_rpc(
4643 connection_id,
4644 frame.header.corr,
4645 None,
4646 ModuleControlRpcOutcome::MalformedResponse(format!(
4647 "malformed {expected_op} response body: {err}"
4648 )),
4649 )
4650 .map_err(RouterError::Forwarding)?;
4651 if !self.observe_module_control_completion(completion) {
4652 debug!(
4653 connection_id = connection_id.get(),
4654 corr = frame.header.corr,
4655 "dropping late malformed module-control RPC response"
4656 );
4657 }
4658 return Ok(Vec::new());
4659 }
4660 let message = format!("malformed route.bind response body: {err}");
4661 secondary_error = Some(control_error_frame(
4662 &frame,
4663 "invalid_control_body",
4664 message.clone(),
4665 )?);
4666 RouteBindRelayOutcome::ModuleGone(message)
4667 }
4668 },
4669 FrameType::Error => {
4670 if self
4671 .forwarding
4672 .pending_module_control_op(connection_id, frame.header.corr)
4673 .map_err(RouterError::Forwarding)?
4674 .is_some()
4675 {
4676 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
4677 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
4678 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
4679 "malformed module-control ERROR body: {err}"
4680 )),
4681 };
4682 let completion = self
4683 .forwarding
4684 .complete_module_control_rpc(
4685 connection_id,
4686 frame.header.corr,
4687 None,
4688 outcome,
4689 )
4690 .map_err(RouterError::Forwarding)?;
4691 if !self.observe_module_control_completion(completion) {
4692 debug!(
4693 connection_id = connection_id.get(),
4694 corr = frame.header.corr,
4695 "dropping late or unknown module-control RPC error"
4696 );
4697 }
4698 return Ok(Vec::new());
4699 }
4700 match serde_json::from_slice::<ErrorBody>(&frame.body) {
4701 Ok(body) => RouteBindRelayOutcome::Rejected(body),
4702 Err(err) => {
4703 let message = format!("malformed route.bind ERROR body: {err}");
4704 secondary_error = Some(control_error_frame(
4705 &frame,
4706 "invalid_control_body",
4707 message.clone(),
4708 )?);
4709 RouteBindRelayOutcome::ModuleGone(message)
4710 }
4711 }
4712 }
4713 ty => {
4714 return Ok(vec![control_error_frame(
4715 &frame,
4716 "unsupported_control_frame",
4717 format!("unsupported module channel-0 frame {ty:?}"),
4718 )?])
4719 }
4720 };
4721
4722 let settled =
4723 self.forwarding
4724 .complete_pending_relay(connection_id, frame.header.corr, outcome);
4725 let completion = match settled {
4726 Ok(completion) => completion,
4727 Err(err) => {
4728 self.refuse_to_end_module_connection_for_a_client(
4729 connection_id,
4730 frame.header.corr,
4731 err,
4732 )?;
4733 return Ok(secondary_error.into_iter().collect());
4734 }
4735 };
4736 if let Some(target) = completion.abandoned.as_ref() {
4737 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
4738 }
4739 if !completion.settled {
4740 debug!(
4741 connection_id = connection_id.get(),
4742 corr = frame.header.corr,
4743 frame_type = ?frame.header.ty,
4744 "dropping late or unknown route.bind relay response"
4745 );
4746 }
4747 Ok(secondary_error.into_iter().collect())
4748 }
4749
4750 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
4751 debug!(connection_id = connection_id.get(), "handling GOODBYE");
4752 let registrations = self
4753 .deregister_connection(connection_id)
4754 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
4755 let released_routes = self
4756 .forwarding
4757 .cleanup_connection(connection_id)
4758 .map_err(RouterError::Forwarding)?;
4759 self.emit_route_goodbyes(released_routes);
4760 if !registrations.is_empty() {
4762 crate::supervise::notify_registration_release();
4763 }
4764 Ok(Vec::new())
4765 }
4766}
4767
4768impl Default for ControlHandler {
4769 fn default() -> Self {
4770 Self::new(Arc::new(Registry::default()))
4771 }
4772}
4773
4774impl crate::supervise::SwapPromotionObserver for ControlHandler {
4775 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
4776 self.apply_registration_capabilities(registration);
4777 }
4778}
4779
4780fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
4781 CapabilityRequirementStatus {
4782 consumer: status.consumer,
4783 capability: status.capability,
4784 need: match status.need {
4785 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
4786 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
4787 },
4788 verdict: status.verdict.as_str().to_string(),
4789 episode_seq: status.episode_seq,
4790 config_satisfiable: status.config_satisfiable,
4791 runtime_available: status.runtime_available,
4792 detail: status.detail,
4793 }
4794}
4795
4796fn append_capability_problem_detail(
4797 detail: Option<String>,
4798 capability_detail: Option<String>,
4799) -> Option<String> {
4800 match (detail, capability_detail) {
4801 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
4802 (Some(detail), None) => Some(detail),
4803 (None, Some(capability_detail)) => Some(capability_detail),
4804 (None, None) => None,
4805 }
4806}
4807
4808fn subc_ops() -> Vec<String> {
4809 SUBC_CONTROL_OPS
4810 .iter()
4811 .map(|op| (*op).to_string())
4812 .collect()
4813}
4814
4815fn module_subc_ops() -> Vec<String> {
4816 SUBC_CONTROL_OPS
4817 .iter()
4818 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
4819 .map(|op| (*op).to_string())
4820 .collect()
4821}
4822
4823#[cfg(test)]
4824fn module_baseline_control_ops() -> Vec<String> {
4825 MODULE_BASELINE_CONTROL_OPS
4826 .iter()
4827 .map(|op| (*op).to_string())
4828 .collect()
4829}
4830
4831fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
4832 let mut seen = HashSet::new();
4833 let mut effective = Vec::new();
4834 for op in MODULE_BASELINE_CONTROL_OPS {
4835 if seen.insert((*op).to_string()) {
4836 effective.push((*op).to_string());
4837 }
4838 }
4839 for op in declared.unwrap_or_default() {
4840 if seen.insert(op.clone()) {
4841 effective.push(op);
4842 }
4843 }
4844 effective
4845}
4846
4847fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
4848 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
4849}
4850
4851fn target_module_id(target: &RouteTarget) -> &str {
4852 match target {
4853 RouteTarget::ToolProvider { module_id }
4854 | RouteTarget::ManagementSurface { module_id }
4855 | RouteTarget::InternalService { module_id, .. } => module_id,
4856 }
4857}
4858
4859fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
4860 roles.iter().any(|role| match (target, role) {
4861 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
4862 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
4863 (
4864 RouteTarget::InternalService { service_id, .. },
4865 ProviderRole::InternalService {
4866 service_id: provided,
4867 ..
4868 },
4869 ) => service_id == provided,
4870 _ => false,
4871 })
4872}
4873
4874fn is_routable_role(role: &ProviderRole) -> bool {
4875 matches!(
4876 role,
4877 ProviderRole::ToolProvider { .. }
4878 | ProviderRole::ManagementSurface { .. }
4879 | ProviderRole::InternalService { .. }
4880 )
4881}
4882
4883#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4884enum ControlRequestBodyError {
4885 UnknownOp,
4886 InvalidBody,
4887}
4888
4889#[derive(Debug, Deserialize)]
4890struct ControlOpProbe {
4891 op: String,
4892}
4893
4894const MODULE_PUSH_OPS: &[&str] = &["route.status"];
4897
4898fn is_known_module_push_op(body: &[u8]) -> bool {
4899 serde_json::from_slice::<ControlOpProbe>(body)
4900 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
4901 .unwrap_or(false)
4902}
4903
4904fn is_known_module_request_op(body: &[u8]) -> bool {
4905 serde_json::from_slice::<ControlOpProbe>(body)
4906 .map(|probe| MODULE_TO_SUBC_CONTROL_OPS.contains(&probe.op.as_str()))
4907 .unwrap_or(false)
4908}
4909
4910fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
4911 debug!(
4912 op = %op,
4913 connection_id = connection_id.get(),
4914 corr,
4915 "control dispatch"
4916 );
4917}
4918
4919fn log_slow_control_dispatch(
4920 dispatch_started_at: Option<StdInstant>,
4921 op: &'static str,
4922 connection_id: ConnectionId,
4923 corr: u64,
4924) {
4925 let Some(dispatch_started_at) = dispatch_started_at else {
4926 return;
4927 };
4928 let elapsed = dispatch_started_at.elapsed();
4929 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
4930 warn!(
4931 op = %op,
4932 connection_id = connection_id.get(),
4933 corr,
4934 elapsed_ms = elapsed.as_millis() as u64,
4935 "slow control dispatch"
4936 );
4937 }
4938}
4939
4940fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
4941 match request {
4942 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
4943 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
4944 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
4945 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
4946 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
4947 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
4948 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
4949 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
4950 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
4951 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
4952 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
4953 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
4954 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
4955 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
4956 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
4957 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
4958 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
4959 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
4960 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
4961 }
4962}
4963
4964fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
4965 match request {
4966 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
4967 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
4968 }
4969}
4970
4971fn parse_client_control_request(
4972 body: &[u8],
4973) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
4974 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
4975 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
4976 Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
4977 ControlRequestBodyError::InvalidBody
4978 }
4979 Ok(_) => ControlRequestBodyError::UnknownOp,
4980 Err(_) => ControlRequestBodyError::InvalidBody,
4981 };
4982 (err, classification)
4983 })
4984}
4985
4986fn parse_module_control_request_from_module(
4987 body: &[u8],
4988) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
4989 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
4990 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
4991 Ok(probe) if MODULE_TO_SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
4992 ControlRequestBodyError::InvalidBody
4993 }
4994 Ok(_) => ControlRequestBodyError::UnknownOp,
4995 Err(_) => ControlRequestBodyError::InvalidBody,
4996 };
4997 (err, classification)
4998 })
4999}
5000
5001#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
5002enum ProviderRoleKind {
5003 ToolProvider,
5004 PipelineStage,
5005 ManagementSurface,
5006 InternalService,
5007}
5008
5009fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
5010 match role {
5011 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
5012 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
5013 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
5014 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
5015 }
5016}
5017
5018fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
5019 roles.iter().map(provider_role_kind).collect()
5020}
5021
5022fn capability_census_trigger(
5026 old: Option<&CapabilityDeclarations>,
5027 new: Option<&CapabilityDeclarations>,
5028) -> bool {
5029 let old_provides = old
5030 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
5031 .unwrap_or_default();
5032 let old_denies = old
5033 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
5034 .unwrap_or_default();
5035 let new = new.cloned().unwrap_or(CapabilityDeclarations {
5036 provides: Vec::new(),
5037 requires: Vec::new(),
5038 must_never_reach: Vec::new(),
5039 });
5040
5041 new.provides
5042 .iter()
5043 .any(|capability| !old_provides.contains(capability))
5044 || new
5045 .must_never_reach
5046 .iter()
5047 .any(|capability| !old_denies.contains(capability))
5048}
5049
5050fn denied_capability<'a>(
5053 opening_manifest: &'a ModuleManifest,
5054 target_manifest: &ModuleManifest,
5055) -> Option<&'a str> {
5056 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
5057 let target_capabilities = target_manifest.capabilities.as_ref()?;
5058 opening_capabilities
5059 .must_never_reach
5060 .iter()
5061 .find(|denied| {
5062 target_capabilities
5063 .provides
5064 .iter()
5065 .any(|provided| provided == *denied)
5066 })
5067 .map(String::as_str)
5068}
5069
5070fn catalog_update_frozen_field_message(
5071 registered: &ModuleManifest,
5072 provides: &[ProviderRole],
5073) -> Option<String> {
5074 let old_has_provides = !registered.provides.is_empty();
5075 let new_has_provides = !provides.is_empty();
5076 if old_has_provides != new_has_provides {
5077 return Some(format!(
5078 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
5079 registered.module_id
5080 ));
5081 }
5082
5083 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
5084 return Some(format!(
5085 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
5086 registered.module_id
5087 ));
5088 }
5089
5090 let registered_concurrency = manifest_concurrency(registered);
5091 let mut candidate = registered.clone();
5092 candidate.provides = provides.to_vec();
5093 let candidate_concurrency = manifest_concurrency(&candidate);
5094 if candidate_concurrency != registered_concurrency {
5095 return Some(format!(
5096 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
5097 registered.module_id, registered_concurrency, candidate_concurrency
5098 ));
5099 }
5100
5101 None
5104}
5105
5106fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
5107 manifest.provides.iter().any(is_routable_role)
5108}
5109
5110fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
5116 manifest
5117 .provides
5118 .iter()
5119 .find_map(|provider| match provider {
5120 ProviderRole::ToolProvider { concurrency, .. }
5121 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
5122 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
5123 })
5124 .unwrap_or(Concurrency::ModuleManaged)
5125}
5126
5127fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
5134 let has_management_surface = manifest
5135 .provides
5136 .iter()
5137 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
5138 if !has_management_surface {
5139 return false;
5140 }
5141 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
5142 return false;
5143 };
5144 let Some(provides) = raw
5145 .get("manifest")
5146 .and_then(|manifest| manifest.get("provides"))
5147 .and_then(serde_json::Value::as_array)
5148 else {
5149 return false;
5150 };
5151 provides.iter().any(|role| {
5156 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
5157 && role.get("concurrency").is_none()
5158 })
5159}
5160
5161fn negotiate_version(peer_version: u8) -> Result<u8, String> {
5162 if peer_version != PROTOCOL_VERSION {
5163 return Err(format!(
5164 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
5165 ));
5166 }
5167 Ok(PROTOCOL_VERSION)
5168}
5169
5170fn pong(frame: &Frame) -> Result<Frame, RouterError> {
5171 Frame::build_with_version(
5172 response_version(frame),
5173 FrameType::Pong,
5174 frame.header.flags,
5175 0,
5176 0,
5177 frame.header.corr,
5178 Vec::new(),
5179 )
5180 .map_err(RouterError::FrameBuild)
5181}
5182
5183fn control_error_frame(
5184 frame: &Frame,
5185 code: &'static str,
5186 message: impl Into<String>,
5187) -> Result<Frame, RouterError> {
5188 control_error_body_frame(
5189 frame,
5190 ErrorBody {
5191 code: code.to_string(),
5192 message: message.into(),
5193 detail: None,
5194 },
5195 )
5196}
5197
5198fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
5199 let body = serde_json::to_vec(&error).map_err(|err| {
5200 RouterError::backend(
5201 0,
5202 frame.header.corr,
5203 format!("failed to encode control ERROR: {err}"),
5204 )
5205 })?;
5206
5207 Frame::build_with_version(
5208 response_version(frame),
5209 FrameType::Error,
5210 control_flags(),
5211 0,
5212 0,
5213 frame.header.corr,
5214 body,
5215 )
5216 .map_err(RouterError::FrameBuild)
5217}
5218
5219fn control_response_body_frame<T: Serialize>(
5220 frame: &Frame,
5221 reply: &T,
5222 label: &'static str,
5223) -> Result<Frame, RouterError> {
5224 let body = serde_json::to_vec(reply).map_err(|err| {
5225 RouterError::backend(
5226 0,
5227 frame.header.corr,
5228 format!("failed to encode {label}: {err}"),
5229 )
5230 })?;
5231
5232 Frame::build_with_version(
5233 response_version(frame),
5234 FrameType::Response,
5235 control_flags(),
5236 0,
5237 0,
5238 frame.header.corr,
5239 body,
5240 )
5241 .map_err(RouterError::FrameBuild)
5242}
5243
5244fn forwarding_error_code(err: &ForwardingError) -> &'static str {
5274 match err {
5275 ForwardingError::NoModuleConnection => "target_unavailable",
5276 ForwardingError::ModuleReloading { .. } => "module_reloading",
5277 ForwardingError::ClientRouteChannelExhausted { .. }
5278 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
5279 ForwardingError::StaleModuleEndpoint
5280 | ForwardingError::UnknownReservation { .. }
5281 | ForwardingError::ConnectionClosing { .. }
5282 | ForwardingError::ClientEgressClosed { .. }
5283 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
5284 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
5287 ForwardingError::RelayCorrelationExhausted
5288 | ForwardingError::RouteOpenBuild(_)
5289 | ForwardingError::Poisoned => "forwarding_error",
5290 }
5291}
5292
5293fn response_version(frame: &Frame) -> u8 {
5294 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
5295 frame.header.ver
5296 } else {
5297 PROTOCOL_VERSION
5298 }
5299}
5300
5301fn control_flags() -> Flags {
5302 Flags::new(false, Priority::Passive, false)
5303}
5304
5305fn send_goodbye_target_best_effort(
5310 counters: &DaemonCounters,
5311 target: &GoodbyeTarget,
5312 context: &'static str,
5313) {
5314 let Ok(frame) = Frame::build_with_version(
5315 target.negotiated_ver,
5316 FrameType::Goodbye,
5317 control_flags(),
5318 target.channel,
5319 target.epoch,
5320 0,
5321 Vec::new(),
5322 ) else {
5323 return;
5324 };
5325 crate::forwarding::send_module_route_goodbye(
5326 counters,
5327 &target.sink,
5328 frame,
5329 target.module_id.as_deref(),
5330 context,
5331 );
5332}
5333
5334pub(crate) fn send_route_control_pushes(
5335 forwarding: &ForwardingTable,
5336 routes: Vec<EndpointRoute>,
5337 push: ClientControlPush,
5338) {
5339 let body = match serde_json::to_vec(&push) {
5340 Ok(body) => body,
5341 Err(err) => {
5342 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
5343 return;
5344 }
5345 };
5346 let mut targets = Vec::new();
5347 for route in routes {
5348 let target = route.goodbye_target;
5349 if let Some(existing) = targets
5350 .iter()
5351 .find(|existing: &&GoodbyeTarget| existing.connection_id == target.connection_id)
5352 {
5353 debug_assert_eq!(
5354 existing.negotiated_ver, target.negotiated_ver,
5355 "one connection cannot negotiate multiple frame versions"
5356 );
5357 continue;
5358 }
5359 targets.push(target);
5360 }
5361 for target in targets {
5362 let frame = match Frame::build_with_version(
5363 target.negotiated_ver,
5364 FrameType::Push,
5365 control_flags(),
5366 0,
5367 0,
5368 0,
5369 body.clone(),
5370 ) {
5371 Ok(frame) => frame,
5372 Err(err) => {
5373 warn!(
5374 route_channel = target.channel,
5375 error = %err,
5376 "failed to build route lifecycle control PUSH frame"
5377 );
5378 continue;
5379 }
5380 };
5381 if let Err(err) = target.sink.try_send(frame) {
5382 if target.close_on_delivery_failure() {
5383 warn!(
5384 target_connection_id = target.connection_id.get(),
5385 route_channel = target.channel,
5386 error = %err,
5387 "route lifecycle control PUSH was not delivered to client; closing target connection"
5388 );
5389 let _ = forwarding.escalate_client_delivery_failure(
5390 target.connection_id,
5391 target.channel,
5392 target.epoch,
5393 CloseReason::new(
5394 "route_lifecycle_push_delivery_failed",
5395 format!(
5396 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
5397 target.channel
5398 ),
5399 ),
5400 crate::forwarding::UndeliveredFrame {
5401 module_id: target.module_id.as_deref(),
5402 sink: &target.sink,
5403 },
5404 );
5405 }
5406 }
5407 }
5408}
5409
5410#[cfg(test)]
5411mod tests {
5412 use std::{
5413 collections::BTreeMap,
5414 fmt,
5415 path::PathBuf,
5416 sync::{Arc, Mutex},
5417 time::Duration,
5418 };
5419
5420 use serde_json::{json, Value};
5421 use subc_protocol::{
5422 manifest::{
5423 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
5424 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
5425 },
5426 session::HealthStatus,
5427 FrameType,
5428 };
5429
5430 use super::*;
5431 use crate::{
5432 forwarding::{DataRoute, DataRouteState},
5433 registry::ChannelState,
5434 router::FrameSink,
5435 stderr_tail::DEFAULT_MAX_LINE_BYTES,
5436 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
5437 test_support::TestTempDir,
5438 RouteCtx, Router,
5439 };
5440 use tokio::{
5441 sync::mpsc,
5442 time::{sleep, Instant},
5443 };
5444 use tracing::{
5445 field::{Field, Visit},
5446 Event, Subscriber,
5447 };
5448 use tracing_subscriber::{layer::Context, prelude::*, Layer};
5449
5450 fn fake_aft_stub_path() -> PathBuf {
5469 let mut path = std::env::current_exe().expect("current_exe available in tests");
5470 path.pop(); path.pop(); path.push(if cfg!(windows) {
5473 "fake-aft-stub.exe"
5474 } else {
5475 "fake-aft-stub"
5476 });
5477 assert!(
5478 path.exists(),
5479 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
5480 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
5481 path.display()
5482 );
5483 path
5484 }
5485
5486 fn client_retries(code: &str) -> bool {
5491 subc_protocol::error_codes::is_retryable_route_open(code)
5492 }
5493
5494 #[test]
5505 fn retryability_of_forwarding_codes_matches_the_failure() {
5506 let transient = [
5509 ForwardingError::NoModuleConnection,
5510 ForwardingError::ModuleReloading {
5511 module_id: "m".into(),
5512 },
5513 ForwardingError::StaleModuleEndpoint,
5514 ForwardingError::UnknownReservation {
5515 client_channel: 1,
5516 module_channel: 1,
5517 },
5518 ForwardingError::ConnectionClosing {
5519 connection_id: ConnectionId::new(1),
5520 },
5521 ForwardingError::ClientEgressClosed {
5522 connection_id: ConnectionId::new(1),
5523 },
5524 ForwardingError::ModuleEgressUnavailable {
5525 connection_id: ConnectionId::new(1),
5526 },
5527 ];
5528 for err in transient {
5529 let code = forwarding_error_code(&err);
5530 assert!(
5531 client_retries(code),
5532 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
5533 );
5534 }
5535
5536 let permanent = [
5541 ForwardingError::ClientRouteChannelExhausted {
5542 connection_id: ConnectionId::new(1),
5543 },
5544 ForwardingError::ModuleRouteChannelExhausted {
5545 endpoint: ModuleEndpointId {
5546 connection_id: ConnectionId::new(1),
5547 generation: 1,
5548 },
5549 },
5550 ForwardingError::RelayCorrelationExhausted,
5551 ForwardingError::RouteOpenBuild("x".into()),
5552 ForwardingError::Poisoned,
5553 ];
5554 for err in permanent {
5555 let code = forwarding_error_code(&err);
5556 assert!(
5557 !client_retries(code),
5558 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
5559 );
5560 }
5561 }
5562
5563 #[tokio::test]
5591 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
5592 let handler = ControlHandler::default();
5593 let frame =
5594 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
5595
5596 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
5600 assert_eq!(
5601 stamped,
5602 Principal::Direct,
5603 "a caller that proved nothing must not be stamped as a supervised module"
5604 );
5605
5606 let forged = handler
5611 .route_open_principal(
5612 &frame,
5613 Some(ConsumerIdentity {
5614 module_id: "aft".to_string(),
5615 launch_nonce: "not-a-real-nonce".to_string(),
5616 }),
5617 )
5618 .unwrap();
5619 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
5620 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
5621 }
5622
5623 #[test]
5640 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
5641 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
5642 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
5643 let ClientControlRequest::RouteOpen {
5644 consumer_identity, ..
5645 } = parsed
5646 else {
5647 panic!("route.open body must parse as RouteOpen");
5648 };
5649 assert_eq!(
5650 consumer_identity,
5651 Some(ConsumerIdentity {
5652 module_id: "aft".to_string(),
5653 launch_nonce: "n".to_string(),
5654 }),
5655 "the wire field name must reach the value route_open_principal reads"
5656 );
5657 }
5658
5659 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
5660 ModuleManifest::builder(module_id, "0.1.0")
5661 .protocol_ver(protocol_ver)
5662 .provides(vec![ProviderRole::ToolProvider {
5663 tools: vec![Tool {
5664 name: "read".to_string(),
5665 description: None,
5666 execution_mode: ExecutionMode::Pure,
5667 schema: json!({"type": "object"}),
5668 }],
5669 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
5670 concurrency: Concurrency::ModuleManaged,
5671 emits_push: true,
5672 sub_supervises: true,
5673 }])
5674 .build()
5675 }
5676
5677 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
5678 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
5679 }
5680
5681 fn hello_frame_with_control_ops(
5682 module_id: &str,
5683 protocol_ver: u8,
5684 corr: u64,
5685 control_ops: Option<Vec<String>>,
5686 ) -> Frame {
5687 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
5688 }
5689
5690 fn hello_frame_with_nonce(
5691 module_id: &str,
5692 protocol_ver: u8,
5693 corr: u64,
5694 launch_nonce: Option<&str>,
5695 ) -> Frame {
5696 hello_frame_full(
5697 module_id,
5698 protocol_ver,
5699 corr,
5700 None,
5701 launch_nonce.map(ToOwned::to_owned),
5702 )
5703 }
5704
5705 fn hello_frame_full(
5706 module_id: &str,
5707 protocol_ver: u8,
5708 corr: u64,
5709 control_ops: Option<Vec<String>>,
5710 launch_nonce: Option<String>,
5711 ) -> Frame {
5712 let body = serde_json::to_vec(&ModuleHelloBody {
5713 manifest: manifest(module_id, protocol_ver),
5714 protocol_ver,
5715 control_ops,
5716 launch_nonce,
5717 })
5718 .unwrap();
5719 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
5720 }
5721
5722 fn non_routable_hello_frame_with_control_ops(
5723 module_id: &str,
5724 corr: u64,
5725 control_ops: Option<Vec<String>>,
5726 ) -> Frame {
5727 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
5728 manifest.provides.clear();
5729 let body = serde_json::to_vec(&ModuleHelloBody {
5730 manifest,
5731 protocol_ver: PROTOCOL_VERSION,
5732 control_ops,
5733 launch_nonce: None,
5734 })
5735 .unwrap();
5736 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
5737 }
5738
5739 fn capability_grammar_hello_frame(
5740 capabilities: Value,
5741 runtime_computed: Option<Value>,
5742 corr: u64,
5743 ) -> Frame {
5744 let mut body = serde_json::to_value(ModuleHelloBody {
5745 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
5746 protocol_ver: PROTOCOL_VERSION,
5747 control_ops: None,
5748 launch_nonce: None,
5749 })
5750 .expect("HELLO body serializes");
5751 body["manifest"]["capabilities"] = capabilities;
5752 if let Some(runtime_computed) = runtime_computed {
5753 body["runtime_computed"] = runtime_computed;
5754 }
5755 Frame::build(
5756 FrameType::Hello,
5757 control_flags(),
5758 0,
5759 0,
5760 corr,
5761 serde_json::to_vec(&body).expect("HELLO body reserializes"),
5762 )
5763 .expect("HELLO frame builds")
5764 }
5765
5766 fn channel_request(channel: u16, corr: u64) -> Frame {
5767 Frame::build(
5768 FrameType::Request,
5769 Flags::new(true, Priority::Interactive, false),
5770 channel,
5771 0,
5772 corr,
5773 b"opaque".to_vec(),
5774 )
5775 .unwrap()
5776 }
5777
5778 fn route_ctx(
5779 connection_id: ConnectionId,
5780 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
5781 let (tx, rx) = mpsc::channel(8);
5782 (
5783 RouteCtx {
5784 connection_id,
5785 egress: FrameSink::new(tx),
5786 },
5787 rx,
5788 )
5789 }
5790
5791 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
5792 serde_json::from_slice(&frame.body).unwrap()
5793 }
5794
5795 async fn hello_via_sink(
5800 handler: &ControlHandler,
5801 ctx: &RouteCtx,
5802 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
5803 hello: Frame,
5804 ) -> Frame {
5805 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
5806 assert!(
5807 replies.is_empty(),
5808 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
5809 );
5810 let ack = rx
5811 .try_recv()
5812 .expect("HELLO_ACK is queued on the module sink")
5813 .frame;
5814 assert_eq!(ack.header.ty, FrameType::HelloAck);
5815 ack
5816 }
5817
5818 fn parse_error(frame: &Frame) -> Value {
5819 serde_json::from_slice(&frame.body).unwrap()
5820 }
5821
5822 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
5823 serde_json::from_slice(&frame.body).unwrap()
5824 }
5825
5826 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
5827 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
5828 route_channel,
5829 route_epoch: 0,
5830 kind,
5831 })
5832 .unwrap();
5833 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
5834 }
5835
5836 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
5837 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
5838 module_id: module_id.to_string(),
5839 })
5840 .unwrap();
5841 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
5842 }
5843
5844 fn route_open_frame(corr: u64, module_id: &str, project_root: TestTempDir) -> Frame {
5845 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
5846 }
5847
5848 fn route_open_frame_with_consumer_capabilities(
5849 corr: u64,
5850 module_id: &str,
5851 project_root: TestTempDir,
5852 consumer_capabilities: Option<Vec<String>>,
5853 ) -> Frame {
5854 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
5855 target: RouteTarget::ToolProvider {
5856 module_id: module_id.to_string(),
5857 },
5858 identity: BindIdentity::new(
5859 project_root.path().to_path_buf(),
5860 "unit".to_string(),
5861 "session".to_string(),
5862 ),
5863 consumer_identity: None,
5864 consumer_capabilities,
5865 admission_facts: None,
5866 })
5867 .unwrap();
5868 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
5869 }
5870
5871 fn route_open_frame_with_admission_facts(
5872 corr: u64,
5873 module_id: &str,
5874 project_root: TestTempDir,
5875 consumer_identity: Option<subc_control::ConsumerIdentity>,
5876 facts: Option<Value>,
5877 ) -> Frame {
5878 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
5879 target: RouteTarget::ToolProvider {
5880 module_id: module_id.to_string(),
5881 },
5882 identity: BindIdentity::new(
5883 project_root.path().to_path_buf(),
5884 "unit".to_string(),
5885 format!("session-{corr}"),
5886 ),
5887 consumer_identity,
5888 consumer_capabilities: None,
5889 admission_facts: facts,
5890 })
5891 .unwrap();
5892 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
5893 }
5894
5895 #[derive(Clone, Default)]
5896 struct EventCapture {
5897 events: Arc<Mutex<Vec<CapturedEvent>>>,
5898 }
5899
5900 #[derive(Clone, Debug)]
5901 struct CapturedEvent {
5902 target: String,
5903 fields: BTreeMap<String, String>,
5904 }
5905
5906 impl EventCapture {
5907 fn events(&self) -> Vec<CapturedEvent> {
5908 self.events.lock().unwrap().clone()
5909 }
5910 }
5911
5912 impl<S> Layer<S> for EventCapture
5913 where
5914 S: Subscriber,
5915 {
5916 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
5917 let mut visitor = EventFieldVisitor::default();
5918 event.record(&mut visitor);
5919 self.events.lock().unwrap().push(CapturedEvent {
5920 target: event.metadata().target().to_string(),
5921 fields: visitor.fields,
5922 });
5923 }
5924 }
5925
5926 #[derive(Default)]
5927 struct EventFieldVisitor {
5928 fields: BTreeMap<String, String>,
5929 }
5930
5931 impl Visit for EventFieldVisitor {
5932 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
5933 self.fields
5934 .insert(field.name().to_string(), format!("{value:?}"));
5935 }
5936 }
5937
5938 fn health_response(corr: u64, status: HealthStatus) -> Frame {
5939 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
5940 status,
5941 detail: Some("warming".to_string()),
5942 metrics: Some(json!({"queue_depth": 3})),
5943 })
5944 .unwrap();
5945 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
5946 }
5947
5948 fn route_bind_ack(corr: u64) -> Frame {
5949 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
5950 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
5951 }
5952
5953 fn unique_project_root(label: &str) -> TestTempDir {
5954 TestTempDir::new(label)
5955 }
5956
5957 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
5958 match parse_route_poll(frame) {
5959 ClientControlResponse::RoutePoll {
5960 status: None,
5961 live: Some(live),
5962 ..
5963 } => assert_eq!(live, expected_live),
5964 other => panic!("unexpected route.poll response: {other:?}"),
5965 }
5966 }
5967
5968 fn bind_liveness_route(
5969 registry: &Registry,
5970 forwarding: &ForwardingTable,
5971 module_id: &str,
5972 ) -> (RouteCtx, u16, u32) {
5973 let module_connection = ConnectionId::new(101);
5974 let client_connection = ConnectionId::new(202);
5975 let registration = registry
5976 .register_with_control_ops(
5977 manifest(module_id, PROTOCOL_VERSION),
5978 PROTOCOL_VERSION,
5979 module_connection,
5980 module_baseline_control_ops(),
5981 )
5982 .unwrap();
5983 let (module_tx, _module_rx) = mpsc::channel(8);
5984 let endpoint = forwarding
5985 .register_module_connection(
5986 module_connection,
5987 module_id.to_string(),
5988 PROTOCOL_VERSION,
5989 manifest_concurrency(®istration.manifest),
5990 FrameSink::new(module_tx),
5991 )
5992 .unwrap();
5993 let (client_ctx, _client_rx) = route_ctx(client_connection);
5994 let pending = forwarding
5995 .begin_route_bind_relay_for_test(
5996 client_connection,
5997 client_ctx.egress.clone(),
5998 1,
5999 module_id,
6000 )
6001 .unwrap();
6002 assert_eq!(pending.endpoint, endpoint);
6003 let route_channel = pending.client_channel;
6004 let route_epoch = pending.client_epoch;
6005 forwarding
6006 .complete_pending_relay(
6007 module_connection,
6008 pending.corr,
6009 RouteBindRelayOutcome::Accepted,
6010 )
6011 .unwrap();
6012 (client_ctx, route_channel, route_epoch)
6013 }
6014
6015 struct FakeProcessLiveness {
6016 live: Option<bool>,
6017 }
6018
6019 impl ModuleProcessLiveness for FakeProcessLiveness {
6020 fn process_live(&self, _module_id: &str) -> Option<bool> {
6021 self.live
6022 }
6023 }
6024
6025 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6026 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
6027 {
6028 let registry = Arc::new(Registry::default());
6029 let supervisor_handle = SupervisorHandle::new();
6030 let supervisor = Supervisor::new(
6031 Arc::clone(®istry),
6032 RestartPolicy::new(1, Duration::from_millis(10)),
6033 )
6034 .with_handle(supervisor_handle.clone());
6035 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
6036 let module = supervisor
6037 .spawn(ModuleSpec {
6038 module_id: "stderr-tail-wire".to_string(),
6039 program: fake_aft_stub_path(),
6040 args: Vec::new(),
6041 env: vec![
6042 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
6043 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
6044 ],
6045 reserved: false,
6046 reserved_prefixes: Vec::new(),
6047 protocol: ModuleProtocol::Subc,
6048 overlap: Default::default(),
6049 })
6050 .unwrap();
6051
6052 let deadline = Instant::now() + Duration::from_secs(5);
6053 loop {
6054 let tail = module.stderr_tail(None, None);
6055 if tail
6056 .entries
6057 .iter()
6058 .any(|entry| matches!(entry, TailEntry::ProcessStart))
6059 && tail.entries.iter().any(|entry| {
6060 matches!(
6061 entry,
6062 TailEntry::Line {
6063 truncated: true,
6064 ..
6065 }
6066 )
6067 })
6068 {
6069 break;
6070 }
6071 assert!(
6072 Instant::now() < deadline,
6073 "module did not produce a truncated line and restart boundary: {tail:?}"
6074 );
6075 sleep(Duration::from_millis(10)).await;
6076 }
6077
6078 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
6079 let request = ClientControlRequest::SupervisorStderrTail {
6080 module_id: "stderr-tail-wire".to_string(),
6081 max_lines: None,
6082 max_bytes: None,
6083 };
6084 let frame = Frame::build(
6085 FrameType::Request,
6086 control_flags(),
6087 0,
6088 0,
6089 1,
6090 serde_json::to_vec(&request).unwrap(),
6091 )
6092 .unwrap();
6093 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6094 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
6095 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
6096 serde_json::from_slice(&responses[0].body).unwrap()
6097 else {
6098 panic!("expected supervisor.stderr_tail response");
6099 };
6100
6101 assert!(
6102 tail.entries
6103 .iter()
6104 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
6105 "the control response lost the restart boundary"
6106 );
6107 let Some(StderrTailEntry::Line { text, truncated }) = tail.entries.iter().find(|entry| {
6108 matches!(
6109 entry,
6110 StderrTailEntry::Line {
6111 truncated: true,
6112 ..
6113 }
6114 )
6115 }) else {
6116 panic!("the control response lost the truncated line");
6117 };
6118 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
6119 assert!(*truncated);
6120 }
6121
6122 #[tokio::test(flavor = "current_thread")]
6127 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
6128 let dir = TestTempDir::new("terminals-off-worker");
6129 let journal_path = dir.join("terminals.jsonl");
6130 let registry = Arc::new(Registry::default());
6131 let supervisor_handle = SupervisorHandle::new();
6132 let supervisor =
6133 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
6134 .with_handle(supervisor_handle.clone())
6135 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
6136 let module = supervisor
6137 .spawn(ModuleSpec {
6138 module_id: "terminal-off-worker".to_string(),
6139 program: fake_aft_stub_path(),
6140 args: Vec::new(),
6141 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
6142 reserved: false,
6143 reserved_prefixes: Vec::new(),
6144 protocol: ModuleProtocol::Subc,
6145 overlap: Default::default(),
6146 })
6147 .unwrap();
6148 let deadline = Instant::now() + Duration::from_secs(5);
6149 while module.terminal_history().entries.len() != 2 {
6150 assert!(Instant::now() < deadline, "module did not record two exits");
6151 sleep(Duration::from_millis(10)).await;
6152 }
6153
6154 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
6155 let handler =
6156 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
6157 let frame = Frame::build(
6158 FrameType::Request,
6159 control_flags(),
6160 0,
6161 0,
6162 1,
6163 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
6164 module_id: "terminal-off-worker".to_string(),
6165 })
6166 .unwrap(),
6167 )
6168 .unwrap();
6169 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6170 let spawned_at = std::time::Instant::now();
6171 let read = tokio::spawn({
6172 let handler = Arc::clone(&handler);
6173 async move { handler.handle_control_frame(&ctx, frame).await }
6174 });
6175 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
6178 .await
6179 .unwrap()
6180 .expect("the history read reached its pause");
6181 let elapsed = spawned_at.elapsed();
6182 assert!(
6183 elapsed < Duration::from_secs(2) && !read.is_finished(),
6184 "this task could not run while the history read was paused \
6185 (resumed after {elapsed:?}, read finished: {})",
6186 read.is_finished()
6187 );
6188
6189 drop(release);
6190 let responses = read.await.unwrap().unwrap();
6191 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
6192 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
6193 panic!("expected supervisor.terminals response");
6194 };
6195 assert_eq!(terminals.entries.len(), 2);
6196 assert_eq!(terminals.journal_skipped_lines, 0);
6197 assert_eq!(terminals.journal_read_errors, 0);
6198 }
6199
6200 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6201 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
6202 let registry = Arc::new(Registry::default());
6203 let supervisor_handle = SupervisorHandle::new();
6204 let supervisor =
6205 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
6206 .with_handle(supervisor_handle.clone());
6207 let module = supervisor
6208 .spawn(ModuleSpec {
6209 module_id: "terminal-golden".to_string(),
6210 program: fake_aft_stub_path(),
6211 args: Vec::new(),
6212 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
6213 reserved: false,
6214 reserved_prefixes: Vec::new(),
6215 protocol: ModuleProtocol::Subc,
6216 overlap: Default::default(),
6217 })
6218 .unwrap();
6219
6220 let deadline = Instant::now() + Duration::from_secs(5);
6221 while module.terminal_history().entries.len() != 2 {
6222 assert!(
6223 Instant::now() < deadline,
6224 "module did not retain two terminal exits: {:?}",
6225 module.terminal_history()
6226 );
6227 sleep(Duration::from_millis(10)).await;
6228 }
6229
6230 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
6231 let request = ClientControlRequest::SupervisorTerminals {
6232 module_id: "terminal-golden".to_string(),
6233 };
6234 let frame = Frame::build(
6235 FrameType::Request,
6236 control_flags(),
6237 0,
6238 0,
6239 1,
6240 serde_json::to_vec(&request).unwrap(),
6241 )
6242 .unwrap();
6243 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6244 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
6245 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
6246 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
6247 panic!("expected supervisor.terminals response");
6248 };
6249 assert_eq!(terminals.entries.len(), 2);
6250 assert_eq!(terminals.dropped, 0);
6251
6252 let mut rendered = serde_json::to_value(response).unwrap();
6253 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
6256 for (index, entry) in rendered["entries"]
6257 .as_array_mut()
6258 .expect("terminal response entries array")
6259 .iter_mut()
6260 .enumerate()
6261 {
6262 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
6263 }
6264
6265 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
6266 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
6267 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
6268 if std::env::var_os("UPDATE_GOLDEN").is_some() {
6269 std::fs::write(&golden_path, &serialized).unwrap();
6270 }
6271 let expected: Value =
6272 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
6273 assert_eq!(rendered, expected);
6274 }
6275
6276 #[test]
6277 fn hello_registers_manifest_and_returns_ack() {
6278 let registry = Arc::new(Registry::default());
6279 let handler = ControlHandler::new(Arc::clone(®istry));
6280 let conn = ConnectionId::new(1);
6281
6282 let responses = handler
6283 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
6284 .unwrap();
6285
6286 assert_eq!(responses.len(), 1);
6287 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
6288 assert_eq!(responses[0].header.channel, 0);
6289 assert_eq!(responses[0].header.corr, 7);
6290 let ack = parse_ack(&responses[0]);
6291 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
6292 assert!(ack
6293 .subc_capabilities
6294 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
6295 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
6296 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
6297 assert!(ack
6298 .subc_ops
6299 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
6300 assert!(ack
6301 .subc_ops
6302 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
6303
6304 let registration = registry.get_module("aft").unwrap().unwrap();
6305 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
6306 assert_eq!(registration.state, ChannelState::Active);
6307 assert_eq!(registration.connection_id, conn);
6308 assert_eq!(registration.control_ops, module_baseline_control_ops());
6309 }
6310
6311 #[test]
6312 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
6313 let invalid_identifiers = [
6314 ("case_change", "credentials-Provider/v1"),
6315 ("leading_zero", "credentials-provider/v01"),
6316 ("trailing_hyphen", "credentials-provider-/v1"),
6317 ("consecutive_hyphens", "credentials--provider/v1"),
6318 ("uppercase", "Credentials-provider/v1"),
6319 ("missing_v", "credentials-provider/1"),
6320 ("whitespace", "credentials provider/v1"),
6321 ("zero_version", "credentials-provider/v0"),
6322 ("out_of_range_version", "credentials-provider/v4294967296"),
6323 (
6324 "overlength_name",
6325 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
6326 ),
6327 ];
6328 let mut cases = invalid_identifiers
6329 .into_iter()
6330 .map(|(name, identifier)| {
6331 (
6332 format!("identifier_{name}"),
6333 "capabilities.provides[0]".to_string(),
6334 identifier.to_string(),
6335 json!({ "provides": [identifier] }),
6336 None,
6337 )
6338 })
6339 .collect::<Vec<_>>();
6340 cases.extend([
6341 (
6342 "unknown_need".to_string(),
6343 "capabilities.requires[0].need".to_string(),
6344 "deferred".to_string(),
6345 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
6346 None,
6347 ),
6348 (
6349 "duplicate_provides".to_string(),
6350 "capabilities.provides[1]".to_string(),
6351 "credentials-provider/v1".to_string(),
6352 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
6353 None,
6354 ),
6355 (
6356 "duplicate_must_never_reach".to_string(),
6357 "capabilities.must_never_reach[1]".to_string(),
6358 "credentials-provider/v1".to_string(),
6359 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
6360 None,
6361 ),
6362 (
6363 "duplicate_requires_same_need".to_string(),
6364 "capabilities.requires[1]".to_string(),
6365 "credentials-provider/v1".to_string(),
6366 json!({ "requires": [
6367 { "capability": "credentials-provider/v1", "need": "required" },
6368 { "capability": "credentials-provider/v1", "need": "required" }
6369 ] }),
6370 None,
6371 ),
6372 (
6373 "duplicate_requires_conflicting_need".to_string(),
6374 "capabilities.requires[1]".to_string(),
6375 "credentials-provider/v1".to_string(),
6376 json!({ "requires": [
6377 { "capability": "credentials-provider/v1", "need": "required" },
6378 { "capability": "credentials-provider/v1", "need": "optional" }
6379 ] }),
6380 None,
6381 ),
6382 (
6383 "capabilities_root_pointer".to_string(),
6384 "runtime_computed[0]".to_string(),
6385 "/capabilities".to_string(),
6386 json!({}),
6387 Some(json!(["/capabilities"])),
6388 ),
6389 (
6390 "capabilities_descendant_pointer".to_string(),
6391 "runtime_computed[0]".to_string(),
6392 "/capabilities/provides".to_string(),
6393 json!({}),
6394 Some(json!(["/capabilities/provides"])),
6395 ),
6396 (
6397 "malformed_pointer_without_leading_slash".to_string(),
6398 "runtime_computed[0]".to_string(),
6399 "capabilities".to_string(),
6400 json!({}),
6401 Some(json!(["capabilities"])),
6402 ),
6403 (
6404 "malformed_pointer_escape".to_string(),
6405 "runtime_computed[0]".to_string(),
6406 "/roles/~2/tools".to_string(),
6407 json!({}),
6408 Some(json!(["/roles/~2/tools"])),
6409 ),
6410 (
6411 "unknown_capabilities_field".to_string(),
6412 "capabilities.future".to_string(),
6413 "<array>".to_string(),
6414 json!({ "future": [] }),
6415 None,
6416 ),
6417 ]);
6418
6419 for (index, (name, field, value, capabilities, runtime_computed)) in
6420 cases.into_iter().enumerate()
6421 {
6422 let registry = Arc::new(Registry::default());
6423 let handler = ControlHandler::new(Arc::clone(®istry));
6424 let response = handler
6425 .handle_control(
6426 ConnectionId::new((index + 1) as u64),
6427 capability_grammar_hello_frame(
6428 capabilities,
6429 runtime_computed,
6430 index as u64 + 1,
6431 ),
6432 )
6433 .expect("invalid HELLO returns a refusal");
6434
6435 assert_eq!(response.len(), 1, "{name} must emit one refusal");
6436 let error = parse_error(&response[0]);
6437 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
6438 let message = error["message"]
6439 .as_str()
6440 .expect("error message is a string");
6441 assert!(
6442 message.contains(&field),
6443 "{name}: field missing from {message}"
6444 );
6445 assert!(
6446 message.contains(&value),
6447 "{name}: value missing from {message}"
6448 );
6449 assert_eq!(
6450 registry
6451 .active_registration_count()
6452 .expect("registry reads"),
6453 0,
6454 "{name}: refused HELLO must not create a catalog entry"
6455 );
6456 }
6457 }
6458
6459 #[test]
6460 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
6461 let registry = Arc::new(Registry::default());
6462 let handler = ControlHandler::new(Arc::clone(®istry));
6463 let capabilities = json!({
6464 "provides": ["credentials-provider/v1"],
6465 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
6466 "must_never_reach": ["federation-transport/v1"]
6467 });
6468 let response = handler
6469 .handle_control(
6470 ConnectionId::new(99),
6471 capability_grammar_hello_frame(
6472 capabilities.clone(),
6473 Some(json!(["/roles/0/tools"])),
6474 99,
6475 ),
6476 )
6477 .expect("valid HELLO registers");
6478 assert_eq!(response[0].header.ty, FrameType::HelloAck);
6479
6480 let request = Frame::build(
6481 FrameType::Request,
6482 control_flags(),
6483 0,
6484 0,
6485 100,
6486 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
6487 .expect("catalog request serializes"),
6488 )
6489 .expect("catalog request frame builds");
6490 let response = handler
6491 .handle_catalog_list(request, None)
6492 .expect("catalog list succeeds");
6493 let ClientControlResponse::CatalogList { modules, .. } =
6494 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
6495 else {
6496 panic!("catalog request must return catalog.list");
6497 };
6498 assert_eq!(modules.len(), 1);
6499 assert_eq!(
6500 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
6501 capabilities
6502 );
6503 }
6504
6505 #[test]
6506 fn catalog_list_mirrors_management_operation_description() {
6507 let registry = Arc::new(Registry::default());
6508 let handler = ControlHandler::new(Arc::clone(®istry));
6509 let description = "List managed records and return their identifiers and metadata.";
6510 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
6511 manifest.provides = vec![ProviderRole::ManagementSurface {
6512 operations: vec![ManagementOperation {
6513 name: "records.list".to_string(),
6514 kind: ManagementOperationKind::Query,
6515 description: Some(description.to_string()),
6516 }],
6517 config_schema: json!({"type": "object"}),
6518 observability: vec![ObservabilitySurface {
6519 name: "records.stats".to_string(),
6520 kind: ObservabilityKind::Snapshot,
6521 }],
6522 identity_scope: vec![IdentityScope::Project],
6523 concurrency: Concurrency::ModuleManaged,
6524 }];
6525 registry
6526 .register_with_control_ops(
6527 manifest,
6528 PROTOCOL_VERSION,
6529 ConnectionId::new(99),
6530 Vec::new(),
6531 )
6532 .expect("described management manifest registers");
6533
6534 let request = Frame::build(
6535 FrameType::Request,
6536 control_flags(),
6537 0,
6538 0,
6539 100,
6540 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
6541 .expect("catalog request serializes"),
6542 )
6543 .expect("catalog request frame builds");
6544 let response = handler
6545 .handle_catalog_list(request, None)
6546 .expect("catalog list succeeds");
6547 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
6548 assert_eq!(
6549 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
6550 "catalog.list must preserve the declared operation description verbatim"
6551 );
6552 }
6553
6554 #[test]
6555 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
6556 let registry = Arc::new(Registry::default());
6557 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
6558 [("vault".to_string(), true), ("squatter".to_string(), true)],
6559 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
6560 );
6561 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
6562 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
6563 provides: vec!["credentials-provider/v1".to_string()],
6564 requires: Vec::new(),
6565 must_never_reach: Vec::new(),
6566 });
6567 let frame = Frame::build(
6568 FrameType::Hello,
6569 control_flags(),
6570 0,
6571 0,
6572 77,
6573 serde_json::to_vec(&ModuleHelloBody {
6574 manifest: squatter,
6575 protocol_ver: PROTOCOL_VERSION,
6576 control_ops: None,
6577 launch_nonce: None,
6578 })
6579 .expect("HELLO serializes"),
6580 )
6581 .expect("HELLO frame builds");
6582 let response = handler
6583 .handle_control(ConnectionId::new(77), frame)
6584 .expect("reserved claim receives a typed refusal");
6585 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
6586 assert_eq!(
6587 registry
6588 .active_registration_count()
6589 .expect("registry reads"),
6590 0,
6591 "a reserved capability refusal must not leave a catalog entry"
6592 );
6593 }
6594
6595 #[test]
6596 fn server_describe_surfaces_required_capability_verdict_fields() {
6597 let registry = Arc::new(Registry::default());
6598 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
6599 [
6600 ("consumer".to_string(), true),
6601 ("provider".to_string(), false),
6602 ],
6603 BTreeMap::new(),
6604 );
6605 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
6606 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
6607 provides: Vec::new(),
6608 requires: vec![subc_protocol::manifest::CapabilityRequirement {
6609 capability: "credentials-provider/v1".to_string(),
6610 need: subc_protocol::manifest::CapabilityNeed::Required,
6611 }],
6612 must_never_reach: Vec::new(),
6613 });
6614 let hello = Frame::build(
6615 FrameType::Hello,
6616 control_flags(),
6617 0,
6618 0,
6619 78,
6620 serde_json::to_vec(&ModuleHelloBody {
6621 manifest: consumer,
6622 protocol_ver: PROTOCOL_VERSION,
6623 control_ops: None,
6624 launch_nonce: None,
6625 })
6626 .expect("HELLO serializes"),
6627 )
6628 .expect("HELLO frame builds");
6629 handler
6630 .handle_control(ConnectionId::new(78), hello)
6631 .expect("consumer registers");
6632 let describe = Frame::build(
6633 FrameType::Request,
6634 control_flags(),
6635 0,
6636 0,
6637 79,
6638 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
6639 .expect("request serializes"),
6640 )
6641 .expect("describe frame builds");
6642 let response = handler
6643 .handle_server_describe(describe)
6644 .expect("server.describe succeeds");
6645 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
6646 let requirement = &rendered["capability_requirements"][0];
6647 assert_eq!(requirement["consumer"], "consumer");
6648 assert_eq!(requirement["verdict"], "never_provided");
6649 assert_eq!(requirement["episode_seq"], 1);
6650 assert_eq!(requirement["config_satisfiable"], false);
6651 assert_eq!(requirement["runtime_available"], false);
6652 assert!(requirement["detail"]
6653 .as_str()
6654 .expect("detail string")
6655 .contains("credentials-provider/v1"));
6656 }
6657
6658 #[test]
6659 fn catalog_list_omits_capabilities_for_legacy_manifest() {
6660 let registry = Arc::new(Registry::default());
6661 let handler = ControlHandler::new(Arc::clone(®istry));
6662 let hello = handler
6663 .handle_control(
6664 ConnectionId::new(101),
6665 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
6666 )
6667 .expect("legacy HELLO registers");
6668 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
6669
6670 let request = Frame::build(
6671 FrameType::Request,
6672 control_flags(),
6673 0,
6674 0,
6675 102,
6676 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
6677 .expect("catalog request serializes"),
6678 )
6679 .expect("catalog request frame builds");
6680 let response = handler
6681 .handle_catalog_list(request, None)
6682 .expect("catalog list succeeds");
6683 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
6684 assert!(
6685 body["modules"][0].get("capabilities").is_none(),
6686 "legacy manifest must retain an absent capabilities field on catalog.list"
6687 );
6688 }
6689
6690 #[test]
6691 fn hello_ack_omits_storage_when_no_storage_config() {
6692 let registry = Arc::new(Registry::default());
6693 let handler = ControlHandler::new(Arc::clone(®istry));
6694 let responses = handler
6695 .handle_control(
6696 ConnectionId::new(1),
6697 hello_frame("aft", PROTOCOL_VERSION, 7),
6698 )
6699 .unwrap();
6700 let ack = parse_ack(&responses[0]);
6701 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
6702 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
6703 }
6704
6705 #[tokio::test]
6706 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
6707 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
6708 let registry = Arc::new(Registry::default());
6709 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
6710 let responses = handler
6711 .handle_control(
6712 ConnectionId::new(1),
6713 hello_frame("aft", PROTOCOL_VERSION, 7),
6714 )
6715 .unwrap();
6716 let ack = parse_ack(&responses[0]);
6717 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
6718
6719 let described = handler
6720 .handle_control_frame(
6721 &route_ctx(ConnectionId::new(2)).0,
6722 Frame::build(
6723 FrameType::Request,
6724 control_flags(),
6725 0,
6726 0,
6727 9,
6728 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
6729 )
6730 .unwrap(),
6731 )
6732 .await
6733 .unwrap();
6734 let ClientControlResponse::ServerDescribe { machine_id, .. } =
6735 serde_json::from_slice(&described[0].body).unwrap()
6736 else {
6737 panic!("server.describe answered with another shape");
6738 };
6739 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
6740 }
6741
6742 #[test]
6743 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
6744 let registry = Arc::new(Registry::default());
6747 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
6748 crate::daemon_config::StorageConfig::Sqlite {
6749 data_home: std::path::PathBuf::from("/data"),
6750 },
6751 ));
6752
6753 let responses = handler
6754 .handle_control(
6755 ConnectionId::new(1),
6756 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
6757 )
6758 .unwrap();
6759 let ack = parse_ack(&responses[0]);
6760 assert_eq!(
6761 ack.storage,
6762 Some(serde_json::json!({
6763 "module_id": "alfonso-routing",
6764 "storage_namespace": "default",
6765 "isolation": { "kind": "module" },
6766 "backend": {
6767 "backend": "sqlite",
6768 "path": "/data/cortexkit/alfonso-routing/store.db"
6769 }
6770 })),
6771 "the delivered descriptor is the module's own sqlite store path"
6772 );
6773 }
6774
6775 #[test]
6776 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
6777 let registry = Arc::new(Registry::default());
6778 let handler = ControlHandler::new(Arc::clone(®istry));
6779 let conn = ConnectionId::new(1);
6780 let responses = handler
6781 .handle_control(
6782 conn,
6783 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
6784 )
6785 .unwrap();
6786 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
6787 let registration = registry.get_module("aft").unwrap().unwrap();
6788 assert_eq!(registration.control_ops, module_baseline_control_ops());
6789
6790 let frame =
6791 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
6792 assert!(handler
6793 .guard_module_control_op(&frame, "aft", "route.bind")
6794 .unwrap()
6795 .is_none());
6796 let error = handler
6797 .guard_module_control_op(&frame, "aft", "test.synthetic")
6798 .unwrap()
6799 .expect("synthetic ungranted op should be rejected");
6800 assert_eq!(error.header.ty, FrameType::Error);
6801 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
6802 }
6803
6804 #[test]
6805 fn hello_control_ops_some_adds_optional_grants() {
6806 let registry = Arc::new(Registry::default());
6807 let handler = ControlHandler::new(Arc::clone(®istry));
6808 handler
6809 .handle_control(
6810 ConnectionId::new(1),
6811 hello_frame_with_control_ops(
6812 "aft",
6813 PROTOCOL_VERSION,
6814 7,
6815 Some(vec![
6816 "future.synthetic".to_string(),
6817 "route.bind".to_string(),
6818 ]),
6819 ),
6820 )
6821 .unwrap();
6822 let registration = registry.get_module("aft").unwrap().unwrap();
6823 assert_eq!(
6824 registration.control_ops,
6825 vec![
6826 "route.bind".to_string(),
6827 "route.status".to_string(),
6828 "future.synthetic".to_string(),
6829 ]
6830 );
6831 let frame =
6832 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
6833 assert!(handler
6834 .guard_module_control_op(&frame, "aft", "future.synthetic")
6835 .unwrap()
6836 .is_none());
6837 }
6838
6839 #[tokio::test]
6840 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
6841 let registry = Arc::new(Registry::default());
6842 let forwarding = Arc::new(ForwardingTable::default());
6843 let handler =
6844 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
6845 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
6846 hello_via_sink(
6847 &handler,
6848 &module_ctx,
6849 &mut module_rx,
6850 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
6851 )
6852 .await;
6853
6854 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
6855 let responses = handler
6856 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
6857 .await
6858 .unwrap();
6859 assert_eq!(responses.len(), 1);
6860 assert_eq!(responses[0].header.ty, FrameType::Error);
6861 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
6862 assert!(module_rx.try_recv().is_err());
6863 }
6864
6865 #[tokio::test]
6866 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
6867 let registry = Arc::new(Registry::default());
6868 let forwarding = Arc::new(ForwardingTable::default());
6869 let handler =
6870 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
6871 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
6872 hello_via_sink(
6873 &handler,
6874 &module_ctx,
6875 &mut module_rx,
6876 hello_frame_with_control_ops(
6877 "aft",
6878 PROTOCOL_VERSION,
6879 7,
6880 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
6881 ),
6882 )
6883 .await;
6884
6885 let project_root = unique_project_root("demux");
6886 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
6887 let route_handler = handler.clone();
6888 let route_task = tokio::spawn(async move {
6889 route_handler
6890 .handle_control_frame(
6891 &route_client_ctx,
6892 route_open_frame(100, "aft", project_root),
6893 )
6894 .await
6895 .unwrap()
6896 });
6897 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
6898 .await
6899 .unwrap()
6900 .unwrap();
6901 assert!(matches!(
6902 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
6903 ModuleControlRequest::RouteBind { .. }
6904 ));
6905
6906 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
6907 let health_handler = handler.clone();
6908 let health_task = tokio::spawn(async move {
6909 health_handler
6910 .handle_control_frame(
6911 &health_client_ctx,
6912 supervisor_health_probe_frame(101, "aft"),
6913 )
6914 .await
6915 .unwrap()
6916 });
6917 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
6918 .await
6919 .unwrap()
6920 .unwrap();
6921 assert_eq!(
6922 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
6923 ModuleControlRequest::HealthCheck {}
6924 );
6925
6926 handler
6927 .handle_control_frame(
6928 &module_ctx,
6929 health_response(health_frame.header.corr, HealthStatus::Degraded),
6930 )
6931 .await
6932 .unwrap();
6933 let health_response = health_task.await.unwrap();
6934 assert_eq!(health_response.len(), 1);
6935 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
6936 ClientControlResponse::SupervisorHealthProbe {
6937 module_id,
6938 status,
6939 detail,
6940 metrics,
6941 } => {
6942 assert_eq!(module_id, "aft");
6943 assert_eq!(status, HealthStatus::Degraded);
6944 assert_eq!(detail.as_deref(), Some("warming"));
6945 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
6946 }
6947 other => panic!("unexpected health response: {other:?}"),
6948 }
6949
6950 handler
6951 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
6952 .await
6953 .unwrap();
6954 let route_response = route_task.await.unwrap();
6955 assert!(route_response.is_empty());
6956 let published = route_client_rx.recv().await.unwrap();
6957 assert!(matches!(
6958 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
6959 ClientControlResponse::RouteOpen { .. }
6960 ));
6961 }
6962
6963 async fn relay_route_open(
6968 handler: &ControlHandler,
6969 client_connection: ConnectionId,
6970 client_egress: &FrameSink,
6971 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
6972 corr: u64,
6973 module_id: &str,
6974 project_root_label: &str,
6975 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
6976 let ctx = RouteCtx {
6977 connection_id: client_connection,
6978 egress: client_egress.clone(),
6979 };
6980 let handler = handler.clone();
6981 let project_root = unique_project_root(project_root_label);
6982 let module_id = module_id.to_string();
6983 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
6984 let task = tokio::spawn(async move {
6985 let _guard = tracing::dispatcher::set_default(&dispatch);
6986 handler
6987 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
6988 .await
6989 .unwrap()
6990 });
6991 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
6992 .await
6993 .expect("module receives the relayed route.bind")
6994 .expect("module egress is open");
6995 (task, bind.frame)
6996 }
6997
6998 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
6999 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
7000 ModuleControlRequest::RouteBind {
7001 route_channel,
7002 epoch,
7003 ..
7004 } => (route_channel, epoch),
7005 other => panic!("expected a route.bind request, got {other:?}"),
7006 }
7007 }
7008
7009 fn published_route(frame: &Frame) -> (u16, u32) {
7010 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
7011 ClientControlResponse::RouteOpen {
7012 route_channel,
7013 route_epoch,
7014 } => (route_channel, route_epoch),
7015 other => panic!("expected a route.open response, got {other:?}"),
7016 }
7017 }
7018
7019 #[tokio::test]
7039 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
7040 let registry = Arc::new(Registry::default());
7041 let forwarding = Arc::new(ForwardingTable::default());
7042 let handler =
7043 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
7044
7045 let module_connection = ConnectionId::new(30);
7046 let (module_ctx, mut module_rx) = route_ctx(module_connection);
7047 hello_via_sink(
7048 &handler,
7049 &module_ctx,
7050 &mut module_rx,
7051 hello_frame("aft", PROTOCOL_VERSION, 7),
7052 )
7053 .await;
7054
7055 let dying_client = ConnectionId::new(31);
7056 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
7057
7058 let (first_task, first_bind) = relay_route_open(
7061 &handler,
7062 dying_client,
7063 &dying_ctx.egress,
7064 &mut module_rx,
7065 100,
7066 "aft",
7067 "closing-first",
7068 )
7069 .await;
7070 handler
7071 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
7072 .await
7073 .unwrap();
7074 assert!(first_task.await.unwrap().is_empty());
7075 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
7076
7077 let (second_task, second_bind) = relay_route_open(
7079 &handler,
7080 dying_client,
7081 &dying_ctx.egress,
7082 &mut module_rx,
7083 101,
7084 "aft",
7085 "closing-second",
7086 )
7087 .await;
7088 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
7089
7090 assert!(forwarding
7093 .escalate_client_delivery_failure(
7094 dying_client,
7095 first_channel,
7096 first_epoch,
7097 CloseReason::new(
7098 "module_to_client_delivery_failed",
7099 "client egress refused a module frame",
7100 ),
7101 crate::forwarding::UndeliveredFrame {
7102 module_id: None,
7103 sink: &dying_ctx.egress,
7104 },
7105 )
7106 .unwrap());
7107 assert!(!dying_ctx.egress.is_closed());
7108
7109 let ack = handler
7111 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
7112 .await;
7113 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
7114 if module_loop_error.is_some() {
7115 handler.cleanup_connection(module_connection).unwrap();
7119 }
7120 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7124 .await
7125 .ok()
7126 .flatten();
7127
7128 assert!(
7130 registry
7131 .get_module_by_connection(module_connection)
7132 .unwrap()
7133 .is_some(),
7134 "one client's closing connection ended the shared module connection: \
7135 {module_loop_error:?}"
7136 );
7137 let cotenant = ConnectionId::new(32);
7139 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
7140 let (cotenant_task, cotenant_bind) = relay_route_open(
7141 &handler,
7142 cotenant,
7143 &cotenant_ctx.egress,
7144 &mut module_rx,
7145 102,
7146 "aft",
7147 "closing-cotenant",
7148 )
7149 .await;
7150 handler
7151 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
7152 .await
7153 .unwrap();
7154 assert!(cotenant_task.await.unwrap().is_empty());
7155 let (cotenant_channel, cotenant_epoch) =
7156 published_route(&cotenant_rx.recv().await.unwrap());
7157 assert!(matches!(
7158 forwarding
7159 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
7160 .unwrap(),
7161 DataRoute::Client(DataRouteState::Bound(_))
7162 ));
7163
7164 let goodbye = post_ack_module_frame
7167 .expect("module receives a GOODBYE for the abandoned route channel");
7168 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
7169 assert_eq!(goodbye.header.channel, abandoned_channel);
7170 assert_eq!(goodbye.header.epoch, abandoned_epoch);
7171
7172 assert!(dying_rx.try_recv().is_err());
7176 let second_response = second_task.await.unwrap();
7177 assert_eq!(second_response.len(), 1);
7178 assert_eq!(
7179 parse_error(&second_response[0])["code"],
7180 "target_unavailable"
7181 );
7182 }
7183
7184 #[test]
7191 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
7192 let handler = ControlHandler::default();
7193 let module_connection = ConnectionId::new(30);
7194 let client_connection = ConnectionId::new(31);
7195
7196 handler
7197 .refuse_to_end_module_connection_for_a_client(
7198 module_connection,
7199 77,
7200 ForwardingError::ConnectionClosing {
7201 connection_id: client_connection,
7202 },
7203 )
7204 .expect("a closing client must never end the module connection");
7205
7206 assert!(matches!(
7207 handler.refuse_to_end_module_connection_for_a_client(
7208 module_connection,
7209 78,
7210 ForwardingError::ConnectionClosing {
7211 connection_id: module_connection,
7212 },
7213 ),
7214 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
7215 connection_id
7216 })) if connection_id == module_connection
7217 ));
7218 assert!(matches!(
7219 handler.refuse_to_end_module_connection_for_a_client(
7220 module_connection,
7221 79,
7222 ForwardingError::Poisoned,
7223 ),
7224 Err(RouterError::Forwarding(ForwardingError::Poisoned))
7225 ));
7226 assert!(matches!(
7227 handler.refuse_to_end_module_connection_for_a_client(
7228 module_connection,
7229 80,
7230 ForwardingError::StaleModuleEndpoint,
7231 ),
7232 Err(RouterError::Forwarding(
7233 ForwardingError::StaleModuleEndpoint
7234 ))
7235 ));
7236 }
7237
7238 #[tokio::test]
7251 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
7252 let registry = Arc::new(Registry::default());
7253 let forwarding = Arc::new(ForwardingTable::default());
7254 let supervisor = SupervisorHandle::new();
7255 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7256 let handler =
7257 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
7258 .with_supervisor(supervisor);
7259
7260 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
7261 hello_via_sink(
7262 &handler,
7263 &target_ctx,
7264 &mut target_rx,
7265 hello_frame("target", PROTOCOL_VERSION, 1),
7266 )
7267 .await;
7268
7269 let wrong_nonce = handler
7273 .handle_control_frame(
7274 &route_ctx(ConnectionId::new(91)).0,
7275 route_open_frame_with_admission_facts(
7276 20,
7277 "target",
7278 unique_project_root("admission-facts"),
7279 Some(subc_control::ConsumerIdentity {
7280 module_id: "fed".to_string(),
7281 launch_nonce: "not-the-real-nonce".to_string(),
7282 }),
7283 None,
7284 ),
7285 )
7286 .await
7287 .unwrap();
7288 assert_eq!(
7289 parse_error(&wrong_nonce[0])["code"],
7290 "bad_consumer_identity",
7291 "a mismatched launch nonce must be refused, not stamped Reserved"
7292 );
7293
7294 let never_spawned = handler
7298 .handle_control_frame(
7299 &route_ctx(ConnectionId::new(92)).0,
7300 route_open_frame_with_admission_facts(
7301 21,
7302 "target",
7303 unique_project_root("admission-facts"),
7304 Some(subc_control::ConsumerIdentity {
7305 module_id: "never-spawned".to_string(),
7306 launch_nonce: "any-nonce".to_string(),
7307 }),
7308 None,
7309 ),
7310 )
7311 .await
7312 .unwrap();
7313 assert_eq!(
7314 parse_error(&never_spawned[0])["code"],
7315 "bad_consumer_identity",
7316 "an unspawned module_id must be refused rather than accepted for lack of a record"
7317 );
7318 }
7319
7320 #[tokio::test]
7343 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
7344 let registry = Arc::new(Registry::default());
7345 let forwarding = Arc::new(ForwardingTable::default());
7346 let supervisor = SupervisorHandle::new();
7347 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7348 let handler =
7349 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
7350 .with_supervisor(supervisor);
7351
7352 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
7353 hello_via_sink(
7354 &handler,
7355 &target_ctx,
7356 &mut target_rx,
7357 hello_frame("target", PROTOCOL_VERSION, 1),
7358 )
7359 .await;
7360
7361 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
7362 let route_handler = handler.clone();
7363 let route_task = tokio::spawn(async move {
7364 route_handler
7365 .handle_control_frame(
7366 &client_ctx,
7367 route_open_frame_with_admission_facts(
7368 30,
7369 "target",
7370 unique_project_root("admission-facts"),
7371 Some(subc_control::ConsumerIdentity {
7372 module_id: "fed".to_string(),
7373 launch_nonce: "fed-nonce".to_string(),
7374 }),
7375 None,
7376 ),
7377 )
7378 .await
7379 .unwrap()
7380 });
7381
7382 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7389 .await
7390 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
7391 .expect("module control channel closed before route.bind");
7392 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
7393 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
7394 panic!("expected route.bind")
7395 };
7396 assert_eq!(
7397 principal,
7398 Some(Principal::Reserved {
7399 module_id: "fed".to_string()
7400 }),
7401 "a correctly attested consumer must be stamped Reserved for its own id"
7402 );
7403
7404 handler
7405 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
7406 .await
7407 .unwrap();
7408 assert!(route_task.await.unwrap().is_empty());
7409 assert!(
7410 matches!(
7411 serde_json::from_slice::<ClientControlResponse>(
7412 &client_rx.recv().await.unwrap().body
7413 )
7414 .unwrap(),
7415 ClientControlResponse::RouteOpen { .. }
7416 ),
7417 "the route must actually open, not merely avoid an error"
7418 );
7419 }
7420
7421 #[tokio::test(start_paused = true)]
7422 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
7423 let registry = Arc::new(Registry::default());
7424 let forwarding = Arc::new(ForwardingTable::default());
7425 let supervisor = SupervisorHandle::new();
7426 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7427 let handler =
7428 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
7429 .with_supervisor(supervisor);
7430
7431 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
7432 hello_via_sink(
7433 &handler,
7434 &target_ctx,
7435 &mut target_rx,
7436 hello_frame("target", PROTOCOL_VERSION, 1),
7437 )
7438 .await;
7439
7440 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
7441 let direct_handler = handler.clone();
7442 let direct_open = tokio::spawn(async move {
7443 direct_handler
7444 .handle_control_frame(
7445 &direct_ctx,
7446 route_open_frame(2, "target", unique_project_root("route-census-direct")),
7447 )
7448 .await
7449 .unwrap()
7450 });
7451 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7452 .await
7453 .expect("no direct route.bind within 5s")
7454 .expect("target control channel closed before direct route.bind");
7455 handler
7456 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
7457 .await
7458 .unwrap();
7459 assert!(direct_open.await.unwrap().is_empty());
7460 let _ = direct_rx.recv().await.unwrap();
7461
7462 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
7463 let reserved_handler = handler.clone();
7464 let reserved_open = tokio::spawn(async move {
7465 reserved_handler
7466 .handle_control_frame(
7467 &reserved_ctx,
7468 route_open_frame_with_admission_facts(
7469 3,
7470 "target",
7471 unique_project_root("admission-facts"),
7472 Some(ConsumerIdentity {
7473 module_id: "fed".to_string(),
7474 launch_nonce: "fed-nonce".to_string(),
7475 }),
7476 None,
7477 ),
7478 )
7479 .await
7480 .unwrap()
7481 });
7482 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7483 .await
7484 .expect("no reserved route.bind within 5s")
7485 .expect("target control channel closed before reserved route.bind");
7486 handler
7487 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
7488 .await
7489 .unwrap();
7490 assert!(reserved_open.await.unwrap().is_empty());
7491 let _ = reserved_rx.recv().await.unwrap();
7492
7493 forwarding
7494 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
7495 .unwrap();
7496 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
7497 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
7498 module_id: Some("target".to_string()),
7499 })
7500 .unwrap();
7501 let census_frame =
7502 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
7503 let response = handler
7504 .handle_control_frame(&census_ctx, census_frame)
7505 .await
7506 .unwrap()
7507 .pop()
7508 .unwrap();
7509 let actual: Value = serde_json::from_slice(&response.body).unwrap();
7510 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
7511 assert!(matches!(
7512 decoded,
7513 ClientControlResponse::SupervisorRoutes { .. }
7514 ));
7515 let routes = actual["modules"][0]["routes"].as_array().unwrap();
7516 assert_eq!(routes.len(), 2);
7517 assert!(routes.iter().all(|route| route["draining"] == true));
7518 assert!(
7521 routes.iter().all(|route| route["drain_reason"] == "reload"),
7522 "draining routes must name the drain's reason: {routes:?}"
7523 );
7524 assert!(routes.iter().any(|route| {
7525 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
7526 }));
7527 assert!(routes.iter().any(|route| {
7528 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
7529 }));
7530
7531 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
7532 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
7533 if std::env::var_os("UPDATE_GOLDEN").is_some() {
7534 std::fs::write(
7535 &golden_path,
7536 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
7537 )
7538 .unwrap();
7539 }
7540 let expected: Value =
7541 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
7542 assert_eq!(actual, expected);
7543 }
7544
7545 async fn query_live_roots(
7546 handler: &ControlHandler,
7547 module_ctx: &RouteCtx,
7548 ) -> ModuleControlResponseToModule {
7549 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
7550 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
7551 let response = handler
7552 .handle_control_frame(module_ctx, frame)
7553 .await
7554 .unwrap()
7555 .pop()
7556 .unwrap();
7557 serde_json::from_slice(&response.body).unwrap()
7558 }
7559
7560 #[tokio::test(start_paused = true)]
7561 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
7562 let registry = Arc::new(Registry::default());
7563 let forwarding = Arc::new(ForwardingTable::default());
7564 let handler = ControlHandler::with_forwarding(registry, forwarding);
7565 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
7566 hello_via_sink(
7567 &handler,
7568 &target_ctx,
7569 &mut target_rx,
7570 hello_frame("target", PROTOCOL_VERSION, 1),
7571 )
7572 .await;
7573 let root = unique_project_root("live-roots-known");
7574 let path = ProjectRootId::from_path_allowing_missing(root.path())
7575 .unwrap()
7576 .as_path()
7577 .to_path_buf();
7578 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
7579 let open_handler = handler.clone();
7580 let opened = tokio::spawn(async move {
7581 open_handler
7582 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
7583 .await
7584 .unwrap()
7585 });
7586 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7587 .await
7588 .unwrap()
7589 .unwrap();
7590 handler
7591 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
7592 .await
7593 .unwrap();
7594 assert!(opened.await.unwrap().is_empty());
7595 let _ = client_rx.recv().await.unwrap();
7596
7597 let root = unique_project_root("live-roots-pending");
7598 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
7599 .unwrap()
7600 .as_path()
7601 .to_path_buf();
7602 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
7603 let open_handler = handler.clone();
7604 let pending = tokio::spawn(async move {
7605 open_handler
7606 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
7607 .await
7608 .unwrap()
7609 });
7610 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7611 .await
7612 .unwrap()
7613 .unwrap();
7614 let actual = query_live_roots(&handler, &target_ctx).await;
7615 let ModuleControlResponseToModule::LiveRoots {
7616 roots,
7617 unknown_root_bindings,
7618 total_bindings,
7619 } = actual
7620 else {
7621 panic!("expected live roots")
7622 };
7623 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
7624 assert_eq!(unknown_root_bindings, 0);
7625 assert_eq!(
7626 roots.len(),
7627 2,
7628 "root-known arm must retain each canonical root"
7629 );
7630 assert_eq!(
7631 total_bindings,
7632 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
7633 );
7634 let counts = roots
7635 .iter()
7636 .map(|root| (root.project_root.clone(), root.bound, root.pending))
7637 .collect::<Vec<_>>();
7638 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
7639 expected.sort_by(|a, b| a.0.cmp(&b.0));
7640 assert_eq!(
7641 counts, expected,
7642 "roots must sort by path and count pending separately"
7643 );
7644 handler
7645 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
7646 .await
7647 .unwrap();
7648 assert!(pending.await.unwrap().is_empty());
7649 }
7650
7651 #[tokio::test(start_paused = true)]
7652 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
7653 let forwarding = Arc::new(ForwardingTable::default());
7654 let handler =
7655 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
7656 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
7657 hello_via_sink(
7658 &handler,
7659 &target_ctx,
7660 &mut target_rx,
7661 hello_frame("target", PROTOCOL_VERSION, 1),
7662 )
7663 .await;
7664 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
7665 let pending = forwarding
7666 .begin_route_bind_relay_for_test(
7667 client_ctx.connection_id,
7668 client_ctx.egress.clone(),
7669 2,
7670 "target",
7671 )
7672 .unwrap();
7673 forwarding
7674 .complete_pending_relay(
7675 target_ctx.connection_id,
7676 pending.corr,
7677 RouteBindRelayOutcome::Accepted,
7678 )
7679 .unwrap();
7680 let actual = query_live_roots(&handler, &target_ctx).await;
7681 let ModuleControlResponseToModule::LiveRoots {
7682 roots,
7683 unknown_root_bindings,
7684 total_bindings,
7685 } = actual
7686 else {
7687 panic!("expected live roots")
7688 };
7689 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
7690 assert_eq!(
7691 unknown_root_bindings, 1,
7692 "unknown-root arm must not read as no bindings"
7693 );
7694 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
7695 assert_eq!(
7696 total_bindings,
7697 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
7698 );
7699 }
7700
7701 #[tokio::test(start_paused = true)]
7709 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
7710 let forwarding = Arc::new(ForwardingTable::default());
7711 let handler =
7712 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
7713 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
7714 let replies = handler
7715 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
7716 .await
7717 .unwrap();
7718 let queued_by_hello = module_rx.len();
7719
7720 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
7721 let open_handler = handler.clone();
7722 let open = tokio::spawn(async move {
7723 open_handler
7724 .handle_control_frame(
7725 &client_ctx,
7726 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
7727 )
7728 .await
7729 .unwrap()
7730 });
7731 let mut spins = 0;
7733 while module_rx.len() == queued_by_hello {
7734 spins += 1;
7735 assert!(spins < 10_000, "route.open never queued a route.bind");
7736 tokio::task::yield_now().await;
7737 }
7738
7739 for reply in replies {
7741 module_ctx.egress.send(reply).await.unwrap();
7742 }
7743
7744 let first = module_rx.recv().await.unwrap().frame;
7745 assert_eq!(
7746 first.header.ty,
7747 FrameType::HelloAck,
7748 "the first frame a registering module reads must be its HELLO_ACK"
7749 );
7750 assert_eq!(first.header.corr, 7);
7751 let second = module_rx.recv().await.unwrap().frame;
7752 assert_eq!(second.header.ty, FrameType::Request);
7753 assert!(
7754 matches!(
7755 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
7756 ModuleControlRequest::RouteBind { .. }
7757 ),
7758 "the route.bind follows the ack"
7759 );
7760 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
7761
7762 handler
7763 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
7764 .await
7765 .unwrap();
7766 assert!(open.await.unwrap().is_empty());
7767 let _ = client_rx.recv().await.unwrap();
7768 }
7769
7770 #[tokio::test(start_paused = true)]
7771 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
7772 let handler = ControlHandler::with_forwarding(
7773 Arc::new(Registry::default()),
7774 Arc::new(ForwardingTable::default()),
7775 );
7776 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
7777 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
7778 hello_via_sink(
7779 &handler,
7780 &first_ctx,
7781 &mut first_rx,
7782 hello_frame("first", PROTOCOL_VERSION, 1),
7783 )
7784 .await;
7785 hello_via_sink(
7786 &handler,
7787 &second_ctx,
7788 &mut second_rx,
7789 hello_frame("second", PROTOCOL_VERSION, 2),
7790 )
7791 .await;
7792 let root = unique_project_root("second-only");
7793 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
7794 let cloned = handler.clone();
7795 let open = tokio::spawn(async move {
7796 cloned
7797 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
7798 .await
7799 .unwrap()
7800 });
7801 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
7802 .await
7803 .unwrap()
7804 .unwrap();
7805 let first = query_live_roots(&handler, &first_ctx).await;
7806 let second = query_live_roots(&handler, &second_ctx).await;
7807 assert!(
7808 matches!(
7809 first,
7810 ModuleControlResponseToModule::LiveRoots {
7811 total_bindings: 0,
7812 ..
7813 }
7814 ),
7815 "cross-module scope must not expose another module's roots"
7816 );
7817 assert!(
7818 matches!(
7819 second,
7820 ModuleControlResponseToModule::LiveRoots {
7821 total_bindings: 1,
7822 ..
7823 }
7824 ),
7825 "second module must see its pending route"
7826 );
7827 handler
7828 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
7829 .await
7830 .unwrap();
7831 assert!(open.await.unwrap().is_empty());
7832 }
7833
7834 #[tokio::test(start_paused = true)]
7835 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
7836 let handler = ControlHandler::with_forwarding(
7837 Arc::new(Registry::default()),
7838 Arc::new(ForwardingTable::default()),
7839 );
7840 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
7841 hello_via_sink(
7842 &handler,
7843 &target_ctx,
7844 &mut target_rx,
7845 hello_frame("target", PROTOCOL_VERSION, 1),
7846 )
7847 .await;
7848 let actual = query_live_roots(&handler, &target_ctx).await;
7849 let ModuleControlResponseToModule::LiveRoots {
7850 roots,
7851 unknown_root_bindings,
7852 total_bindings,
7853 } = actual
7854 else {
7855 panic!("expected live roots")
7856 };
7857 assert!(roots.is_empty());
7858 assert_eq!(unknown_root_bindings, 0);
7859 assert_eq!(total_bindings, 0);
7860 assert_eq!(
7861 total_bindings,
7862 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
7863 );
7864 }
7865
7866 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
7873 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
7874 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
7875 let text = std::fs::read_to_string(&path)
7876 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
7877 let vectors: Vec<(String, Value)> = text
7878 .lines()
7879 .filter(|line| !line.trim().is_empty())
7880 .map(|line| {
7881 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
7882 let id = entry["corpus_id"]
7883 .as_str()
7884 .expect("every vector carries a corpus_id")
7885 .to_string();
7886 (id, entry["package"].clone())
7887 })
7888 .collect();
7889 assert_eq!(
7893 vectors.len(),
7894 3,
7895 "vendored fed corpus changed size; re-sync from subc-federation"
7896 );
7897
7898 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
7913 let richest = vectors
7914 .iter()
7915 .filter_map(|(_, package)| package.as_object())
7916 .map(|object| {
7917 object
7918 .keys()
7919 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
7920 .count()
7921 })
7922 .max()
7923 .unwrap_or(0);
7924 assert!(
7925 richest >= 2,
7926 "vendored corpus no longer carries a package with unmodelled fields, \
7927 so the relay test can no longer distinguish a verbatim relay from a lossy one"
7928 );
7929
7930 vectors
7931 }
7932
7933 #[tokio::test]
7941 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
7942 for (corpus_id, package) in fed_admission_facts_vectors() {
7943 let registry = Arc::new(Registry::default());
7944 let forwarding = Arc::new(ForwardingTable::default());
7945 let supervisor = SupervisorHandle::new();
7946 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7947 let handler =
7948 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
7949 .with_supervisor(supervisor)
7950 .with_admission_facts_config(
7951 Some("fed".to_string()),
7952 Some(vec!["target".to_string()]),
7953 );
7954
7955 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
7956 hello_via_sink(
7957 &handler,
7958 &target_ctx,
7959 &mut target_rx,
7960 hello_frame("target", PROTOCOL_VERSION, 1),
7961 )
7962 .await;
7963
7964 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
7965 let route_handler = handler.clone();
7966 let expected = package.clone();
7967 let route_task = tokio::spawn(async move {
7968 route_handler
7969 .handle_control_frame(
7970 &client_ctx,
7971 route_open_frame_with_admission_facts(
7972 20,
7973 "target",
7974 unique_project_root("admission-facts"),
7975 Some(subc_control::ConsumerIdentity {
7976 module_id: "fed".to_string(),
7977 launch_nonce: "fed-nonce".to_string(),
7978 }),
7979 Some(package),
7980 ),
7981 )
7982 .await
7983 .unwrap()
7984 });
7985
7986 let bind_frame = target_rx.recv().await.unwrap();
7987 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
7988 let ModuleControlRequest::RouteBind {
7989 admission_facts, ..
7990 } = bind
7991 else {
7992 panic!("{corpus_id}: expected route.bind")
7993 };
7994 assert_eq!(
7995 admission_facts,
7996 Some(expected),
7997 "{corpus_id}: relay must not add, drop or reshape any field"
7998 );
7999
8000 handler
8001 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8002 .await
8003 .unwrap();
8004 route_task.await.unwrap();
8005 }
8006 }
8007
8008 #[tokio::test]
8009 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
8010 let registry = Arc::new(Registry::default());
8011 let forwarding = Arc::new(ForwardingTable::default());
8012 let supervisor = SupervisorHandle::new();
8013 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8014 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
8015 let handler =
8016 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8017 .with_supervisor(supervisor)
8018 .with_admission_facts_config(
8019 Some("fed".to_string()),
8020 Some(vec!["target".to_string()]),
8021 );
8022
8023 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
8024 hello_via_sink(
8025 &handler,
8026 &target_ctx,
8027 &mut target_rx,
8028 hello_frame("target", PROTOCOL_VERSION, 1),
8029 )
8030 .await;
8031 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
8032 hello_via_sink(
8033 &handler,
8034 &other_ctx,
8035 &mut other_rx,
8036 hello_frame("other", PROTOCOL_VERSION, 2),
8037 )
8038 .await;
8039
8040 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
8041 let expected_facts = facts.clone();
8042 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
8043 let route_handler = handler.clone();
8044 let route_task = tokio::spawn(async move {
8045 route_handler
8046 .handle_control_frame(
8047 &client_ctx,
8048 route_open_frame_with_admission_facts(
8049 10,
8050 "target",
8051 unique_project_root("admission-facts"),
8052 Some(subc_control::ConsumerIdentity {
8053 module_id: "fed".to_string(),
8054 launch_nonce: "fed-nonce".to_string(),
8055 }),
8056 Some(facts.clone()),
8057 ),
8058 )
8059 .await
8060 .unwrap()
8061 });
8062 let bind_frame = target_rx.recv().await.unwrap();
8063 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8064 let ModuleControlRequest::RouteBind {
8065 admission_facts, ..
8066 } = bind
8067 else {
8068 panic!("expected route.bind")
8069 };
8070 assert_eq!(admission_facts, Some(expected_facts));
8071 handler
8072 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8073 .await
8074 .unwrap();
8075 assert!(route_task.await.unwrap().is_empty());
8076 assert!(matches!(
8077 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
8078 .unwrap(),
8079 ClientControlResponse::RouteOpen { .. }
8080 ));
8081
8082 let direct = handler
8083 .handle_control_frame(
8084 &route_ctx(ConnectionId::new(73)).0,
8085 route_open_frame_with_admission_facts(
8086 11,
8087 "target",
8088 unique_project_root("admission-facts"),
8089 None,
8090 Some(json!({"x": 1})),
8091 ),
8092 )
8093 .await
8094 .unwrap();
8095 assert_eq!(
8096 parse_error(&direct[0])["code"],
8097 "admission_facts_not_permitted"
8098 );
8099
8100 let different_reserved = handler
8101 .handle_control_frame(
8102 &route_ctx(ConnectionId::new(77)).0,
8103 route_open_frame_with_admission_facts(
8104 15,
8105 "target",
8106 unique_project_root("admission-facts"),
8107 Some(subc_control::ConsumerIdentity {
8108 module_id: "other".to_string(),
8109 launch_nonce: "other-nonce".to_string(),
8110 }),
8111 Some(json!({"x": 1})),
8112 ),
8113 )
8114 .await
8115 .unwrap();
8116 assert_eq!(
8117 parse_error(&different_reserved[0])["code"],
8118 "admission_facts_not_permitted"
8119 );
8120
8121 let other_target = handler
8122 .handle_control_frame(
8123 &route_ctx(ConnectionId::new(74)).0,
8124 route_open_frame_with_admission_facts(
8125 12,
8126 "other",
8127 unique_project_root("admission-facts"),
8128 Some(subc_control::ConsumerIdentity {
8129 module_id: "fed".to_string(),
8130 launch_nonce: "fed-nonce".to_string(),
8131 }),
8132 Some(json!({"x": 1})),
8133 ),
8134 )
8135 .await
8136 .unwrap();
8137 assert_eq!(
8138 parse_error(&other_target[0])["code"],
8139 "admission_facts_target_not_allowed"
8140 );
8141
8142 let nonexistent = handler
8143 .handle_control_frame(
8144 &route_ctx(ConnectionId::new(75)).0,
8145 route_open_frame_with_admission_facts(
8146 13,
8147 "missing",
8148 unique_project_root("admission-facts"),
8149 None,
8150 Some(json!({"x": 1})),
8151 ),
8152 )
8153 .await
8154 .unwrap();
8155 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
8156
8157 let described = handler
8158 .handle_control_frame(
8159 &route_ctx(ConnectionId::new(76)).0,
8160 Frame::build(
8161 FrameType::Request,
8162 control_flags(),
8163 0,
8164 0,
8165 14,
8166 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
8167 )
8168 .unwrap(),
8169 )
8170 .await
8171 .unwrap();
8172 let ClientControlResponse::ServerDescribe { capabilities, .. } =
8173 serde_json::from_slice(&described[0].body).unwrap()
8174 else {
8175 panic!("expected server.describe response")
8176 };
8177 assert!(capabilities
8178 .iter()
8179 .any(|cap| cap == "admission_facts_relay_v1"));
8180 }
8181
8182 #[tokio::test]
8183 async fn admission_facts_without_configured_carrier_are_rejected() {
8184 let registry = Arc::new(Registry::default());
8185 let forwarding = Arc::new(ForwardingTable::default());
8186 let handler = ControlHandler::with_forwarding(registry, forwarding);
8187 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
8188 hello_via_sink(
8189 &handler,
8190 &target_ctx,
8191 &mut target_rx,
8192 hello_frame("target", PROTOCOL_VERSION, 1),
8193 )
8194 .await;
8195
8196 let responses = handler
8197 .handle_control_frame(
8198 &route_ctx(ConnectionId::new(79)).0,
8199 route_open_frame_with_admission_facts(
8200 16,
8201 "target",
8202 unique_project_root("admission-facts"),
8203 None,
8204 Some(json!({"x": 1})),
8205 ),
8206 )
8207 .await
8208 .unwrap();
8209 assert_eq!(
8210 parse_error(&responses[0])["code"],
8211 "admission_facts_not_permitted"
8212 );
8213 }
8214
8215 #[tokio::test]
8216 async fn route_open_relays_consumer_capabilities_verbatim() {
8217 let registry = Arc::new(Registry::default());
8218 let forwarding = Arc::new(ForwardingTable::default());
8219 let handler =
8220 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8221 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
8222 hello_via_sink(
8223 &handler,
8224 &module_ctx,
8225 &mut module_rx,
8226 hello_frame("aft", PROTOCOL_VERSION, 7),
8227 )
8228 .await;
8229
8230 let expected = vec!["elicitation".to_string(), "roots".to_string()];
8231 let expected_for_request = expected.clone();
8232 let project_root = unique_project_root("consumer-capabilities-present");
8233 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
8234 let route_handler = handler.clone();
8235 let route_task = tokio::spawn(async move {
8236 route_handler
8237 .handle_control_frame(
8238 &client_ctx,
8239 route_open_frame_with_consumer_capabilities(
8240 401,
8241 "aft",
8242 project_root,
8243 Some(expected_for_request),
8244 ),
8245 )
8246 .await
8247 .unwrap()
8248 });
8249 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8250 .await
8251 .unwrap()
8252 .unwrap();
8253 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8254 let ModuleControlRequest::RouteBind {
8255 consumer_capabilities,
8256 ..
8257 } = bind
8258 else {
8259 panic!("expected route.bind request, got {bind:?}");
8260 };
8261 assert_eq!(consumer_capabilities, Some(expected.clone()));
8262
8263 handler
8264 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8265 .await
8266 .unwrap();
8267 let route_response = route_task.await.unwrap();
8268 assert!(route_response.is_empty());
8269 let published = client_rx.recv().await.unwrap();
8270 assert!(matches!(
8271 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8272 ClientControlResponse::RouteOpen { .. }
8273 ));
8274 }
8275
8276 #[tokio::test]
8277 async fn route_open_without_consumer_capabilities_relays_none() {
8278 let registry = Arc::new(Registry::default());
8279 let forwarding = Arc::new(ForwardingTable::default());
8280 let handler =
8281 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8282 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
8283 hello_via_sink(
8284 &handler,
8285 &module_ctx,
8286 &mut module_rx,
8287 hello_frame("aft", PROTOCOL_VERSION, 7),
8288 )
8289 .await;
8290
8291 let project_root = unique_project_root("consumer-capabilities-absent");
8292 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
8293 let route_handler = handler.clone();
8294 let route_task = tokio::spawn(async move {
8295 route_handler
8296 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
8297 .await
8298 .unwrap()
8299 });
8300 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8301 .await
8302 .unwrap()
8303 .unwrap();
8304 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8305 let ModuleControlRequest::RouteBind {
8306 consumer_capabilities,
8307 ..
8308 } = bind
8309 else {
8310 panic!("expected route.bind request, got {bind:?}");
8311 };
8312 assert_eq!(consumer_capabilities, None);
8313
8314 handler
8315 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8316 .await
8317 .unwrap();
8318 let route_response = route_task.await.unwrap();
8319 assert!(route_response.is_empty());
8320 let published = client_rx.recv().await.unwrap();
8321 assert!(matches!(
8322 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8323 ClientControlResponse::RouteOpen { .. }
8324 ));
8325 }
8326
8327 #[tokio::test]
8328 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
8329 let registry = Arc::new(Registry::default());
8330 let forwarding = Arc::new(ForwardingTable::default());
8331 let handler =
8332 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8333 .with_health_probe_timeout(Duration::from_secs(5));
8334 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
8335 hello_via_sink(
8336 &handler,
8337 &module_ctx,
8338 &mut module_rx,
8339 non_routable_hello_frame_with_control_ops(
8340 "mcp",
8341 300,
8342 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
8343 ),
8344 )
8345 .await;
8346 assert!(registry
8347 .get_module("mcp")
8348 .unwrap()
8349 .unwrap()
8350 .manifest
8351 .provides
8352 .is_empty());
8353
8354 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
8355 let route_response = handler
8356 .handle_control_frame(
8357 &route_client_ctx,
8358 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
8359 )
8360 .await
8361 .unwrap();
8362 assert_eq!(route_response[0].header.ty, FrameType::Error);
8363 assert_eq!(
8364 parse_error(&route_response[0])["code"],
8365 "target_unavailable"
8366 );
8367 assert!(parse_error(&route_response[0])["message"]
8368 .as_str()
8369 .unwrap()
8370 .contains("does not provide the requested target"));
8371 assert!(module_rx.try_recv().is_err());
8372
8373 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
8374 let health_handler = handler.clone();
8375 let health_task = tokio::spawn(async move {
8376 health_handler
8377 .handle_control_frame(
8378 &health_client_ctx,
8379 supervisor_health_probe_frame(302, "mcp"),
8380 )
8381 .await
8382 .unwrap()
8383 });
8384 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8385 .await
8386 .unwrap()
8387 .unwrap();
8388 assert_eq!(
8389 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
8390 ModuleControlRequest::HealthCheck {}
8391 );
8392 handler
8393 .handle_control_frame(
8394 &module_ctx,
8395 health_response(health_frame.header.corr, HealthStatus::Ok),
8396 )
8397 .await
8398 .unwrap();
8399 let health_response = health_task.await.unwrap();
8400 assert_eq!(health_response[0].header.ty, FrameType::Response);
8401 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
8402 ClientControlResponse::SupervisorHealthProbe {
8403 module_id, status, ..
8404 } => {
8405 assert_eq!(module_id, "mcp");
8406 assert_eq!(status, HealthStatus::Ok);
8407 }
8408 other => panic!("unexpected health response: {other:?}"),
8409 }
8410
8411 forwarding
8416 .cleanup_connection(module_ctx.connection_id)
8417 .unwrap();
8418 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
8419 let cleanup_response = tokio::time::timeout(
8420 Duration::from_millis(200),
8421 handler.handle_control_frame(
8422 &cleanup_probe_ctx,
8423 supervisor_health_probe_frame(303, "mcp"),
8424 ),
8425 )
8426 .await
8427 .expect("probe should fail immediately when the control lane is gone")
8428 .unwrap();
8429 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
8430 assert_eq!(
8431 parse_error(&cleanup_response[0])["code"],
8432 "target_unavailable"
8433 );
8434 assert!(parse_error(&cleanup_response[0])["message"]
8435 .as_str()
8436 .unwrap()
8437 .contains("no module connection"));
8438
8439 handler
8440 .cleanup_connection(module_ctx.connection_id)
8441 .unwrap();
8442 }
8443
8444 #[tokio::test]
8445 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
8446 let registry = Arc::new(Registry::default());
8447 let supervisor_handle = SupervisorHandle::new();
8448 let supervisor =
8449 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
8450 .with_handle(supervisor_handle.clone())
8451 .with_connection_file_path(
8452 std::env::temp_dir()
8453 .join(format!("subc-route-open-warming-{}", std::process::id())),
8454 );
8455 let module = supervisor
8456 .supervise_configured(
8457 ModuleSpec {
8458 module_id: "warming".to_string(),
8459 program: fake_aft_stub_path(),
8460 args: Vec::new(),
8461 env: Vec::new(),
8462 reserved: false,
8463 reserved_prefixes: Vec::new(),
8464 protocol: ModuleProtocol::Subc,
8465 overlap: Default::default(),
8466 },
8467 true,
8468 )
8469 .unwrap();
8470 assert_eq!(module.state().unwrap(), ModuleState::Running);
8471
8472 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8473 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
8474 let response = handler
8475 .handle_control_frame(
8476 &ctx,
8477 route_open_frame(304, "warming", unique_project_root("warming")),
8478 )
8479 .await
8480 .unwrap();
8481 module.stop().await.unwrap();
8482
8483 assert_eq!(response[0].header.ty, FrameType::Error);
8484 let error = parse_error(&response[0]);
8485 assert_eq!(error["code"], "module_warming");
8486 assert!(error["message"]
8487 .as_str()
8488 .unwrap()
8489 .contains("state=running, enabled=true, live=false"));
8490 }
8491
8492 #[tokio::test]
8497 async fn route_open_refusal_names_the_check_that_refused() {
8498 let handler = ControlHandler::new(Arc::new(Registry::default()));
8499 let capture = EventCapture::default();
8500 let _subscriber =
8501 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8502 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
8503 let response = handler
8504 .handle_control_frame(
8505 &ctx,
8506 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
8507 )
8508 .await
8509 .unwrap();
8510
8511 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
8512 let event = capture
8513 .events()
8514 .into_iter()
8515 .find(|event| {
8516 event.target == "control"
8517 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
8518 })
8519 .expect("route.open refusal event");
8520 assert_eq!(
8521 event.fields.get("reason"),
8522 Some(&"\"not_registered\"".to_string())
8523 );
8524 }
8525
8526 #[tokio::test]
8527 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
8528 let registry = Arc::new(Registry::default());
8529 let supervisor_handle = SupervisorHandle::new();
8530 let supervisor =
8531 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
8532 .with_handle(supervisor_handle.clone())
8533 .with_connection_file_path(std::env::temp_dir().join(format!(
8534 "subc-route-open-refusal-info-{}",
8535 std::process::id()
8536 )));
8537 let module = supervisor
8538 .supervise_configured(
8539 ModuleSpec {
8540 module_id: "warming".to_string(),
8541 program: fake_aft_stub_path(),
8542 args: Vec::new(),
8543 env: Vec::new(),
8544 reserved: false,
8545 reserved_prefixes: Vec::new(),
8546 protocol: ModuleProtocol::Subc,
8547 overlap: Default::default(),
8548 },
8549 true,
8550 )
8551 .unwrap();
8552 assert_eq!(module.state().unwrap(), ModuleState::Running);
8553
8554 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8555 assert!(handler
8556 .counters()
8557 .snapshot()
8558 .get("route_open_refused_by_code")
8559 .is_none());
8560 let capture = EventCapture::default();
8561 let _subscriber =
8562 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8563 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
8564 let response = handler
8565 .handle_control_frame(
8566 &ctx,
8567 route_open_frame(394, "warming", unique_project_root("refusal-info")),
8568 )
8569 .await
8570 .unwrap();
8571 module.stop().await.unwrap();
8572
8573 assert_eq!(parse_error(&response[0])["code"], "module_warming");
8574 let event = capture
8575 .events()
8576 .into_iter()
8577 .find(|event| {
8578 event.target == "control"
8579 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
8580 })
8581 .expect("route.open refusal event");
8582 assert_eq!(
8583 event.fields.get("module_id"),
8584 Some(&"\"warming\"".to_string())
8585 );
8586 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
8587 assert_eq!(
8588 event.fields.get("reason"),
8589 Some(&"\"supervised_not_registered\"".to_string())
8590 );
8591 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
8592 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
8593 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
8594 assert_eq!(
8595 handler.counters().snapshot()["route_open_refused_by_code"],
8596 json!({ "module_warming": 1 })
8597 );
8598 }
8599
8600 #[tokio::test(flavor = "current_thread")]
8601 async fn route_open_unknown_module_escapes_target_module_id() {
8602 let handler = ControlHandler::new(Arc::new(Registry::default()));
8603 let capture = EventCapture::default();
8604 let _subscriber =
8605 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8606 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
8607 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
8608 let response = handler
8609 .handle_control_frame(
8610 &ctx,
8611 route_open_frame(
8612 395,
8613 hostile_module_id,
8614 unique_project_root("hostile-target-module-id"),
8615 ),
8616 )
8617 .await
8618 .unwrap();
8619
8620 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
8621 let event = capture
8622 .events()
8623 .into_iter()
8624 .find(|event| {
8625 event.target == "control"
8626 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
8627 })
8628 .expect("route.open unknown-module refusal event");
8629 let logged = event.fields.get("module_id").expect("module_id field");
8630 assert!(!logged.bytes().any(|byte| byte < 0x20));
8631 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
8632 }
8633
8634 #[tokio::test(flavor = "current_thread")]
8635 async fn route_open_module_rejection_uses_daemon_counter_key() {
8636 let registry = Arc::new(Registry::default());
8637 let forwarding = Arc::new(ForwardingTable::default());
8638 let handler =
8639 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8640 let module_connection = ConnectionId::new(95);
8641 let (module_ctx, mut module_rx) = route_ctx(module_connection);
8642 hello_via_sink(
8643 &handler,
8644 &module_ctx,
8645 &mut module_rx,
8646 hello_frame("aft", PROTOCOL_VERSION, 395),
8647 )
8648 .await;
8649
8650 let client_connection = ConnectionId::new(96);
8651 let (client_ctx, _client_rx) = route_ctx(client_connection);
8652 let capture = EventCapture::default();
8653 let _subscriber =
8654 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8655 let (route_task, bind) = relay_route_open(
8656 &handler,
8657 client_connection,
8658 &client_ctx.egress,
8659 &mut module_rx,
8660 396,
8661 "aft",
8662 "hostile-module-code",
8663 )
8664 .await;
8665 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
8666 let rejection = Frame::build(
8667 FrameType::Error,
8668 control_flags(),
8669 0,
8670 0,
8671 bind.header.corr,
8672 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
8673 )
8674 .unwrap();
8675 handler
8676 .handle_control_frame(&module_ctx, rejection)
8677 .await
8678 .unwrap();
8679
8680 let response = route_task.await.unwrap();
8681 assert_eq!(parse_error(&response[0])["code"], hostile_code);
8682 let counters = handler.counters().snapshot();
8683 assert_eq!(
8684 counters["route_open_refused_by_code"],
8685 json!({ "module_rejected": 1 })
8686 );
8687 assert!(counters["route_open_refused_by_code"]
8688 .get(hostile_code)
8689 .is_none());
8690
8691 let event = capture
8692 .events()
8693 .into_iter()
8694 .find(|event| {
8695 event.target == "control"
8696 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
8697 })
8698 .expect("route.open module-rejection refusal event");
8699 let logged = event.fields.get("module_code").expect("module_code field");
8700 assert!(!logged.bytes().any(|byte| byte < 0x20));
8701 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
8702 }
8703
8704 #[tokio::test]
8705 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
8706 let registry = Arc::new(Registry::default());
8707 let supervisor_handle = SupervisorHandle::new();
8708 let missing_program = std::env::temp_dir().join(format!(
8709 "subc-route-open-missing-program-{}",
8710 std::process::id()
8711 ));
8712 let supervisor =
8713 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
8714 .with_handle(supervisor_handle.clone());
8715 let module = supervisor
8716 .supervise_configured(
8717 ModuleSpec {
8718 module_id: "failed".to_string(),
8719 program: missing_program,
8720 args: Vec::new(),
8721 env: Vec::new(),
8722 reserved: false,
8723 reserved_prefixes: Vec::new(),
8724 protocol: ModuleProtocol::Subc,
8725 overlap: Default::default(),
8726 },
8727 true,
8728 )
8729 .unwrap();
8730 assert_eq!(module.state().unwrap(), ModuleState::Failed);
8731
8732 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8733 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
8734 let response = handler
8735 .handle_control_frame(
8736 &ctx,
8737 route_open_frame(305, "failed", unique_project_root("failed")),
8738 )
8739 .await
8740 .unwrap();
8741
8742 assert_eq!(response[0].header.ty, FrameType::Error);
8743 let error = parse_error(&response[0]);
8744 assert_eq!(error["code"], "target_unavailable");
8745 assert!(error["message"]
8746 .as_str()
8747 .unwrap()
8748 .contains("state=failed, enabled=true, live=false"));
8749 }
8750
8751 #[tokio::test]
8752 async fn route_open_role_mismatch_remains_target_unavailable() {
8753 let registry = Arc::new(Registry::default());
8754 let handler = ControlHandler::new(Arc::clone(®istry));
8755 handler
8756 .handle_control(
8757 ConnectionId::new(41),
8758 non_routable_hello_frame_with_control_ops("health-only", 306, None),
8759 )
8760 .unwrap();
8761
8762 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
8763 let response = handler
8764 .handle_control_frame(
8765 &ctx,
8766 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
8767 )
8768 .await
8769 .unwrap();
8770
8771 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
8772 assert!(parse_error(&response[0])["message"]
8773 .as_str()
8774 .unwrap()
8775 .contains("does not provide the requested target"));
8776 }
8777
8778 #[tokio::test]
8779 async fn route_open_inactive_registration_remains_target_unavailable() {
8780 let registry = Arc::new(Registry::default());
8781 let handler = ControlHandler::new(Arc::clone(®istry));
8782 handler
8783 .handle_control(
8784 ConnectionId::new(43),
8785 hello_frame("inactive", PROTOCOL_VERSION, 308),
8786 )
8787 .unwrap();
8788 assert!(registry
8789 .set_module_state_for_test("inactive", ChannelState::Closed)
8790 .unwrap());
8791
8792 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
8793 let response = handler
8794 .handle_control_frame(
8795 &ctx,
8796 route_open_frame(309, "inactive", unique_project_root("inactive")),
8797 )
8798 .await
8799 .unwrap();
8800
8801 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
8802 assert!(parse_error(&response[0])["message"]
8803 .as_str()
8804 .unwrap()
8805 .contains("is not active"));
8806 }
8807
8808 #[tokio::test]
8809 async fn late_health_reply_is_recorded_through_the_module_response_path() {
8810 let registry = Arc::new(Registry::default());
8811 let forwarding = Arc::new(ForwardingTable::default());
8812 let supervisor_handle = SupervisorHandle::new();
8813 let supervisor = Supervisor::new(Arc::clone(®istry), crate::RestartPolicy::default())
8814 .with_forwarding(Arc::clone(&forwarding))
8815 .with_handle(supervisor_handle.clone());
8816 let module = supervisor
8817 .supervise_configured(
8818 crate::ModuleSpec {
8819 module_id: "late-health-response".to_string(),
8820 program: PathBuf::from("disabled-module"),
8821 args: Vec::new(),
8822 env: Vec::new(),
8823 reserved: false,
8824 reserved_prefixes: Vec::new(),
8825 protocol: ModuleProtocol::Subc,
8826 overlap: Default::default(),
8827 },
8828 false,
8829 )
8830 .unwrap();
8831 let handler =
8832 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8833 .with_supervisor(supervisor_handle);
8834 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
8835 handler
8836 .handle_control_frame(
8837 &module_ctx,
8838 hello_frame_with_control_ops(
8839 "late-health-response",
8840 PROTOCOL_VERSION,
8841 7,
8842 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
8843 ),
8844 )
8845 .await
8846 .unwrap();
8847 let probe_started_at = Instant::now() - Duration::from_millis(80);
8848 let pending = forwarding
8849 .begin_health_probe_rpc_for(
8850 "late-health-response",
8851 MODULE_CONTROL_OP_HEALTH_CHECK,
8852 probe_started_at,
8853 Instant::now() - Duration::from_millis(1),
8854 )
8855 .unwrap();
8856 assert!(forwarding
8857 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
8858 .unwrap());
8859
8860 let responses = handler
8861 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
8862 .await
8863 .unwrap();
8864
8865 assert!(responses.is_empty());
8866 let health = module.status().unwrap().health;
8867 assert_eq!(health.late_answer_count, 1);
8868 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
8869 }
8870
8871 #[tokio::test]
8872 async fn health_probe_timeout_and_module_death_are_typed() {
8873 let registry = Arc::new(Registry::default());
8874 let forwarding = Arc::new(ForwardingTable::default());
8875 let handler =
8876 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8877 .with_health_probe_timeout(Duration::from_millis(50));
8878 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
8879 hello_via_sink(
8880 &handler,
8881 &module_ctx,
8882 &mut module_rx,
8883 hello_frame_with_control_ops(
8884 "aft",
8885 PROTOCOL_VERSION,
8886 7,
8887 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
8888 ),
8889 )
8890 .await;
8891
8892 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
8893 let responses = handler
8894 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
8895 .await
8896 .unwrap();
8897 assert_eq!(responses[0].header.ty, FrameType::Error);
8898 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
8899 let _ = module_rx.try_recv();
8900
8901 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
8902 let health_handler = handler.clone();
8903 let death_task = tokio::spawn(async move {
8904 health_handler
8905 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
8906 .await
8907 .unwrap()
8908 });
8909 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8910 .await
8911 .unwrap()
8912 .unwrap();
8913 handler
8914 .cleanup_connection(module_ctx.connection_id)
8915 .unwrap();
8916 let responses = death_task.await.unwrap();
8917 assert_eq!(responses[0].header.ty, FrameType::Error);
8918 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
8919 }
8920
8921 #[test]
8922 fn hello_requires_exact_protocol_version() {
8923 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
8924 let registry = Arc::new(Registry::default());
8925 let handler = ControlHandler::new(Arc::clone(®istry));
8926 let responses = handler
8927 .handle_control(
8928 ConnectionId::new(connection),
8929 hello_frame("aft", offered, 9),
8930 )
8931 .unwrap();
8932
8933 assert_eq!(responses.len(), 1);
8934 assert_eq!(responses[0].header.ty, FrameType::Error);
8935 let error = parse_error(&responses[0]);
8936 assert_eq!(error["code"], "version_unsupported");
8937 assert!(registry.get_module("aft").unwrap().is_none());
8938 assert_eq!(registry.active_registration_count().unwrap(), 0);
8939 }
8940 }
8941
8942 #[test]
8943 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
8944 let registry = Arc::new(Registry::default());
8945 let forwarding = Arc::new(ForwardingTable::default());
8946 let handler =
8947 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8948 let module_connection = ConnectionId::new(301);
8949 let registration = registry
8950 .register_with_control_ops(
8951 manifest("aft-push", PROTOCOL_VERSION),
8952 PROTOCOL_VERSION,
8953 module_connection,
8954 module_baseline_control_ops(),
8955 )
8956 .unwrap();
8957 let (module_tx, _module_rx) = mpsc::channel(8);
8958 let endpoint = forwarding
8959 .register_module_connection(
8960 module_connection,
8961 "aft-push".to_string(),
8962 PROTOCOL_VERSION,
8963 manifest_concurrency(®istration.manifest),
8964 FrameSink::new(module_tx),
8965 )
8966 .unwrap();
8967
8968 let unknown = Frame::build(
8970 FrameType::Push,
8971 control_flags(),
8972 0,
8973 0,
8974 5,
8975 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
8976 )
8977 .unwrap();
8978 let out = handler.handle_status_update(endpoint, unknown).unwrap();
8979 assert!(
8980 out.is_empty(),
8981 "unknown push op must be ignored, got {out:?}"
8982 );
8983
8984 let malformed = Frame::build(
8986 FrameType::Push,
8987 control_flags(),
8988 0,
8989 0,
8990 6,
8991 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
8992 )
8993 .unwrap();
8994 let out = handler.handle_status_update(endpoint, malformed).unwrap();
8995 assert_eq!(out.len(), 1);
8996 assert_eq!(out[0].header.ty, FrameType::Error);
8997 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
8998 }
8999
9000 #[test]
9001 fn hello_rejected_when_connection_already_owns_client_routes() {
9002 let registry = Arc::new(Registry::default());
9003 let forwarding = Arc::new(ForwardingTable::default());
9004 let handler =
9005 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9006 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
9008 let client_connection = ConnectionId::new(202);
9009
9010 let responses = handler
9013 .handle_control(
9014 client_connection,
9015 hello_frame("aft-second", PROTOCOL_VERSION, 9),
9016 )
9017 .unwrap();
9018 assert_eq!(responses[0].header.ty, FrameType::Error);
9019 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
9020 assert!(registry.get_module("aft-second").unwrap().is_none());
9021 }
9022
9023 #[test]
9024 fn reserved_module_hello_requires_matching_launch_nonce() {
9025 let registry = Arc::new(Registry::default());
9026 let supervisor = SupervisorHandle::new();
9027 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
9030 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9031
9032 let no_nonce = handler
9034 .handle_control(
9035 ConnectionId::new(1),
9036 hello_frame("vault", PROTOCOL_VERSION, 1),
9037 )
9038 .unwrap();
9039 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
9040 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
9041 assert!(registry.get_module("vault").unwrap().is_none());
9042
9043 let wrong = handler
9045 .handle_control(
9046 ConnectionId::new(2),
9047 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
9048 )
9049 .unwrap();
9050 assert_eq!(wrong[0].header.ty, FrameType::Error);
9051 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
9052 assert!(registry.get_module("vault").unwrap().is_none());
9053
9054 let ok = handler
9056 .handle_control(
9057 ConnectionId::new(3),
9058 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
9059 )
9060 .unwrap();
9061 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
9062 assert!(registry.get_module("vault").unwrap().is_some());
9063 }
9064
9065 #[test]
9066 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
9067 let registry = Arc::new(Registry::default());
9068 let supervisor = SupervisorHandle::new();
9069 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
9070 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
9071 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9072
9073 let squat = handler
9074 .handle_control(
9075 ConnectionId::new(1),
9076 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
9077 )
9078 .unwrap();
9079 assert_eq!(squat[0].header.ty, FrameType::Error);
9080 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
9081 assert!(parse_error(&squat[0])["message"]
9082 .as_str()
9083 .unwrap()
9084 .contains("fed:"));
9085
9086 let accepted_peer = handler
9087 .handle_control(
9088 ConnectionId::new(2),
9089 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
9090 )
9091 .unwrap();
9092 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
9093
9094 let accepted_short = handler
9095 .handle_control(
9096 ConnectionId::new(3),
9097 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
9098 )
9099 .unwrap();
9100 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
9101
9102 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
9103 let response = handler
9104 .handle_control(
9105 ConnectionId::new(conn),
9106 hello_frame(module_id, PROTOCOL_VERSION, conn),
9107 )
9108 .unwrap();
9109 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
9110 }
9111 }
9112
9113 #[test]
9114 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
9115 let registry = Arc::new(Registry::default());
9116 let supervisor = SupervisorHandle::new();
9117 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
9118 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
9119 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
9120 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9121
9122 let owner_nonce = handler
9123 .handle_control(
9124 ConnectionId::new(1),
9125 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
9126 )
9127 .unwrap();
9128 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
9129 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
9130 assert!(registry.get_module("fed:special").unwrap().is_none());
9131
9132 let exact_nonce = handler
9133 .handle_control(
9134 ConnectionId::new(2),
9135 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
9136 )
9137 .unwrap();
9138 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
9139 assert!(registry.get_module("fed:special").unwrap().is_some());
9140 }
9141
9142 #[test]
9143 fn non_reserved_module_ignores_launch_nonce() {
9144 let registry = Arc::new(Registry::default());
9145 let handler = ControlHandler::new(Arc::clone(®istry));
9148 let no_nonce = handler
9149 .handle_control(
9150 ConnectionId::new(1),
9151 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
9152 )
9153 .unwrap();
9154 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
9155 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
9156
9157 let echoed_nonce = handler
9158 .handle_control(
9159 ConnectionId::new(2),
9160 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
9161 )
9162 .unwrap();
9163 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
9164 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
9165 }
9166
9167 #[test]
9168 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
9169 let handler = ControlHandler::default();
9170 let conn = ConnectionId::new(1);
9171 let malformed = Frame::build(
9172 FrameType::Hello,
9173 control_flags(),
9174 0,
9175 0,
9176 3,
9177 b"{not json".to_vec(),
9178 )
9179 .unwrap();
9180
9181 let error = handler.handle_control(conn, malformed).unwrap();
9182 assert_eq!(error[0].header.ty, FrameType::Error);
9183 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
9184
9185 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
9186 let pong = handler.handle_control(conn, ping).unwrap();
9187 assert_eq!(pong[0].header.ty, FrameType::Pong);
9188 assert_eq!(pong[0].header.corr, 4);
9189 }
9190
9191 #[test]
9192 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
9193 let registry = Arc::new(Registry::default());
9194 let handler = ControlHandler::new(Arc::clone(®istry));
9195
9196 handler
9197 .handle_control(
9198 ConnectionId::new(1),
9199 hello_frame("aft", PROTOCOL_VERSION, 1),
9200 )
9201 .unwrap();
9202 let duplicate = handler
9203 .handle_control(
9204 ConnectionId::new(2),
9205 hello_frame("aft", PROTOCOL_VERSION, 2),
9206 )
9207 .unwrap();
9208
9209 assert_eq!(duplicate[0].header.ty, FrameType::Error);
9210 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
9211 let registration = registry.get_module("aft").unwrap().unwrap();
9212 assert_eq!(registration.connection_id, ConnectionId::new(1));
9213 }
9214
9215 #[test]
9216 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
9217 let registry = Arc::new(Registry::default());
9218 let forwarding = Arc::new(ForwardingTable::default());
9219 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
9220 let handler =
9221 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9222 .with_process_liveness(process_liveness);
9223 let (ctx, route_channel, route_epoch) =
9224 bind_liveness_route(®istry, &forwarding, "aft-dead");
9225 let responses = handler
9226 .handle_route_poll(
9227 &ctx,
9228 route_poll_frame(41, PollKind::Liveness, route_channel),
9229 route_channel,
9230 route_epoch,
9231 PollKind::Liveness,
9232 )
9233 .unwrap();
9234
9235 assert_eq!(responses.len(), 1);
9236 assert_eq!(responses[0].header.ty, FrameType::Response);
9237 assert_route_poll_liveness(&responses[0], false);
9238 }
9239
9240 #[test]
9241 fn liveness_poll_without_process_source_uses_bound_route() {
9242 let registry = Arc::new(Registry::default());
9243 let forwarding = Arc::new(ForwardingTable::default());
9244 let handler =
9245 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9246 let (ctx, route_channel, route_epoch) =
9247 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
9248 let responses = handler
9249 .handle_route_poll(
9250 &ctx,
9251 route_poll_frame(42, PollKind::Liveness, route_channel),
9252 route_channel,
9253 route_epoch,
9254 PollKind::Liveness,
9255 )
9256 .unwrap();
9257
9258 assert_route_poll_liveness(&responses[0], true);
9259 }
9260
9261 #[test]
9262 fn liveness_poll_untracked_process_source_uses_bound_route() {
9263 let registry = Arc::new(Registry::default());
9264 let forwarding = Arc::new(ForwardingTable::default());
9265 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
9266 let handler =
9267 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9268 .with_process_liveness(process_liveness);
9269 let (ctx, route_channel, route_epoch) =
9270 bind_liveness_route(®istry, &forwarding, "aft-untracked");
9271 let responses = handler
9272 .handle_route_poll(
9273 &ctx,
9274 route_poll_frame(43, PollKind::Liveness, route_channel),
9275 route_channel,
9276 route_epoch,
9277 PollKind::Liveness,
9278 )
9279 .unwrap();
9280
9281 assert_route_poll_liveness(&responses[0], true);
9282 }
9283
9284 #[tokio::test]
9285 async fn unknown_op_returns_unknown_control_op() {
9286 let handler = ControlHandler::default();
9287 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
9288 let request = Frame::build(
9289 FrameType::Request,
9290 control_flags(),
9291 0,
9292 0,
9293 55,
9294 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
9295 )
9296 .unwrap();
9297
9298 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
9299
9300 assert_eq!(response.len(), 1);
9301 assert_eq!(response[0].header.ty, FrameType::Error);
9302 assert_eq!(response[0].header.corr, 55);
9303 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
9304 }
9305
9306 #[tokio::test]
9307 async fn supervisor_provenance_rejects_unknown_exact_module() {
9308 let handler = ControlHandler::default();
9309 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
9310 let request = Frame::build(
9311 FrameType::Request,
9312 control_flags(),
9313 0,
9314 0,
9315 57,
9316 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
9317 )
9318 .unwrap();
9319
9320 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
9321
9322 assert_eq!(response.len(), 1);
9323 assert_eq!(response[0].header.ty, FrameType::Error);
9324 assert_eq!(response[0].header.corr, 57);
9325 let error = parse_error(&response[0]);
9326 assert_eq!(error["code"], "unknown_module");
9327 assert_eq!(error["message"], "module_id 'missing' is not supervised");
9328 }
9329
9330 #[test]
9331 fn provenance_probe_override_keeps_handler_tests_deterministic() {
9332 let expected = subc_control::RunningImageAgreement::Unavailable {
9333 reason: subc_control::RunningImageUnavailableReason::HashFailed,
9334 };
9335 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
9336 assert_eq!(handler.provenance_probe_override, Some(expected));
9337 }
9338
9339 #[tokio::test]
9340 async fn malformed_control_bodies_return_invalid_control_body() {
9341 let handler = ControlHandler::default();
9342 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
9343
9344 for (corr, body) in [
9345 (56, br#"{"route_channel":1}"#.as_slice()),
9346 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
9347 (
9348 58,
9349 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
9350 ),
9351 ] {
9352 let request = Frame::build(
9353 FrameType::Request,
9354 control_flags(),
9355 0,
9356 0,
9357 corr,
9358 body.to_vec(),
9359 )
9360 .unwrap();
9361 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
9362
9363 assert_eq!(response.len(), 1);
9364 assert_eq!(response[0].header.ty, FrameType::Error);
9365 assert_eq!(response[0].header.corr, corr);
9366 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
9367 }
9368 }
9369
9370 #[tokio::test]
9371 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
9372 let registry = Arc::new(Registry::default());
9373 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
9374 let router = Router::with_control_handler(Arc::clone(&control));
9375 let connection = router.begin_connection();
9376 let (ctx, mut rx) = route_ctx(connection.id());
9377
9378 router
9379 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
9380 .await
9381 .unwrap();
9382 let response = rx.recv().await.unwrap();
9383 let ack = parse_ack(&response);
9384 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9385 let channel = 1;
9386
9387 let goodbye =
9388 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
9389 router.route_for_connection(&ctx, goodbye).await.unwrap();
9390 assert!(rx.try_recv().is_err());
9391 assert!(registry.get_module("aft").unwrap().is_none());
9392
9393 router
9394 .route_for_connection(&ctx, channel_request(channel, 13))
9395 .await
9396 .unwrap();
9397 let error_frame = rx.recv().await.unwrap();
9398 assert_eq!(error_frame.header.ty, FrameType::Error);
9399 assert_eq!(error_frame.header.channel, channel);
9400 }
9401
9402 #[tokio::test]
9403 async fn dropping_router_connection_releases_registration() {
9404 let registry = Arc::new(Registry::default());
9405 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
9406 let router = Router::with_control_handler(control);
9407 let connection = router.begin_connection();
9408 let (ctx, mut rx) = route_ctx(connection.id());
9409
9410 router
9411 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
9412 .await
9413 .unwrap();
9414 let response = rx.recv().await.unwrap();
9415 let ack = parse_ack(&response);
9416 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9417 assert!(registry.get_module("aft").unwrap().is_some());
9418
9419 drop(connection);
9420
9421 assert!(registry.get_module("aft").unwrap().is_none());
9422 assert_eq!(registry.active_registration_count().unwrap(), 0);
9423 }
9424
9425 fn capability_manifest(
9426 module_id: &str,
9427 provides: &[&str],
9428 must_never_reach: &[&str],
9429 ) -> ModuleManifest {
9430 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
9431 manifest.capabilities = Some(CapabilityDeclarations {
9432 provides: provides
9433 .iter()
9434 .map(|capability| (*capability).to_string())
9435 .collect(),
9436 requires: Vec::new(),
9437 must_never_reach: must_never_reach
9438 .iter()
9439 .map(|capability| (*capability).to_string())
9440 .collect(),
9441 });
9442 manifest
9443 }
9444
9445 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
9446 Frame::build(
9447 FrameType::Hello,
9448 control_flags(),
9449 0,
9450 0,
9451 corr,
9452 serde_json::to_vec(&ModuleHelloBody {
9453 protocol_ver: manifest.protocol_ver,
9454 manifest,
9455 control_ops: None,
9456 launch_nonce: None,
9457 })
9458 .expect("capability test HELLO serializes"),
9459 )
9460 .expect("capability test HELLO frame builds")
9461 }
9462
9463 fn catalog_update_with_capabilities_frame(
9464 corr: u64,
9465 capabilities: CapabilityDeclarations,
9466 ) -> Frame {
9467 Frame::build(
9468 FrameType::Request,
9469 control_flags(),
9470 0,
9471 0,
9472 corr,
9473 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
9474 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
9475 capabilities: Some(capabilities),
9476 ready: None,
9477 })
9478 .expect("capability catalog.update serializes"),
9479 )
9480 .expect("capability catalog.update frame builds")
9481 }
9482
9483 async fn register_capability_manifest(
9484 handler: &ControlHandler,
9485 ctx: &RouteCtx,
9486 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9487 manifest: ModuleManifest,
9488 corr: u64,
9489 ) {
9490 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
9491 }
9492
9493 async fn open_route_for_capability_test(
9494 handler: &ControlHandler,
9495 target_ctx: &RouteCtx,
9496 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9497 client_connection_id: u64,
9498 corr: u64,
9499 target_module_id: &str,
9500 consumer_identity: Option<ConsumerIdentity>,
9501 ) -> (
9502 mpsc::Receiver<crate::router::OutboundFrame>,
9503 ModuleControlRequest,
9504 ) {
9505 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
9506 let route_handler = handler.clone();
9507 let target_module_id = target_module_id.to_string();
9508 let route_task = tokio::spawn(async move {
9509 route_handler
9510 .handle_control_frame(
9511 &client_ctx,
9512 route_open_frame_with_admission_facts(
9513 corr,
9514 &target_module_id,
9515 unique_project_root("admission-facts"),
9516 consumer_identity,
9517 None,
9518 ),
9519 )
9520 .await
9521 .expect("capability test route.open succeeds")
9522 });
9523 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
9524 .await
9525 .expect("capability test route.open must reach route.bind")
9526 .expect("target control receiver stays open");
9527 let bind_request: ModuleControlRequest =
9528 serde_json::from_slice(&bind.body).expect("route.bind decodes");
9529 handler
9530 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
9531 .await
9532 .expect("capability test route.bind ACK succeeds");
9533 assert!(route_task.await.expect("route.open task joins").is_empty());
9534 let opened = client_rx
9535 .recv()
9536 .await
9537 .expect("successful route.open publishes a response");
9538 assert!(matches!(
9539 serde_json::from_slice::<ClientControlResponse>(&opened.body),
9540 Ok(ClientControlResponse::RouteOpen { .. })
9541 ));
9542 (client_rx, bind_request)
9543 }
9544
9545 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
9546 assert_eq!(frame.header.ty, FrameType::Push);
9547 assert_eq!(frame.header.channel, 0);
9548 assert_eq!(
9549 serde_json::from_slice::<ClientControlPush>(&frame.body)
9550 .expect("route.closed control push decodes"),
9551 ClientControlPush::RouteClosed {
9552 module_id: target_module_id.to_string(),
9553 reason: RouteCloseReason::CapabilityDenied,
9554 drained: false,
9555 abandoned: 0,
9556 excluded_subscriptions: 0,
9557 terminal: Some(false),
9558 }
9559 );
9560 }
9561
9562 #[tokio::test]
9563 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
9564 let registry = Arc::new(Registry::default());
9565 let forwarding = Arc::new(ForwardingTable::default());
9566 let supervisor = SupervisorHandle::new();
9567 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9568 let handler =
9569 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9570 .with_supervisor(supervisor);
9571 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
9572 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
9573 register_capability_manifest(
9574 &handler,
9575 &target_ctx,
9576 &mut target_rx,
9577 capability_manifest("target", &["credentials-provider/v1"], &[]),
9578 1,
9579 )
9580 .await;
9581 register_capability_manifest(
9582 &handler,
9583 &opener_ctx,
9584 &mut opener_rx,
9585 capability_manifest("opener", &[], &["credentials-provider/v1"]),
9586 2,
9587 )
9588 .await;
9589
9590 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
9591 let replies = handler
9592 .handle_control_frame(
9593 &client_ctx,
9594 route_open_frame_with_admission_facts(
9595 3,
9596 "target",
9597 unique_project_root("admission-facts"),
9598 Some(ConsumerIdentity {
9599 module_id: "opener".to_string(),
9600 launch_nonce: "opener-nonce".to_string(),
9601 }),
9602 None,
9603 ),
9604 )
9605 .await
9606 .expect("denied route.open returns a typed frame");
9607 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
9608 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
9609 assert!(
9610 target_rx.try_recv().is_err(),
9611 "forbidden route.open must not relay route.bind"
9612 );
9613 }
9614
9615 #[tokio::test]
9616 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
9617 let registry = Arc::new(Registry::default());
9618 let forwarding = Arc::new(ForwardingTable::default());
9619 let supervisor = SupervisorHandle::new();
9620 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9621 let handler =
9622 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9623 .with_supervisor(supervisor);
9624 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
9625 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
9626 register_capability_manifest(
9627 &handler,
9628 &target_ctx,
9629 &mut target_rx,
9630 capability_manifest("target", &["credentials-provider/v1"], &[]),
9631 1,
9632 )
9633 .await;
9634 register_capability_manifest(
9635 &handler,
9636 &old_opener_ctx,
9637 &mut old_opener_rx,
9638 capability_manifest("opener", &[], &[]),
9639 2,
9640 )
9641 .await;
9642 let (mut client_rx, _) = open_route_for_capability_test(
9643 &handler,
9644 &target_ctx,
9645 &mut target_rx,
9646 712,
9647 3,
9648 "target",
9649 Some(ConsumerIdentity {
9650 module_id: "opener".to_string(),
9651 launch_nonce: "opener-nonce".to_string(),
9652 }),
9653 )
9654 .await;
9655 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
9656
9657 handler
9658 .cleanup_connection(old_opener_ctx.connection_id)
9659 .expect("old opener registration cleans up");
9660 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
9661 register_capability_manifest(
9662 &handler,
9663 &new_opener_ctx,
9664 &mut new_opener_rx,
9665 capability_manifest("opener", &[], &["credentials-provider/v1"]),
9666 4,
9667 )
9668 .await;
9669
9670 assert_capability_denied_push(
9671 client_rx
9672 .try_recv()
9673 .expect("HELLO deny addition must emit route.closed")
9674 .frame,
9675 "target",
9676 );
9677 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
9678 assert!(matches!(
9679 target_rx.try_recv(),
9680 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
9681 ));
9682 }
9683
9684 #[tokio::test]
9685 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
9686 let registry = Arc::new(Registry::default());
9687 let forwarding = Arc::new(ForwardingTable::default());
9688 let supervisor = SupervisorHandle::new();
9689 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9690 let handler =
9691 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9692 .with_supervisor(supervisor);
9693 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
9694 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
9695 register_capability_manifest(
9696 &handler,
9697 &target_ctx,
9698 &mut target_rx,
9699 capability_manifest("target", &[], &[]),
9700 1,
9701 )
9702 .await;
9703 register_capability_manifest(
9704 &handler,
9705 &opener_ctx,
9706 &mut opener_rx,
9707 capability_manifest("opener", &[], &["credentials-provider/v1"]),
9708 2,
9709 )
9710 .await;
9711 let (mut client_rx, _) = open_route_for_capability_test(
9712 &handler,
9713 &target_ctx,
9714 &mut target_rx,
9715 722,
9716 3,
9717 "target",
9718 Some(ConsumerIdentity {
9719 module_id: "opener".to_string(),
9720 launch_nonce: "opener-nonce".to_string(),
9721 }),
9722 )
9723 .await;
9724 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
9725
9726 let replies = handler
9727 .handle_control_frame(
9728 &target_ctx,
9729 catalog_update_with_capabilities_frame(
9730 4,
9731 CapabilityDeclarations {
9732 provides: vec!["credentials-provider/v1".to_string()],
9733 requires: Vec::new(),
9734 must_never_reach: Vec::new(),
9735 },
9736 ),
9737 )
9738 .await
9739 .expect("claim catalog.update succeeds");
9740 assert!(matches!(
9741 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
9742 Ok(ModuleControlResponseToModule::CatalogUpdate {})
9743 ));
9744 assert_capability_denied_push(
9745 client_rx
9746 .try_recv()
9747 .expect("claim addition must emit route.closed")
9748 .frame,
9749 "target",
9750 );
9751 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
9752 assert!(matches!(
9753 target_rx.try_recv(),
9754 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
9755 ));
9756 }
9757
9758 #[tokio::test]
9759 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
9760 let registry = Arc::new(Registry::default());
9761 let forwarding = Arc::new(ForwardingTable::default());
9762 let supervisor = SupervisorHandle::new();
9763 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9764 let handler =
9765 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9766 .with_supervisor(supervisor);
9767 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
9768 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
9769 register_capability_manifest(
9770 &handler,
9771 &target_ctx,
9772 &mut target_rx,
9773 capability_manifest("target", &["credentials-provider/v1"], &[]),
9774 1,
9775 )
9776 .await;
9777 register_capability_manifest(
9778 &handler,
9779 &opener_ctx,
9780 &mut opener_rx,
9781 capability_manifest("opener", &[], &[]),
9782 2,
9783 )
9784 .await;
9785 let (mut client_rx, _) = open_route_for_capability_test(
9786 &handler,
9787 &target_ctx,
9788 &mut target_rx,
9789 732,
9790 3,
9791 "target",
9792 Some(ConsumerIdentity {
9793 module_id: "opener".to_string(),
9794 launch_nonce: "opener-nonce".to_string(),
9795 }),
9796 )
9797 .await;
9798 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
9799
9800 handler
9801 .handle_control_frame(
9802 &target_ctx,
9803 catalog_update_with_capabilities_frame(
9804 4,
9805 CapabilityDeclarations {
9806 provides: Vec::new(),
9807 requires: Vec::new(),
9808 must_never_reach: Vec::new(),
9809 },
9810 ),
9811 )
9812 .await
9813 .expect("claim removal catalog.update succeeds");
9814 assert_eq!(
9815 forwarding.active_binding_count().unwrap(),
9816 1,
9817 "removing an attested target claim must leave the route census unchanged"
9818 );
9819 assert!(
9820 client_rx.try_recv().is_err(),
9821 "claim removal must not emit route.closed capability_denied"
9822 );
9823 assert!(
9824 target_rx.try_recv().is_err(),
9825 "claim removal must not send the target a route GOODBYE"
9826 );
9827 }
9828
9829 #[tokio::test]
9832 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
9833 let registry = Arc::new(Registry::default());
9834 let forwarding = Arc::new(ForwardingTable::default());
9835 let supervisor = SupervisorHandle::new();
9836 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9837 let handler =
9838 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9839 .with_supervisor(supervisor);
9840 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
9841 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
9842 register_capability_manifest(
9843 &handler,
9844 &target_ctx,
9845 &mut target_rx,
9846 capability_manifest("target", &["credentials-provider/v1"], &[]),
9847 1,
9848 )
9849 .await;
9850 register_capability_manifest(
9851 &handler,
9852 &opener_ctx,
9853 &mut opener_rx,
9854 capability_manifest("opener", &[], &["credentials-provider/v1"]),
9855 2,
9856 )
9857 .await;
9858
9859 let (_client_rx, bind) = open_route_for_capability_test(
9860 &handler,
9861 &target_ctx,
9862 &mut target_rx,
9863 742,
9864 3,
9865 "target",
9866 None,
9867 )
9868 .await;
9869 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
9870 panic!("direct scope-honesty route must bind");
9871 };
9872 assert_eq!(principal, Some(Principal::Direct));
9873 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
9874 }
9875
9876 #[tokio::test]
9879 async fn must_never_reach_self_route_is_capability_forbidden() {
9880 let registry = Arc::new(Registry::default());
9881 let forwarding = Arc::new(ForwardingTable::default());
9882 let supervisor = SupervisorHandle::new();
9883 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
9884 let handler =
9885 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9886 .with_supervisor(supervisor);
9887 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
9888 register_capability_manifest(
9889 &handler,
9890 &self_ctx,
9891 &mut self_rx,
9892 capability_manifest(
9893 "self-provider",
9894 &["credentials-provider/v1"],
9895 &["credentials-provider/v1"],
9896 ),
9897 1,
9898 )
9899 .await;
9900
9901 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
9902 let replies = handler
9903 .handle_control_frame(
9904 &client_ctx,
9905 route_open_frame_with_admission_facts(
9906 2,
9907 "self-provider",
9908 unique_project_root("admission-facts"),
9909 Some(ConsumerIdentity {
9910 module_id: "self-provider".to_string(),
9911 launch_nonce: "self-nonce".to_string(),
9912 }),
9913 None,
9914 ),
9915 )
9916 .await
9917 .expect("self-route refusal returns a typed frame");
9918 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
9919 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
9920 assert!(
9921 self_rx.try_recv().is_err(),
9922 "self denial must not relay route.bind"
9923 );
9924 }
9925
9926 #[test]
9927 fn unsupported_channel_zero_frame_returns_error() {
9928 let handler = ControlHandler::default();
9929 let request = Frame::build(
9930 FrameType::Request,
9931 control_flags(),
9932 0,
9933 0,
9934 21,
9935 b"opaque".to_vec(),
9936 )
9937 .unwrap();
9938
9939 let response = handler
9940 .handle_control(ConnectionId::new(1), request)
9941 .unwrap();
9942
9943 assert_eq!(response[0].header.ty, FrameType::Error);
9944 assert_eq!(
9945 parse_error(&response[0])["code"],
9946 "unsupported_control_frame"
9947 );
9948 }
9949
9950 mod swap {
9955 use super::*;
9956
9957 const INCUMBENT: ConnectionId = ConnectionId::new(30);
9958 const CANDIDATE: ConnectionId = ConnectionId::new(40);
9959
9960 struct Swap {
9961 registry: Arc<Registry>,
9962 forwarding: Arc<ForwardingTable>,
9963 handler: ControlHandler,
9964 incumbent_ctx: RouteCtx,
9965 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
9966 candidate_ctx: RouteCtx,
9967 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
9968 }
9969
9970 async fn swap_with_incumbent() -> Swap {
9971 let registry = Arc::new(Registry::default());
9972 let forwarding = Arc::new(ForwardingTable::default());
9973 let handler =
9974 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9975 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
9976 hello_via_sink(
9977 &handler,
9978 &incumbent_ctx,
9979 &mut incumbent_rx,
9980 hello_frame("aft", PROTOCOL_VERSION, 7),
9981 )
9982 .await;
9983 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
9984 Swap {
9985 registry,
9986 forwarding,
9987 handler,
9988 incumbent_ctx,
9989 incumbent_rx,
9990 candidate_ctx,
9991 candidate_rx,
9992 }
9993 }
9994
9995 fn register_candidate(swap: &Swap, ready: Option<bool>) {
9996 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
9997 candidate_manifest.ready = ready;
9998 let registration = swap
9999 .registry
10000 .register_candidate_with_control_ops(
10001 candidate_manifest,
10002 PROTOCOL_VERSION,
10003 CANDIDATE,
10004 module_baseline_control_ops(),
10005 )
10006 .unwrap();
10007 swap.forwarding
10008 .register_candidate_module_connection(
10009 CANDIDATE,
10010 "aft".to_string(),
10011 PROTOCOL_VERSION,
10012 manifest_concurrency(®istration.manifest),
10013 swap.candidate_ctx.egress.clone(),
10014 )
10015 .unwrap();
10016 }
10017
10018 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
10019 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
10020 swap.registry.promote_candidate("aft").unwrap().unwrap();
10021 cutover.incumbent.unwrap()
10022 }
10023
10024 fn keyed_total(counters: &Value, key: &str) -> u64 {
10025 counters[key]
10026 .as_object()
10027 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
10028 .unwrap_or(0)
10029 }
10030
10031 #[tokio::test]
10037 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
10038 let mut swap = swap_with_incumbent().await;
10039 let handler = swap.handler.clone();
10040
10041 let cotenant = ConnectionId::new(31);
10043 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
10044 let (cotenant_task, cotenant_bind) = relay_route_open(
10045 &handler,
10046 cotenant,
10047 &cotenant_ctx.egress,
10048 &mut swap.incumbent_rx,
10049 100,
10050 "aft",
10051 "swap-cotenant",
10052 )
10053 .await;
10054 handler
10055 .handle_control_frame(
10056 &swap.incumbent_ctx,
10057 route_bind_ack(cotenant_bind.header.corr),
10058 )
10059 .await
10060 .unwrap();
10061 assert!(cotenant_task.await.unwrap().is_empty());
10062 let (cotenant_channel, cotenant_epoch) =
10063 published_route(&cotenant_rx.recv().await.unwrap());
10064
10065 let caller = ConnectionId::new(32);
10067 let (caller_ctx, mut caller_rx) = route_ctx(caller);
10068 let (caller_task, caller_bind) = relay_route_open(
10069 &handler,
10070 caller,
10071 &caller_ctx.egress,
10072 &mut swap.incumbent_rx,
10073 101,
10074 "aft",
10075 "swap-caller",
10076 )
10077 .await;
10078 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
10079
10080 register_candidate(&swap, None);
10081 cutover(&swap);
10082
10083 let ack = handler
10085 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
10086 .await;
10087 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
10088 if module_loop_error.is_some() {
10089 handler.cleanup_connection(INCUMBENT).unwrap();
10092 }
10093
10094 assert!(
10096 cotenant_rx.try_recv().is_err(),
10097 "the co-tenant route on the incumbent was torn down by one late ack: \
10098 {module_loop_error:?}"
10099 );
10100 assert!(matches!(
10101 swap.forwarding
10102 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
10103 .unwrap(),
10104 DataRoute::Client(DataRouteState::Bound(_))
10105 ));
10106 assert_eq!(module_loop_error, None);
10107 assert!(swap
10108 .registry
10109 .get_module_by_connection(INCUMBENT)
10110 .unwrap()
10111 .is_some());
10112
10113 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
10115 .await
10116 .expect("the incumbent is told to drop the abandoned binding")
10117 .unwrap()
10118 .frame;
10119 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10120 assert_eq!(goodbye.header.channel, abandoned_channel);
10121 assert_eq!(goodbye.header.epoch, abandoned_epoch);
10122 assert!(swap.incumbent_rx.try_recv().is_err());
10123
10124 let response = caller_task.await.unwrap();
10126 assert_eq!(response.len(), 1);
10127 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
10128 assert!(caller_rx.try_recv().is_err());
10129
10130 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
10134 let counters = handler.counters().snapshot();
10135 assert_eq!(
10136 keyed_total(&counters, "route_open_accepted_by_principal"),
10137 1
10138 );
10139 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
10140 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
10141 }
10142
10143 #[tokio::test]
10147 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
10148 let mut swap = swap_with_incumbent().await;
10149 register_candidate(&swap, None);
10150 let incumbent = cutover(&swap);
10151 swap.forwarding
10152 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
10153 .unwrap()
10154 .expect("the incumbent is still registered");
10155
10156 let client = ConnectionId::new(33);
10157 let (client_ctx, mut client_rx) = route_ctx(client);
10158 let route_handler = swap.handler.clone();
10159 let open_ctx = RouteCtx {
10160 connection_id: client,
10161 egress: client_ctx.egress.clone(),
10162 };
10163 let mut route_task = tokio::spawn(async move {
10164 route_handler
10165 .handle_control_frame(
10166 &open_ctx,
10167 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
10168 )
10169 .await
10170 .unwrap()
10171 });
10172 let bind = tokio::select! {
10173 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
10174 response = &mut route_task => {
10175 let response = response.unwrap();
10176 panic!(
10177 "post-cutover route.open was refused instead of relayed to the candidate: {}",
10178 parse_error(&response[0])["code"]
10179 );
10180 }
10181 };
10182 swap.handler
10183 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
10184 .await
10185 .unwrap();
10186 assert!(route_task.await.unwrap().is_empty());
10187 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
10188 match swap
10189 .forwarding
10190 .lookup_data_route(client, channel, epoch)
10191 .unwrap()
10192 {
10193 DataRoute::Client(DataRouteState::Bound(route)) => {
10194 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
10195 }
10196 other => panic!("expected a bound route on the candidate, got {other:?}"),
10197 }
10198 assert!(swap.incumbent_rx.try_recv().is_err());
10199 }
10200
10201 #[tokio::test]
10206 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
10207 let swap = swap_with_incumbent().await;
10208 register_candidate(&swap, Some(false));
10209 let update = Frame::build(
10210 FrameType::Request,
10211 control_flags(),
10212 0,
10213 0,
10214 55,
10215 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
10216 provides: manifest("aft", PROTOCOL_VERSION).provides,
10217 capabilities: None,
10218 ready: Some(true),
10219 })
10220 .unwrap(),
10221 )
10222 .unwrap();
10223
10224 let replies = swap
10225 .handler
10226 .handle_control_frame(&swap.candidate_ctx, update)
10227 .await
10228 .unwrap();
10229
10230 assert_eq!(replies.len(), 1);
10231 assert_eq!(
10232 replies[0].header.ty,
10233 FrameType::Response,
10234 "candidate catalog.update was refused: {:?}",
10235 serde_json::from_slice::<Value>(&replies[0].body).ok()
10236 );
10237 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
10238 assert_eq!(
10239 swap.registry
10240 .get_module("aft")
10241 .unwrap()
10242 .unwrap()
10243 .connection_id,
10244 INCUMBENT
10245 );
10246 }
10247 }
10248
10249 mod swap_admission {
10253 use super::*;
10254
10255 const INCUMBENT_NONCE: &str = "incumbent-nonce";
10256 const CANDIDATE_NONCE: &str = "candidate-nonce";
10257
10258 fn handler_with_incumbent(
10259 module_id: &str,
10260 reserved: bool,
10261 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
10262 let registry = Arc::new(Registry::default());
10263 let supervisor = SupervisorHandle::new();
10264 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
10265 if reserved {
10266 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
10267 }
10268 let handler =
10269 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
10270 let incumbent = handler
10271 .handle_control(
10272 ConnectionId::new(1),
10273 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
10274 )
10275 .unwrap();
10276 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
10277 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
10278 (registry, supervisor, handler)
10279 }
10280
10281 #[test]
10287 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
10288 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
10289
10290 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
10291 let replies = handler
10292 .handle_control(
10293 ConnectionId::new(connection),
10294 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
10295 )
10296 .unwrap();
10297 assert_eq!(replies[0].header.ty, FrameType::Error);
10298 assert_eq!(
10299 parse_error(&replies[0])["code"],
10300 "swap_token_invalid",
10301 "nonce {nonce:?}"
10302 );
10303 }
10304 assert!(registry.get_candidate("aft").unwrap().is_none());
10305 assert_eq!(
10306 registry.get_module("aft").unwrap().unwrap().connection_id,
10307 ConnectionId::new(1)
10308 );
10309
10310 let admitted = handler
10313 .handle_control(
10314 ConnectionId::new(4),
10315 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
10316 )
10317 .unwrap();
10318 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
10319 assert_eq!(
10320 registry
10321 .get_candidate("aft")
10322 .unwrap()
10323 .unwrap()
10324 .connection_id,
10325 ConnectionId::new(4)
10326 );
10327 assert_eq!(
10328 registry.get_module("aft").unwrap().unwrap().connection_id,
10329 ConnectionId::new(1),
10330 "the candidate must not take the active slot"
10331 );
10332 let replayed = handler
10333 .handle_control(
10334 ConnectionId::new(5),
10335 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
10336 )
10337 .unwrap();
10338 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
10339
10340 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
10344 let squatter = handler
10345 .handle_control(
10346 ConnectionId::new(6),
10347 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
10348 )
10349 .unwrap();
10350 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
10351 assert!(
10352 registry.get_module("aft").unwrap().is_none(),
10353 "a squatter took the active slot of an id being swapped"
10354 );
10355 }
10356
10357 #[test]
10362 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
10363 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
10364
10365 let replies = handler
10366 .handle_control(
10367 ConnectionId::new(2),
10368 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
10369 )
10370 .unwrap();
10371
10372 assert_eq!(
10373 replies[0].header.ty,
10374 FrameType::HelloAck,
10375 "reserved candidate refused: {:?}",
10376 serde_json::from_slice::<Value>(&replies[0].body).ok()
10377 );
10378 assert_eq!(
10379 registry
10380 .get_candidate("vault")
10381 .unwrap()
10382 .unwrap()
10383 .connection_id,
10384 ConnectionId::new(2)
10385 );
10386 }
10387
10388 #[test]
10391 fn without_an_open_swap_the_ordinary_gates_decide() {
10392 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
10393 supervisor.close_swap("vault");
10394
10395 let candidate = handler
10396 .handle_control(
10397 ConnectionId::new(2),
10398 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
10399 )
10400 .unwrap();
10401 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
10402 let duplicate = handler
10403 .handle_control(
10404 ConnectionId::new(3),
10405 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
10406 )
10407 .unwrap();
10408 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
10409 assert!(registry.get_candidate("vault").unwrap().is_none());
10410 }
10411 }
10412}
10413
10414#[cfg(test)]
10415mod concurrency_default_exposure_tests {
10416 use super::*;
10417
10418 fn hello_body(role_json: &str) -> Vec<u8> {
10419 format!(
10420 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
10421 )
10422 .into_bytes()
10423 }
10424
10425 fn manifest_from(body: &[u8]) -> ModuleManifest {
10426 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
10427 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
10428 .expect("manifest parses")
10429 }
10430
10431 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
10432
10433 #[test]
10434 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
10435 let body = hello_body(&format!(
10436 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
10437 ));
10438 let manifest = manifest_from(&body);
10439 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
10442 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
10443 }
10444
10445 #[test]
10446 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
10447 let body = hello_body(&format!(
10448 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
10449 ));
10450 let manifest = manifest_from(&body);
10451 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
10452 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
10453 }
10454
10455 #[test]
10456 fn non_management_roles_are_never_reported() {
10457 let body = hello_body(
10458 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
10459 );
10460 let manifest = manifest_from(&body);
10461 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
10462 }
10463}