1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 path::{Path, PathBuf},
5 sync::{Arc, Mutex},
6 time::{Duration, Instant as StdInstant},
7};
8
9use serde::{Deserialize, Serialize};
10use subc_control::{
11 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
12 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
13 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
14 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
15 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
16 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
17 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
18};
19use subc_protocol::{
20 error_codes,
21 manifest::{
22 validate_hello_capability_grammar, validate_hello_self_signal_declarations,
23 CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
24 ProviderRole,
25 },
26 session::{
27 HealthReport, ModuleControlPush, ModuleControlRequest, ModuleControlRequestFromModule,
28 ModuleControlResponse, ModuleControlResponseToModule, MODULE_CONTROL_OP_HEALTH_CHECK,
29 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
30 },
31 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
32 Priority, RouteTarget, PROTOCOL_VERSION,
33};
34use tokio::time::{timeout_at, Instant};
35use tracing::{debug, info, warn};
36
37use crate::{
38 capability_requirements::{
39 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
40 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
41 RuntimeModule,
42 },
43 daemon_config::RestartRequiredSection,
44 forwarding::{
45 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
46 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
47 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
48 },
49 observability::{
50 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
51 },
52 provenance::{
53 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
54 },
55 registry::{ChannelState, ConnectionId, Registry, RegistryError},
56 router::{RouteCtx, RouterError},
57 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
58 stderr_tail::{CaptureState, TailEntry},
59 supervise::{
60 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
61 SupervisorHandle, SwapHelloAdmission,
62 },
63 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
64};
65
66pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
72
73const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
74const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
75const CAP_PING_PONG: &str = "ping_pong_v1";
76const CAP_SESSION_ATTACH: &str = "session_attach_v1";
77const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
78
79const SUBC_CONTROL_OPS: &[&str] = &[
80 ops::SERVER_DESCRIBE,
81 ops::CATALOG_LIST,
82 ops::ROUTE_OPEN,
83 ops::ROUTE_POLL,
84 ops::ROUTE_CLOSING,
85 ops::ROUTE_CLOSED,
86 ops::SUPERVISOR_LIST,
87 ops::SUPERVISOR_RESTART,
88 ops::SUPERVISOR_SWAP,
89 ops::SUPERVISOR_RELOAD,
90 ops::SUPERVISOR_RESCAN,
91 ops::SUPERVISOR_RELEASE_RESERVED,
92 ops::SUPERVISOR_SET_ENABLED,
93 ops::SUPERVISOR_HEALTH_PROBE,
94 ops::SUPERVISOR_HEALTH,
95 ops::SUPERVISOR_STDERR_TAIL,
96 ops::SUPERVISOR_TERMINALS,
97 ops::SUPERVISOR_ROUTES,
98 ops::SUPERVISOR_PROVENANCE,
99 ops::SUPERVISOR_SPAWN_SNAPSHOT,
100 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
101];
102
103const MODULE_TO_SUBC_CONTROL_OPS: &[&str] =
104 &[MODULE_TO_SUBC_OP_CATALOG_UPDATE, "supervisor.live_roots"];
105
106const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
107
108pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
116
117pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
132
133pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
151
152const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
153const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
154
155fn reload_verdict(
156 configured: &Path,
157 spawned_from: Option<&Path>,
158 image: subc_control::RunningImageAgreement,
159) -> PendingReloadVerdict {
160 let path = match spawned_from {
161 Some(spawned_from) if configured == spawned_from => ReloadPathAgreement::Match,
162 Some(spawned_from) => ReloadPathAgreement::Mismatch {
163 configured: configured.to_path_buf(),
164 spawned_from: spawned_from.to_path_buf(),
165 },
166 None => ReloadPathAgreement::Unavailable {
167 reason: if matches!(
168 image,
169 subc_control::RunningImageAgreement::Unavailable {
170 reason: subc_control::RunningImageUnavailableReason::NotRunning
171 }
172 ) {
173 ReloadPathUnavailableReason::NotRunning
174 } else {
175 ReloadPathUnavailableReason::SpawnedPathUnavailable
176 },
177 },
178 };
179 PendingReloadVerdict { path, image }
180}
181
182#[derive(Clone)]
183struct DaemonProvenanceFacts {
184 build: DaemonBuildProvenance,
185 pid: Option<u32>,
186 started_at_ms: Option<u64>,
187 start_clock: Option<crate::clock::StartClock>,
188 executable_path: Option<PathBuf>,
189 executable_identity: Option<SpawnedFileIdentity>,
190 process_start_time: Option<u64>,
191 probe: ExecutableIdentityProbe,
192}
193
194impl Default for DaemonProvenanceFacts {
195 fn default() -> Self {
196 Self {
197 build: DaemonBuildProvenance {
198 build_git_sha: None,
199 build_lock_digest: None,
200 },
201 pid: None,
202 started_at_ms: None,
203 start_clock: None,
204 executable_path: None,
205 executable_identity: None,
206 process_start_time: None,
207 probe: ExecutableIdentityProbe::default(),
208 }
209 }
210}
211
212#[derive(Debug, Clone)]
213struct SupervisorRescanContext {
214 supervisor: Supervisor,
215 config_path: PathBuf,
216 configured_port: Option<u16>,
217 storage_config: Option<crate::daemon_config::StorageConfig>,
218 admission_facts_carrier_module_id: Option<String>,
219 admission_facts_targets: Option<Vec<String>>,
220}
221
222#[derive(Clone)]
224pub struct ControlHandler {
225 registry: Arc<Registry>,
226 forwarding: Arc<ForwardingTable>,
227 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
228 supervisor: SupervisorHandle,
229 subc_capabilities: Arc<[String]>,
230 route_bind_relay_timeout: Duration,
234 route_bind_relay_timeouts: BTreeMap<String, Duration>,
238 route_bind_breakers: RouteBindBreakers,
241 route_bind_concurrency: RouteBindConcurrency,
244 route_bind_breaker_threshold: u32,
246 route_bind_breaker_cooldown: Duration,
248 health_probe_timeout: Duration,
249 storage_config: Option<crate::daemon_config::StorageConfig>,
252 machine_id: Option<crate::machine_id::MachineId>,
256 admission_facts_carrier_module_id: Option<String>,
257 admission_facts_targets: Option<Vec<String>>,
258 rescan: Option<SupervisorRescanContext>,
259 connected_clients: ConnectedClients,
260 counters: DaemonCounters,
261 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
262 daemon_provenance: DaemonProvenanceFacts,
263 #[cfg(test)]
264 control_dispatch_delay: Option<Duration>,
265 #[cfg(test)]
266 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
267}
268
269impl fmt::Debug for ControlHandler {
270 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
271 f.debug_struct("ControlHandler")
272 .field("registry", &self.registry)
273 .field("forwarding", &self.forwarding)
274 .field("process_liveness", &self.process_liveness.is_some())
275 .field("supervisor", &self.supervisor)
276 .field("subc_capabilities", &self.subc_capabilities)
277 .finish()
278 }
279}
280
281struct RouteOpenRequest {
282 target: RouteTarget,
283 identity: BindIdentity,
284 consumer_identity: Option<ConsumerIdentity>,
285 consumer_capabilities: Option<Vec<String>>,
286 admission_facts: Option<serde_json::Value>,
287}
288
289struct RouteBindReservationGuard {
290 forwarding: Arc<ForwardingTable>,
291 endpoint: ModuleEndpointId,
292 relay_corr: u64,
293 armed: bool,
294}
295
296struct ModuleControlRpcGuard {
297 forwarding: Arc<ForwardingTable>,
298 endpoint: ModuleEndpointId,
299 corr: u64,
300 armed: bool,
301}
302
303impl ModuleControlRpcGuard {
304 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
305 Self {
306 forwarding,
307 endpoint,
308 corr,
309 armed: true,
310 }
311 }
312
313 fn disarm(&mut self) {
314 self.armed = false;
315 }
316}
317
318impl Drop for ModuleControlRpcGuard {
319 fn drop(&mut self) {
320 if self.armed {
321 let _ = self
322 .forwarding
323 .cancel_module_control_rpc(self.endpoint, self.corr);
324 }
325 }
326}
327
328impl RouteBindReservationGuard {
329 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
330 Self {
331 forwarding,
332 endpoint,
333 relay_corr,
334 armed: true,
335 }
336 }
337
338 fn release_and_disarm(&mut self) {
339 if !self.armed {
340 return;
341 }
342 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
343 self.endpoint,
344 self.relay_corr,
345 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
346 ) {
347 send_goodbye_target_best_effort(
348 &self.forwarding.counters(),
349 &target,
350 "canceled route.bind",
351 );
352 }
353 self.armed = false;
354 }
355
356 fn disarm(&mut self) {
357 self.armed = false;
358 }
359}
360
361impl Drop for RouteBindReservationGuard {
362 fn drop(&mut self) {
363 self.release_and_disarm();
364 }
365}
366
367#[derive(Debug, Clone, Default)]
397pub(crate) struct RouteBindBreakers {
398 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
399}
400
401#[derive(Debug, Clone, Default)]
402pub(crate) struct RouteBindConcurrency {
403 modules: Arc<Mutex<HashMap<String, usize>>>,
404}
405
406struct RouteBindConcurrencyGuard {
407 concurrency: RouteBindConcurrency,
408 module_id: String,
409}
410
411impl RouteBindConcurrency {
412 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
415 let mut modules = self
416 .modules
417 .lock()
418 .expect("route.bind concurrency mutex poisoned");
419 let in_flight = modules.entry(module_id.to_string()).or_default();
420 if *in_flight >= limit {
421 return Err(*in_flight);
422 }
423 *in_flight += 1;
424 Ok(RouteBindConcurrencyGuard {
425 concurrency: self.clone(),
426 module_id: module_id.to_string(),
427 })
428 }
429}
430
431impl Drop for RouteBindConcurrencyGuard {
432 fn drop(&mut self) {
433 let mut modules = self
434 .concurrency
435 .modules
436 .lock()
437 .expect("route.bind concurrency mutex poisoned");
438 let remove = {
439 let in_flight = modules
440 .get_mut(&self.module_id)
441 .expect("admitted route.bind has a concurrency entry");
442 *in_flight -= 1;
443 *in_flight == 0
444 };
445 if remove {
446 modules.remove(&self.module_id);
447 }
448 }
449}
450
451#[derive(Debug, Default)]
452struct ModuleBreakerState {
453 consecutive_timeouts: u32,
455 cooldown_until: Option<Instant>,
458 probe_in_flight: bool,
463}
464
465enum RouteBindAdmission<'a> {
467 Admitted {
468 guard: RouteBindBreakerGuard<'a>,
469 probe: bool,
472 },
473 Refused {
474 consecutive_timeouts: u32,
475 retry_in: Duration,
479 probe_in_flight: bool,
480 },
481}
482
483struct RouteBindBreakerGuard<'a> {
490 breakers: RouteBindBreakers,
491 module_id: &'a str,
492 settled: bool,
493}
494
495impl RouteBindBreakerGuard<'_> {
496 fn record_accepted(&mut self) -> bool {
500 self.settled = true;
501 self.breakers.record_accepted(self.module_id)
502 }
503
504 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
507 self.settled = true;
508 self.breakers
509 .record_timeout(self.module_id, threshold, cooldown)
510 }
511
512 fn record_inconclusive(&mut self) {
521 self.settled = true;
522 self.breakers.record_inconclusive(self.module_id);
523 }
524}
525
526impl Drop for RouteBindBreakerGuard<'_> {
527 fn drop(&mut self) {
528 if !self.settled {
529 self.breakers.record_inconclusive(self.module_id);
530 }
531 }
532}
533
534struct BreakerOpened {
538 consecutive_timeouts: u32,
539 reopened_after_probe: bool,
542}
543
544impl RouteBindBreakers {
545 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
546 self.modules
547 .lock()
548 .expect("route.bind breaker mutex poisoned")
549 }
550
551 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
554 let admitted = |probe| RouteBindAdmission::Admitted {
555 guard: RouteBindBreakerGuard {
556 breakers: self.clone(),
557 module_id,
558 settled: false,
559 },
560 probe,
561 };
562
563 let mut modules = self.lock();
564 let Some(state) = modules.get_mut(module_id) else {
565 return admitted(false);
566 };
567 let Some(cooldown_until) = state.cooldown_until else {
568 return admitted(false);
569 };
570 if state.probe_in_flight {
571 return RouteBindAdmission::Refused {
572 consecutive_timeouts: state.consecutive_timeouts,
573 retry_in: Duration::ZERO,
574 probe_in_flight: true,
575 };
576 }
577 let now = Instant::now();
578 if now < cooldown_until {
579 return RouteBindAdmission::Refused {
580 consecutive_timeouts: state.consecutive_timeouts,
581 retry_in: cooldown_until - now,
582 probe_in_flight: false,
583 };
584 }
585 state.probe_in_flight = true;
586 admitted(true)
587 }
588
589 fn record_accepted(&self, module_id: &str) -> bool {
590 self.lock()
591 .remove(module_id)
592 .is_some_and(|state| state.cooldown_until.is_some())
593 }
594
595 fn record_timeout(
596 &self,
597 module_id: &str,
598 threshold: u32,
599 cooldown: Duration,
600 ) -> Option<BreakerOpened> {
601 let mut modules = self.lock();
602 let state = modules.entry(module_id.to_string()).or_default();
603 let was_open = state.cooldown_until.is_some();
604 let was_probe = state.probe_in_flight;
605 state.probe_in_flight = false;
606 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
607 if state.consecutive_timeouts < threshold {
608 return None;
609 }
610 state.cooldown_until = Some(Instant::now() + cooldown);
611 Some(BreakerOpened {
612 consecutive_timeouts: state.consecutive_timeouts,
613 reopened_after_probe: was_open && was_probe,
614 })
615 }
616
617 fn record_inconclusive(&self, module_id: &str) {
618 if let Some(state) = self.lock().get_mut(module_id) {
619 state.probe_in_flight = false;
620 }
621 }
622
623 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
638 self.lock()
639 .remove(module_id)
640 .map(|state| state.consecutive_timeouts)
641 .filter(|discarded| *discarded > 0)
642 }
643
644 fn open_snapshot(&self) -> Option<serde_json::Value> {
651 let now = Instant::now();
652 let modules = self.lock();
653 let open = modules
654 .iter()
655 .filter_map(|(module_id, state)| {
656 let cooldown_until = state.cooldown_until?;
657 Some((
658 module_id.clone(),
659 serde_json::json!({
660 "consecutive_timeouts": state.consecutive_timeouts,
661 "cooldown_remaining_ms":
662 cooldown_until.saturating_duration_since(now).as_millis() as u64,
663 "probe_in_flight": state.probe_in_flight,
664 }),
665 ))
666 })
667 .collect::<serde_json::Map<String, serde_json::Value>>();
668 (!open.is_empty()).then_some(serde_json::Value::Object(open))
669 }
670}
671
672impl ControlHandler {
673 pub fn new(registry: Arc<Registry>) -> Self {
674 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
675 }
676
677 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
678 let counters = forwarding.counters();
679 let route_bind_breakers = forwarding.route_bind_breakers();
683 let route_bind_concurrency = forwarding.route_bind_concurrency();
684 Self {
685 registry,
686 forwarding,
687 process_liveness: None,
688 supervisor: SupervisorHandle::new(),
689 subc_capabilities: Arc::from([
690 CAP_MANIFEST_REGISTRATION.to_string(),
691 CAP_CHANNEL_LIFECYCLE.to_string(),
692 CAP_PING_PONG.to_string(),
693 CAP_SESSION_ATTACH.to_string(),
694 CAP_ADMISSION_FACTS_RELAY.to_string(),
695 ]),
696 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
697 route_bind_relay_timeouts: BTreeMap::new(),
698 route_bind_breakers,
699 route_bind_concurrency,
700 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
701 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
702 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
703 storage_config: None,
704 machine_id: None,
705 admission_facts_carrier_module_id: None,
706 admission_facts_targets: None,
707 rescan: None,
708 connected_clients: ConnectedClients::new(),
709 counters,
710 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
711 daemon_provenance: DaemonProvenanceFacts::default(),
712 #[cfg(test)]
713 control_dispatch_delay: None,
714 #[cfg(test)]
715 provenance_probe_override: None,
716 }
717 }
718
719 pub fn with_storage_config(
722 mut self,
723 storage_config: Option<crate::daemon_config::StorageConfig>,
724 ) -> Self {
725 self.storage_config = storage_config;
726 self
727 }
728
729 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
732 self.machine_id = machine_id;
733 self
734 }
735
736 pub fn with_admission_facts_config(
740 mut self,
741 carrier_module_id: Option<String>,
742 targets: Option<Vec<String>>,
743 ) -> Self {
744 self.admission_facts_carrier_module_id = carrier_module_id;
745 self.admission_facts_targets = targets;
746 self
747 }
748
749 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
752 self.route_bind_relay_timeout = timeout;
753 self
754 }
755
756 pub fn with_route_bind_relay_timeouts(
762 mut self,
763 timeouts: impl IntoIterator<Item = (String, Duration)>,
764 ) -> Self {
765 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
766 self
767 }
768
769 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
775 self.route_bind_relay_timeouts
776 .get(module_id)
777 .copied()
778 .unwrap_or(self.route_bind_relay_timeout)
779 }
780
781 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
789 self.route_bind_breaker_threshold = threshold.max(1);
790 self.route_bind_breaker_cooldown = cooldown;
791 self
792 }
793
794 #[cfg(test)]
795 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
796 self.health_probe_timeout = timeout;
797 self
798 }
799
800 #[cfg(test)]
801 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
802 self.control_dispatch_delay = Some(delay);
803 self
804 }
805
806 pub fn with_process_liveness(
807 mut self,
808 process_liveness: Arc<dyn ModuleProcessLiveness>,
809 ) -> Self {
810 self.process_liveness = Some(process_liveness);
811 self
812 }
813
814 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
815 self.supervisor = supervisor;
816 self
817 }
818
819 pub fn with_daemon_provenance(
820 mut self,
821 pid: u32,
822 started_at_ms: u64,
823 executable_path: Option<PathBuf>,
824 build_git_sha: Option<String>,
825 build_lock_digest: Option<String>,
826 ) -> Self {
827 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
828 let process_start_time = process_start_time(pid);
829 self.daemon_provenance = DaemonProvenanceFacts {
830 build: DaemonBuildProvenance {
831 build_git_sha,
832 build_lock_digest,
833 },
834 pid: Some(pid),
835 started_at_ms: Some(started_at_ms),
836 start_clock: None,
837 executable_path,
838 executable_identity,
839 process_start_time,
840 probe: ExecutableIdentityProbe::default(),
841 };
842 self
843 }
844
845 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
846 self.daemon_provenance.start_clock = Some(clock);
847 self
848 }
849
850 #[cfg(test)]
851 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
852 self.provenance_probe_override = Some(result);
853 self
854 }
855
856 pub fn with_capability_config(
860 self,
861 modules: impl IntoIterator<Item = (String, bool)>,
862 reserved_capabilities: BTreeMap<String, String>,
863 ) -> Self {
864 self.capability_evaluator
865 .configure(modules, reserved_capabilities);
866 self
867 }
868
869 pub fn with_supervisor_rescan(
870 mut self,
871 supervisor: Supervisor,
872 config_path: impl Into<PathBuf>,
873 configured_port: Option<u16>,
874 ) -> Self {
875 self.rescan = Some(SupervisorRescanContext {
876 supervisor,
877 config_path: config_path.into(),
878 configured_port,
879 storage_config: self.storage_config.clone(),
880 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
881 admission_facts_targets: self.admission_facts_targets.clone(),
882 });
883 self
884 }
885
886 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
887 self.connected_clients = connected_clients;
888 self
889 }
890
891 pub fn forwarding(&self) -> Arc<ForwardingTable> {
892 Arc::clone(&self.forwarding)
893 }
894
895 pub(crate) fn counters(&self) -> DaemonCounters {
896 self.counters.clone()
897 }
898
899 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
902 tokio::spawn(async move {
903 loop {
904 self.capability_evaluator
905 .wait_for_change_or_deadline()
906 .await;
907 self.refresh_capability_requirements();
908 }
909 });
910 }
911
912 fn runtime_capability_snapshot(
913 &self,
914 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
915 let runtime = self
916 .supervisor
917 .list()
918 .into_iter()
919 .map(|module| {
920 let status = module.status().map_err(|err| {
921 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
922 })?;
923 Ok(RuntimeModule {
924 module_id: status.module_id,
925 state: status.state,
926 enabled: status.enabled,
927 })
928 })
929 .collect::<Result<Vec<_>, RouterError>>()?;
930 let (_, registrations) = self.registry.list_modules().map_err(|err| {
931 RouterError::backend(
932 0,
933 0,
934 format!("failed to list capability registrations: {err}"),
935 )
936 })?;
937 let registrations = registrations
938 .into_iter()
939 .map(|registration| RegisteredModule {
940 module_id: registration.manifest.module_id,
941 module_version: registration.manifest.module_version,
942 capabilities: registration.manifest.capabilities,
943 })
944 .collect();
945 Ok((runtime, registrations))
946 }
947
948 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
955 let cached_registration = RegisteredModule {
956 module_id: registration.manifest.module_id.clone(),
957 module_version: registration.manifest.module_version.clone(),
958 capabilities: registration.manifest.capabilities.clone(),
959 };
960 if self.capability_evaluator.record_hello(&cached_registration) {
961 warn!(
962 module_id = %cached_registration.module_id,
963 "capability claims drifted from the cached manifest"
964 );
965 }
966 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
967 self.enforce_capability_denies();
968 }
969 self.refresh_capability_requirements();
970 }
971
972 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
976 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
977 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
978 self.supervisor.set_swap_promotion_observer(observer);
979 }
980
981 pub fn refresh_capability_requirements(&self) {
982 match self.runtime_capability_snapshot() {
983 Ok((runtime, registrations)) => {
984 log_requirement_events(
985 self.capability_evaluator
986 .evaluate_now(&runtime, ®istrations),
987 );
988 }
989 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
990 }
991 }
992
993 fn enforce_capability_denies(&self) {
997 let (_, registrations) = match self.registry.list_modules() {
998 Ok(snapshot) => snapshot,
999 Err(err) => {
1000 warn!(error = %err, "failed to read registrations for capability deny census");
1001 return;
1002 }
1003 };
1004 let manifests = registrations
1005 .into_iter()
1006 .map(|registration| {
1007 (
1008 registration.manifest.module_id.clone(),
1009 registration.manifest,
1010 )
1011 })
1012 .collect::<BTreeMap<_, _>>();
1013 let census = match self.forwarding.route_census(None) {
1014 Ok(census) => census,
1015 Err(err) => {
1016 warn!(error = %err, "failed to read route census for capability deny enforcement");
1017 return;
1018 }
1019 };
1020
1021 for (target_module_id, routes) in census {
1022 let Some(target_manifest) = manifests.get(&target_module_id) else {
1023 continue;
1024 };
1025 let mut closed_routes = Vec::new();
1026 let mut module_goodbyes = Vec::new();
1027 for route in routes {
1028 let Principal::Reserved {
1029 module_id: opening_module_id,
1030 } = &route.principal
1031 else {
1032 continue;
1033 };
1034 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1035 continue;
1036 };
1037 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1038 continue;
1039 };
1040
1041 match self.forwarding.release_client_route(
1042 route.goodbye_target.connection_id,
1043 route.goodbye_target.channel,
1044 route.goodbye_target.epoch,
1045 ) {
1046 Ok(RouteRelease::Removed(module_goodbye)) => {
1047 warn!(
1048 opening_module_id,
1049 target_module_id,
1050 capability,
1051 "force-closing route because an attested capability deny edge now matches"
1052 );
1053 closed_routes.push(route);
1054 module_goodbyes.push(module_goodbye);
1055 }
1056 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1057 Err(err) => warn!(
1058 opening_module_id,
1059 target_module_id,
1060 capability,
1061 error = %err,
1062 "failed to force-close capability-denied route"
1063 ),
1064 }
1065 }
1066
1067 if closed_routes.is_empty() {
1068 continue;
1069 }
1070 send_route_control_pushes(
1071 &self.forwarding,
1072 closed_routes,
1073 ClientControlPush::RouteClosed {
1074 module_id: target_module_id,
1075 reason: RouteCloseReason::CapabilityDenied,
1076 drained: false,
1077 abandoned: 0,
1078 excluded_subscriptions: 0,
1079 terminal: Some(false),
1080 },
1081 );
1082 self.emit_route_goodbyes(module_goodbyes);
1083 }
1084 }
1085
1086 fn not_ready_reason(
1092 &self,
1093 registration: &crate::registry::ModuleRegistration,
1094 ) -> Option<NotReadyReason> {
1095 if !registration.ready {
1096 return Some(NotReadyReason {
1097 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1098 capability: None,
1099 });
1100 }
1101 self.first_unprovided_required_capability(registration)
1102 .map(|capability| NotReadyReason {
1103 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1104 capability: Some(capability),
1105 })
1106 }
1107
1108 fn first_unprovided_required_capability(
1126 &self,
1127 registration: &crate::registry::ModuleRegistration,
1128 ) -> Option<String> {
1129 let required = registration
1130 .manifest
1131 .capabilities
1132 .iter()
1133 .flat_map(|declarations| declarations.requires.iter())
1134 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1135 .map(|requirement| requirement.capability.as_str())
1136 .collect::<BTreeSet<_>>();
1137 if required.is_empty() {
1138 return None;
1139 }
1140 let module_id = registration.manifest.module_id.as_str();
1141 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1142 if required
1143 .iter()
1144 .any(|capability| verdict(capability).is_none())
1145 {
1146 self.refresh_capability_requirements();
1147 }
1148 required
1149 .into_iter()
1150 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1151 .map(str::to_string)
1152 }
1153
1154 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1155 self.capability_evaluator
1156 .statuses()
1157 .into_iter()
1158 .map(capability_requirement_status)
1159 .collect()
1160 }
1161
1162 fn deregister_connection(
1169 &self,
1170 connection_id: ConnectionId,
1171 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1172 self.registry.deregister_connection(connection_id)
1173 }
1174
1175 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1176 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1177 return None;
1178 }
1179 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1180 parse_client_control_request(&frame.body)
1181 else {
1182 return None;
1183 };
1184 Some(target_module_id(&target).to_string())
1185 }
1186
1187 pub(crate) fn route_open_capacity_refusal(
1188 &self,
1189 ctx: &RouteCtx,
1190 frame: &Frame,
1191 target_module_id: &str,
1192 limit: usize,
1193 ) -> Result<Frame, RouterError> {
1194 self.route_open_admission_refusal_frame(
1195 ctx,
1196 frame,
1197 target_module_id,
1198 format!(
1199 "connection already has {limit} route.open binds in flight; retry after one settles"
1200 ),
1201 )
1202 }
1203
1204 fn route_open_admission_refusal_frame(
1212 &self,
1213 ctx: &RouteCtx,
1214 frame: &Frame,
1215 target_module_id: &str,
1216 message: impl Into<String>,
1217 ) -> Result<Frame, RouterError> {
1218 self.route_open_refusal_frame(
1219 ctx,
1220 frame,
1221 target_module_id,
1222 "open_admission_full",
1223 error_codes::TARGET_UNAVAILABLE,
1224 message,
1225 )
1226 }
1227
1228 #[cfg(test)]
1234 pub fn handle_control(
1235 &self,
1236 connection_id: ConnectionId,
1237 frame: Frame,
1238 ) -> Result<Vec<Frame>, RouterError> {
1239 match frame.header.ty {
1240 FrameType::Ping => Ok(vec![pong(&frame)?]),
1241 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1242 FrameType::Goodbye => self.handle_goodbye(connection_id),
1243 ty => Ok(vec![control_error_frame(
1244 &frame,
1245 "unsupported_control_frame",
1246 format!("unsupported channel-0 frame {ty:?}"),
1247 )?]),
1248 }
1249 }
1250
1251 pub async fn handle_control_frame(
1252 &self,
1253 ctx: &RouteCtx,
1254 frame: Frame,
1255 ) -> Result<Vec<Frame>, RouterError> {
1256 self.handle_control_frame_timed(ctx, frame, None).await
1257 }
1258
1259 pub(crate) async fn handle_control_frame_timed(
1260 &self,
1261 ctx: &RouteCtx,
1262 frame: Frame,
1263 dispatch_started_at: Option<StdInstant>,
1264 ) -> Result<Vec<Frame>, RouterError> {
1265 match frame.header.ty {
1266 FrameType::Ping => Ok(vec![pong(&frame)?]),
1267 FrameType::Hello => {
1268 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1269 }
1270 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1271 FrameType::Cancel => {
1272 if self
1273 .supervisor
1274 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1275 {
1276 Ok(Vec::new())
1277 } else {
1278 Ok(vec![control_error_frame(
1279 &frame,
1280 "unknown_subscription",
1281 "no supervisor spawn subscription has this correlation id",
1282 )?])
1283 }
1284 }
1285 FrameType::Request => {
1286 if self
1287 .forwarding
1288 .module_endpoint_for_connection(ctx.connection_id)
1289 .map_err(RouterError::Forwarding)?
1290 .is_some()
1291 {
1292 if !is_known_module_request_op(&frame.body) {
1293 return Ok(vec![control_error_frame(
1294 &frame,
1295 "unsupported_control_frame",
1296 "module-originated channel-0 REQUEST is not supported",
1297 )?]);
1298 }
1299 let request = match parse_module_control_request_from_module(&frame.body) {
1300 Ok(request) => request,
1301 Err((err, ControlRequestBodyError::UnknownOp)) => {
1302 return Ok(vec![control_error_frame(
1303 &frame,
1304 "unsupported_control_frame",
1305 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1306 )?])
1307 }
1308 Err((err, ControlRequestBodyError::InvalidBody)) => {
1309 return Ok(vec![control_error_frame(
1310 &frame,
1311 "invalid_control_body",
1312 format!("malformed module control body: {err}"),
1313 )?])
1314 }
1315 };
1316 let op = module_control_request_op(&request);
1317 let corr = frame.header.corr;
1318 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1319 let result =
1320 self.handle_module_control_request(ctx.connection_id, frame, request);
1321 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1322 return result;
1323 }
1324
1325 if is_known_module_request_op(&frame.body) {
1326 return Ok(vec![control_error_frame(
1327 &frame,
1328 "not_registered",
1329 "catalog.update requires an active module registration owned by this connection",
1330 )?]);
1331 }
1332
1333 let request = match parse_client_control_request(&frame.body) {
1334 Ok(request) => request,
1335 Err((err, ControlRequestBodyError::UnknownOp)) => {
1336 return Ok(vec![control_error_frame(
1337 &frame,
1338 "unknown_control_op",
1339 format!("unknown client control op: {err}"),
1340 )?])
1341 }
1342 Err((err, ControlRequestBodyError::InvalidBody)) => {
1343 return Ok(vec![control_error_frame(
1344 &frame,
1345 "invalid_control_body",
1346 format!("malformed client control body: {err}"),
1347 )?])
1348 }
1349 };
1350 let op = client_control_request_op(&request);
1351 let corr = frame.header.corr;
1352 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1353 #[cfg(test)]
1354 if let Some(delay) = self.control_dispatch_delay {
1355 tokio::time::sleep(delay).await;
1356 }
1357 let result = self
1358 .handle_client_control_request(ctx, frame, request)
1359 .await;
1360 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1361 result
1362 }
1363 FrameType::Push => {
1364 let Some(endpoint) = self
1365 .forwarding
1366 .module_endpoint_for_connection(ctx.connection_id)
1367 .map_err(RouterError::Forwarding)?
1368 else {
1369 return Ok(vec![control_error_frame(
1370 &frame,
1371 "unsupported_control_frame",
1372 "client-originated channel-0 PUSH is not supported",
1373 )?]);
1374 };
1375 self.handle_status_update(endpoint, frame)
1376 }
1377 FrameType::Response | FrameType::Error
1378 if self
1379 .forwarding
1380 .module_endpoint_for_connection(ctx.connection_id)
1381 .map_err(RouterError::Forwarding)?
1382 .is_some() =>
1383 {
1384 self.handle_module_relay_response(ctx.connection_id, frame)
1385 }
1386 ty => Ok(vec![control_error_frame(
1387 &frame,
1388 "unsupported_control_frame",
1389 format!("unsupported channel-0 frame {ty:?}"),
1390 )?]),
1391 }
1392 }
1393
1394 pub fn cleanup_connection(
1395 &self,
1396 connection_id: ConnectionId,
1397 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1398 let crash_closed = self
1399 .registry
1400 .get_module_by_connection(connection_id)?
1401 .and_then(|registration| {
1402 self.forwarding
1403 .module_endpoint_for_connection(connection_id)
1404 .ok()
1405 .flatten()
1406 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1407 .map(|routes| (registration.manifest.module_id, routes))
1408 });
1409 if let Some((module_id, routes)) = crash_closed {
1410 let terminal = match self.supervisor.get(&module_id) {
1411 None => false,
1412 Some(module) => match module.will_recover_after_connection_loss() {
1413 Ok(will_recover) => !will_recover,
1414 Err(err) => {
1415 warn!(
1416 %module_id,
1417 error = %err,
1418 "failed to read crash recovery verdict; reporting non-terminal conservatively"
1419 );
1420 false
1421 }
1422 },
1423 };
1424 send_route_control_pushes(
1425 &self.forwarding,
1426 routes,
1427 ClientControlPush::RouteClosed {
1428 module_id,
1429 reason: RouteCloseReason::Crash,
1430 drained: false,
1431 abandoned: 0,
1432 excluded_subscriptions: 0,
1433 terminal: Some(terminal),
1434 },
1435 );
1436 }
1437 let registrations = self.deregister_connection(connection_id);
1438 if let Ok(released_routes) = self.forwarding.cleanup_connection(connection_id) {
1439 self.emit_route_goodbyes(released_routes);
1440 }
1441 if matches!(®istrations, Ok(r) if !r.is_empty()) {
1445 crate::supervise::notify_registration_release();
1446 self.capability_evaluator.wake_deadline_loop();
1447 self.refresh_capability_requirements();
1448 }
1449 self.supervisor.remove_spawn_subscribers(connection_id);
1450 registrations
1451 }
1452
1453 pub(crate) fn handle_route_goodbye(
1454 &self,
1455 connection_id: ConnectionId,
1456 route_channel: u16,
1457 route_epoch: u32,
1458 ) -> Result<bool, RouterError> {
1459 debug!(
1460 connection_id = connection_id.get(),
1461 route_channel, route_epoch, "handling route GOODBYE"
1462 );
1463 let RouteRelease::Removed(released_route) = self
1464 .forwarding
1465 .release_client_route(connection_id, route_channel, route_epoch)
1466 .map_err(RouterError::Forwarding)?
1467 else {
1468 return Ok(false);
1469 };
1470 self.emit_route_goodbyes(vec![released_route]);
1471 Ok(true)
1472 }
1473
1474 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
1475 for released in released_routes {
1476 let frame = match Frame::build_with_version(
1477 released.negotiated_ver,
1478 FrameType::Goodbye,
1479 control_flags(),
1480 released.channel,
1481 released.epoch,
1482 0,
1483 Vec::new(),
1484 ) {
1485 Ok(frame) => frame,
1486 Err(err) => {
1487 warn!(
1488 route_channel = released.channel,
1489 error = %err,
1490 "failed to build route GOODBYE frame"
1491 );
1492 continue;
1493 }
1494 };
1495 if !released.close_on_delivery_failure() {
1496 crate::forwarding::send_module_route_goodbye(
1497 &self.counters,
1498 &released.sink,
1499 frame,
1500 released.module_id.as_deref(),
1501 "client route released",
1502 );
1503 continue;
1504 }
1505 if let Err(err) = released.sink.try_send(frame) {
1506 warn!(
1507 target_connection_id = released.connection_id.get(),
1508 route_channel = released.channel,
1509 error = %err,
1510 "route GOODBYE was not delivered to client; closing target connection"
1511 );
1512 if self
1513 .forwarding
1514 .escalate_client_delivery_failure(
1515 released.connection_id,
1516 released.channel,
1517 released.epoch,
1518 CloseReason::new(
1519 "route_goodbye_delivery_failed",
1520 format!(
1521 "failed to enqueue route GOODBYE for channel {}: {err}",
1522 released.channel
1523 ),
1524 ),
1525 crate::forwarding::UndeliveredFrame {
1526 module_id: released.module_id.as_deref(),
1527 sink: &released.sink,
1528 },
1529 )
1530 .unwrap_or(false)
1531 {
1532 self.counters.increment_goodbye_relay_client_failed();
1533 }
1534 }
1535 }
1536 }
1537
1538 fn send_abandoned_route_bind_goodbye(
1550 &self,
1551 module_sink: &crate::FrameSink,
1552 negotiated_ver: u8,
1553 module_channel: u16,
1554 module_epoch: u32,
1555 ) {
1556 let frame = match Frame::build_with_version(
1557 negotiated_ver,
1558 FrameType::Goodbye,
1559 control_flags(),
1560 module_channel,
1561 module_epoch,
1562 0,
1563 Vec::new(),
1564 ) {
1565 Ok(frame) => frame,
1566 Err(err) => {
1567 warn!(
1568 route_channel = module_channel,
1569 error = %err,
1570 "failed to build GOODBYE for abandoned route.bind"
1571 );
1572 return;
1573 }
1574 };
1575 crate::forwarding::send_module_route_goodbye(
1576 &self.counters,
1577 module_sink,
1578 frame,
1579 None,
1580 "abandoned route.bind",
1581 );
1582 }
1583
1584 fn handle_hello(
1585 &self,
1586 connection_id: ConnectionId,
1587 sink: Option<crate::FrameSink>,
1588 frame: Frame,
1589 ) -> Result<Vec<Frame>, RouterError> {
1590 debug!(
1591 connection_id = connection_id.get(),
1592 corr = frame.header.corr,
1593 "handling HELLO"
1594 );
1595 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
1596 Ok(value) => value,
1597 Err(err) => {
1598 return Ok(vec![control_error_frame(
1599 &frame,
1600 "invalid_hello",
1601 format!("malformed HELLO body: {err}"),
1602 )?])
1603 }
1604 };
1605 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
1606 return Ok(vec![control_error_frame(
1607 &frame,
1608 "invalid_capability_grammar",
1609 err.to_string(),
1610 )?]);
1611 }
1612 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
1613 return Ok(vec![control_error_frame(
1614 &frame,
1615 "invalid_manifest",
1616 err.to_string(),
1617 )?]);
1618 }
1619 if let Some(provenance) = hello_value
1620 .get("manifest")
1621 .and_then(|manifest| manifest.get("provenance"))
1622 {
1623 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
1624 return Ok(vec![control_error_frame(
1625 &frame,
1626 "invalid_manifest",
1627 format!("malformed manifest provenance: {err}"),
1628 )?]);
1629 }
1630 }
1631 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
1632 Ok(hello) => hello,
1633 Err(err) => {
1634 return Ok(vec![control_error_frame(
1635 &frame,
1636 "invalid_hello",
1637 format!("malformed HELLO body: {err}"),
1638 )?])
1639 }
1640 };
1641
1642 if hello.protocol_ver != hello.manifest.protocol_ver {
1643 return Ok(vec![control_error_frame(
1644 &frame,
1645 "invalid_manifest",
1646 format!(
1647 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
1648 hello.protocol_ver, hello.manifest.protocol_ver
1649 ),
1650 )?]);
1651 }
1652
1653 if hello.manifest.module_id.trim().is_empty() {
1654 return Ok(vec![control_error_frame(
1655 &frame,
1656 "invalid_manifest",
1657 "manifest module_id must not be empty",
1658 )?]);
1659 }
1660
1661 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
1662 Ok(negotiated_ver) => negotiated_ver,
1663 Err(message) => {
1664 return Ok(vec![control_error_frame(
1665 &frame,
1666 "version_unsupported",
1667 message,
1668 )?])
1669 }
1670 };
1671
1672 let swap_admission = self
1680 .supervisor
1681 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
1682 if swap_admission == SwapHelloAdmission::Refused {
1683 warn!(
1684 module_id = %hello.manifest.module_id,
1685 connection_id = connection_id.get(),
1686 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
1687 );
1688 return Ok(vec![control_error_frame(
1689 &frame,
1690 "swap_token_invalid",
1691 format!(
1692 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
1693 hello.manifest.module_id
1694 ),
1695 )?]);
1696 }
1697 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
1698
1699 if let Some(rejection) = (!swap_candidate)
1707 .then(|| {
1708 self.supervisor.reserved_hello_rejection(
1709 &hello.manifest.module_id,
1710 hello.launch_nonce.as_deref(),
1711 )
1712 })
1713 .flatten()
1714 {
1715 let message = match rejection {
1716 ReservedHelloRejection::Exact { module_id } => format!(
1717 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
1718 ),
1719 ReservedHelloRejection::Prefix {
1720 prefix,
1721 owner_module_id,
1722 } => format!(
1723 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
1724 hello.manifest.module_id
1725 ),
1726 };
1727 return Ok(vec![control_error_frame(
1728 &frame,
1729 "reserved_module",
1730 message,
1731 )?]);
1732 }
1733
1734 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
1735 &hello.manifest.module_id,
1736 hello.manifest.capabilities.as_ref(),
1737 );
1738 if let Some(refusal) = reserved_capability_refusals.first() {
1739 let capability = refusal.capability.clone();
1740 let bound_module = refusal.claimants[0].clone();
1741 log_duplicate_claim_events(reserved_capability_refusals);
1742 return Ok(vec![control_error_frame(
1743 &frame,
1744 "reserved_capability",
1745 format!(
1746 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
1747 capability, bound_module, hello.manifest.module_id
1748 ),
1749 )?]);
1750 }
1751
1752 if self
1755 .forwarding
1756 .connection_has_client_routes(connection_id)
1757 .map_err(RouterError::Forwarding)?
1758 {
1759 return Ok(vec![control_error_frame(
1760 &frame,
1761 "invalid_hello",
1762 "connection has open client routes and cannot also register as a module",
1763 )?]);
1764 }
1765
1766 let control_ops = effective_module_control_ops(hello.control_ops);
1767 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
1770 if swap_candidate {
1771 return self.register_swap_candidate(
1772 connection_id,
1773 sink,
1774 &frame,
1775 hello.manifest,
1776 negotiated_ver,
1777 control_ops,
1778 hello_ack,
1779 );
1780 }
1781 let registration = match self.registry.register_with_control_ops(
1782 hello.manifest,
1783 negotiated_ver,
1784 connection_id,
1785 control_ops,
1786 ) {
1787 Ok(registration) => registration,
1788 Err(RegistryError::DuplicateModuleId { module_id }) => {
1789 return Ok(vec![control_error_frame(
1790 &frame,
1791 "duplicate_module_id",
1792 format!(
1793 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
1794 ),
1795 )?])
1796 }
1797 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
1798 return Ok(vec![control_error_frame(
1799 &frame,
1800 "invalid_module_id",
1801 err.to_string(),
1802 )?])
1803 }
1804 Err(err) => {
1805 return Ok(vec![control_error_frame(
1806 &frame,
1807 "registry_error",
1808 err.to_string(),
1809 )?])
1810 }
1811 };
1812
1813 let reply = if let Some(sink) = sink {
1814 let concurrency = manifest_concurrency(®istration.manifest);
1830 if let Err(err) = self.forwarding.register_module_connection_acked(
1831 connection_id,
1832 registration.manifest.module_id.clone(),
1833 negotiated_ver,
1834 concurrency,
1835 sink,
1836 hello_ack,
1837 ) {
1838 if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
1842 crate::supervise::notify_registration_release();
1843 }
1844 return Ok(vec![control_error_frame(
1845 &frame,
1846 forwarding_error_code(&err),
1847 err.to_string(),
1848 )?]);
1849 }
1850 Vec::new()
1851 } else {
1852 vec![hello_ack]
1855 };
1856
1857 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
1866 info!(
1867 module_id = %registration.manifest.module_id,
1868 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
1869 );
1870 }
1871
1872 self.apply_registration_capabilities(®istration);
1873
1874 info!(
1875 module_id = %registration.manifest.module_id,
1876 module_version = %registration.manifest.module_version,
1877 negotiated_ver,
1878 routable_provider = manifest_provides_routable_role(®istration.manifest),
1879 connection_id = connection_id.get(),
1880 "module registered"
1881 );
1882
1883 Ok(reply)
1884 }
1885
1886 #[allow(clippy::too_many_arguments)]
1896 fn register_swap_candidate(
1897 &self,
1898 connection_id: ConnectionId,
1899 sink: Option<crate::FrameSink>,
1900 frame: &Frame,
1901 manifest: ModuleManifest,
1902 negotiated_ver: u8,
1903 control_ops: Vec<String>,
1904 hello_ack: Frame,
1905 ) -> Result<Vec<Frame>, RouterError> {
1906 let module_id = manifest.module_id.clone();
1907 let registration = match self.registry.register_candidate_with_control_ops(
1908 manifest,
1909 negotiated_ver,
1910 connection_id,
1911 control_ops,
1912 ) {
1913 Ok(registration) => registration,
1914 Err(RegistryError::DuplicateModuleId { module_id }) => {
1915 return Ok(vec![control_error_frame(
1916 frame,
1917 "duplicate_module_id",
1918 format!(
1919 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
1920 ),
1921 )?])
1922 }
1923 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
1924 return Ok(vec![control_error_frame(
1925 frame,
1926 "invalid_module_id",
1927 err.to_string(),
1928 )?])
1929 }
1930 Err(err) => {
1931 return Ok(vec![control_error_frame(
1932 frame,
1933 "registry_error",
1934 err.to_string(),
1935 )?])
1936 }
1937 };
1938 let reply = if let Some(sink) = sink {
1939 let concurrency = manifest_concurrency(®istration.manifest);
1943 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
1944 connection_id,
1945 module_id.clone(),
1946 negotiated_ver,
1947 concurrency,
1948 sink,
1949 hello_ack,
1950 ) {
1951 if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
1952 crate::supervise::notify_registration_release();
1953 }
1954 return Ok(vec![control_error_frame(
1955 frame,
1956 forwarding_error_code(&err),
1957 err.to_string(),
1958 )?]);
1959 }
1960 Vec::new()
1961 } else {
1962 vec![hello_ack]
1963 };
1964 self.supervisor.mark_swap_candidate_admitted(&module_id);
1965 info!(
1966 module_id = %module_id,
1967 module_version = %registration.manifest.module_version,
1968 negotiated_ver,
1969 ready = registration.ready,
1970 connection_id = connection_id.get(),
1971 "swap candidate registered; not routable until cutover"
1972 );
1973 Ok(reply)
1974 }
1975
1976 fn build_hello_ack(
1977 &self,
1978 frame: &Frame,
1979 negotiated_ver: u8,
1980 module_id: &str,
1981 ) -> Result<Frame, RouterError> {
1982 let ack = ModuleHelloAckBody {
1983 negotiated_ver,
1984 subc_ops: module_subc_ops(),
1985 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
1986 storage: self
1987 .storage_config
1988 .as_ref()
1989 .map(|cfg| cfg.descriptor_for(module_id)),
1990 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
1991 };
1992 let body = serde_json::to_vec(&ack).map_err(|err| {
1993 RouterError::backend(
1994 0,
1995 frame.header.corr,
1996 format!("failed to encode HELLO_ACK: {err}"),
1997 )
1998 })?;
1999
2000 Frame::build_with_version(
2001 negotiated_ver,
2002 FrameType::HelloAck,
2003 control_flags(),
2004 0,
2005 0,
2006 frame.header.corr,
2007 body,
2008 )
2009 .map_err(RouterError::FrameBuild)
2010 }
2011
2012 async fn handle_client_control_request(
2013 &self,
2014 ctx: &RouteCtx,
2015 frame: Frame,
2016 request: ClientControlRequest,
2017 ) -> Result<Vec<Frame>, RouterError> {
2018 match request {
2019 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2020 ClientControlRequest::CatalogList { module_id } => {
2021 self.handle_catalog_list(frame, module_id)
2022 }
2023 ClientControlRequest::RouteOpen {
2024 target,
2025 identity,
2026 consumer_identity,
2027 consumer_capabilities,
2028 admission_facts,
2029 } => {
2030 self.handle_route_open(
2031 ctx,
2032 frame,
2033 RouteOpenRequest {
2034 target,
2035 identity,
2036 consumer_identity,
2037 consumer_capabilities,
2038 admission_facts,
2039 },
2040 )
2041 .await
2042 }
2043 ClientControlRequest::RoutePoll {
2044 route_channel,
2045 route_epoch,
2046 kind,
2047 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2048 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2049 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2050 self.handle_supervisor_spawn_snapshot(frame)
2051 }
2052 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2053 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2054 }
2055 ClientControlRequest::SupervisorRestart {
2056 module_id,
2057 drain_timeout_ms,
2058 } => {
2059 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms)
2060 .await
2061 }
2062 ClientControlRequest::SupervisorSwap {
2063 module_id,
2064 ready_timeout_ms,
2065 } => {
2066 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms)
2067 .await
2068 }
2069 ClientControlRequest::SupervisorReload { module_id } => {
2070 self.handle_supervisor_reload(frame, module_id).await
2071 }
2072 ClientControlRequest::SupervisorRescan { preview } => {
2073 self.handle_supervisor_rescan(frame, preview).await
2074 }
2075 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2076 self.handle_supervisor_release_reserved(frame, module_id)
2077 .await
2078 }
2079 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2080 self.handle_supervisor_set_enabled(frame, module_id, enabled)
2081 .await
2082 }
2083 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2084 self.handle_supervisor_health_probe(frame, module_id).await
2085 }
2086 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2087 ClientControlRequest::SupervisorRoutes { module_id } => {
2088 self.handle_supervisor_routes(frame, module_id)
2089 }
2090 ClientControlRequest::SupervisorProvenance { module_id } => {
2091 self.handle_supervisor_provenance(frame, module_id).await
2092 }
2093 ClientControlRequest::SupervisorStderrTail {
2094 module_id,
2095 max_lines,
2096 max_bytes,
2097 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2098 ClientControlRequest::SupervisorTerminals { module_id } => {
2099 self.handle_supervisor_terminals(frame, module_id).await
2100 }
2101 }
2102 }
2103
2104 fn handle_module_control_request(
2105 &self,
2106 connection_id: ConnectionId,
2107 frame: Frame,
2108 request: ModuleControlRequestFromModule,
2109 ) -> Result<Vec<Frame>, RouterError> {
2110 match request {
2111 ModuleControlRequestFromModule::CatalogUpdate {
2112 provides,
2113 capabilities,
2114 ready,
2115 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2116 ModuleControlRequestFromModule::LiveRoots {} => {
2117 let registered = self
2118 .registry
2119 .get_module_by_connection(connection_id)
2120 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2121 let Some(registration) = registered else {
2122 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2123 };
2124 let response = self
2125 .forwarding
2126 .live_roots(®istration.manifest.module_id)
2127 .map_err(RouterError::Forwarding)?;
2128 Ok(vec![control_response_body_frame(
2129 &frame,
2130 &response,
2131 "ModuleControlResponseToModule::LiveRoots",
2132 )?])
2133 }
2134 }
2135 }
2136
2137 fn handle_catalog_update(
2138 &self,
2139 connection_id: ConnectionId,
2140 frame: Frame,
2141 provides: Vec<ProviderRole>,
2142 capabilities: Option<CapabilityDeclarations>,
2143 ready: Option<bool>,
2144 ) -> Result<Vec<Frame>, RouterError> {
2145 self.refresh_capability_requirements();
2146 let Some(registration) = self
2147 .registry
2148 .get_module_by_connection(connection_id)
2149 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2150 else {
2151 return Ok(vec![control_error_frame(
2152 &frame,
2153 "not_registered",
2154 "catalog.update requires an active module registration owned by this connection",
2155 )?]);
2156 };
2157
2158 if let Some(message) =
2159 catalog_update_frozen_field_message(®istration.manifest, &provides)
2160 {
2161 return Ok(vec![control_error_frame(
2162 &frame,
2163 "catalog_update_frozen_field",
2164 message,
2165 )?]);
2166 }
2167
2168 let mut candidate = registration.manifest.clone();
2169 candidate.provides = provides.clone();
2170 candidate.capabilities = capabilities
2171 .clone()
2172 .or_else(|| registration.manifest.capabilities.clone());
2173 if let Err(err) = candidate.validate_capability_grammar() {
2174 return Ok(vec![control_error_frame(
2175 &frame,
2176 "invalid_capability_grammar",
2177 err.to_string(),
2178 )?]);
2179 }
2180
2181 let updated = self
2182 .registry
2183 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
2184 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2185 if updated.is_none() {
2186 return Ok(vec![control_error_frame(
2187 &frame,
2188 "not_registered",
2189 "catalog.update requires an active module registration owned by this connection",
2190 )?]);
2191 }
2192 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
2193 log_duplicate_claim_events(
2194 self.capability_evaluator
2195 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
2196 );
2197 }
2198 if capability_census_trigger(
2199 registration.manifest.capabilities.as_ref(),
2200 updated
2201 .as_ref()
2202 .and_then(|entry| entry.manifest.capabilities.as_ref()),
2203 ) {
2204 self.enforce_capability_denies();
2205 }
2206 self.refresh_capability_requirements();
2207
2208 let response = ModuleControlResponseToModule::CatalogUpdate {};
2209 control_response_body_frame(
2210 &frame,
2211 &response,
2212 "ModuleControlResponseToModule::CatalogUpdate",
2213 )
2214 .map(|frame| vec![frame])
2215 }
2216
2217 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
2218 self.refresh_capability_requirements();
2219 let mut counters = self.counters.snapshot();
2225 if let (Ok((connections_with_routes, max)), Some(obj)) = (
2226 self.forwarding.client_route_concentration(),
2227 counters.as_object_mut(),
2228 ) {
2229 obj.insert(
2230 "client_connections_with_routes".into(),
2231 connections_with_routes.into(),
2232 );
2233 obj.insert("max_routes_on_one_connection".into(), max.into());
2234 }
2235 if let (Some(open_breakers), Some(obj)) = (
2241 self.route_bind_breakers.open_snapshot(),
2242 counters.as_object_mut(),
2243 ) {
2244 obj.insert("route_bind_breakers_open".into(), open_breakers);
2245 }
2246 let response = ClientControlResponse::ServerDescribe {
2247 protocol_ver: PROTOCOL_VERSION,
2248 subc_ops: subc_ops(),
2249 capabilities: self.subc_capabilities.as_ref().to_vec(),
2250 connected_clients: self.connected_clients.count(),
2251 counters: Some(counters),
2252 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
2253 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
2254 capability_requirements: self.capability_requirement_statuses(),
2255 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2256 };
2257 Ok(vec![control_response_body_frame(
2258 &frame,
2259 &response,
2260 "ClientControlResponse::ServerDescribe",
2261 )?])
2262 }
2263
2264 fn handle_catalog_list(
2265 &self,
2266 frame: Frame,
2267 module_id: Option<String>,
2268 ) -> Result<Vec<Frame>, RouterError> {
2269 let (generation, modules) = self.registry.list_modules().map_err(|err| {
2270 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
2271 })?;
2272 let entries = modules
2273 .into_iter()
2274 .filter(|registration| {
2275 module_id
2276 .as_deref()
2277 .map(|wanted| registration.manifest.module_id == wanted)
2278 .unwrap_or(true)
2279 })
2280 .map(|registration| {
2281 let not_ready = self.not_ready_reason(®istration);
2282 let roles = registration.manifest.provides;
2283 CatalogEntry {
2284 module_id: registration.manifest.module_id,
2285 ready: not_ready.is_none(),
2286 not_ready,
2287 module_version: Some(registration.manifest.module_version),
2288 roles,
2289 control_ops: registration.control_ops,
2290 capabilities: registration.manifest.capabilities,
2291 self_signals: registration.manifest.self_signals,
2292 }
2293 })
2294 .collect();
2295 let response = ClientControlResponse::CatalogList {
2296 generation,
2297 modules: entries,
2298 subc_ops: subc_ops(),
2299 };
2300 Ok(vec![control_response_body_frame(
2301 &frame,
2302 &response,
2303 "ClientControlResponse::CatalogList",
2304 )?])
2305 }
2306
2307 fn route_open_principal(
2308 &self,
2309 frame: &Frame,
2310 consumer_identity: Option<ConsumerIdentity>,
2311 ) -> Result<Result<Principal, Frame>, RouterError> {
2312 let Some(consumer_identity) = consumer_identity else {
2313 return Ok(Ok(Principal::Direct));
2314 };
2315
2316 if self.supervisor.spawned_consumer_authorized(
2317 &consumer_identity.module_id,
2318 &consumer_identity.launch_nonce,
2319 ) {
2320 return Ok(Ok(Principal::Reserved {
2321 module_id: consumer_identity.module_id,
2322 }));
2323 }
2324
2325 Ok(Err(control_error_frame(
2326 frame,
2327 "bad_consumer_identity",
2328 format!(
2329 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
2330 consumer_identity.module_id
2331 ),
2332 )?))
2333 }
2334
2335 fn route_open_refusal_frame(
2341 &self,
2342 ctx: &RouteCtx,
2343 frame: &Frame,
2344 module_id: &str,
2345 reason: &'static str,
2346 code: &'static str,
2347 message: impl Into<String>,
2348 ) -> Result<Frame, RouterError> {
2349 self.observe_route_open_refusal(ctx, module_id, reason, code);
2350 control_error_frame(frame, code, message.into())
2351 }
2352
2353 fn route_open_breaker_refusal_frame(
2371 &self,
2372 ctx: &RouteCtx,
2373 frame: &Frame,
2374 module_id: &str,
2375 consecutive_timeouts: u32,
2376 retry_in: Duration,
2377 probe_in_flight: bool,
2378 ) -> Result<Frame, RouterError> {
2379 self.counters
2380 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
2381 debug!(
2382 target: "control",
2383 code = "module_timeout",
2384 module_id = ?module_id,
2385 connection_id = ctx.connection_id.get(),
2386 consecutive_timeouts,
2387 retry_in_ms = retry_in.as_millis() as u64,
2388 probe_in_flight,
2389 "route.open refused by open bind-relay breaker"
2390 );
2391 let detail = if probe_in_flight {
2392 "one probe bind is already in flight; retry once it settles".to_string()
2393 } else {
2394 format!("not relaying for another {retry_in:?}")
2395 };
2396 control_error_frame(
2397 frame,
2398 "module_timeout",
2399 format!(
2400 "module_id '{module_id}' failed {consecutive_timeouts} consecutive route.bind \
2401 relays; {detail}"
2402 ),
2403 )
2404 }
2405
2406 fn observe_route_open_refusal(
2419 &self,
2420 ctx: &RouteCtx,
2421 module_id: &str,
2422 reason: &'static str,
2423 code: &'static str,
2424 ) {
2425 self.counters.increment_route_open_refused(code);
2426 info!(
2427 target: "control",
2428 code,
2429 reason,
2430 module_id = ?module_id,
2431 connection_id = ctx.connection_id.get(),
2432 "route.open refused"
2433 );
2434 }
2435
2436 fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
2481 self.counters.increment_route_open_accepted(principal);
2482 info!(
2483 target: "control",
2484 principal,
2485 module_id,
2486 connection_id = ctx.connection_id.get(),
2487 "route.open accepted"
2488 );
2489 }
2490
2491 fn supervised_absent_route_open_refusal_frame(
2492 &self,
2493 ctx: &RouteCtx,
2494 frame: &Frame,
2495 module_id: &str,
2496 code: &'static str,
2497 status: &crate::supervise::ModuleStatus,
2498 ) -> Result<Frame, RouterError> {
2499 self.counters.increment_route_open_refused(code);
2500 info!(
2501 target: "control",
2502 code,
2503 reason = "supervised_not_registered",
2504 module_id = ?module_id,
2505 connection_id = ctx.connection_id.get(),
2506 state = %status.state,
2507 enabled = status.enabled,
2508 live = status.live,
2509 "route.open refused"
2510 );
2511 control_error_frame(
2512 frame,
2513 code,
2514 format!(
2515 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
2516 status.state, status.enabled, status.live
2517 ),
2518 )
2519 }
2520
2521 async fn handle_route_open(
2522 &self,
2523 ctx: &RouteCtx,
2524 frame: Frame,
2525 request: RouteOpenRequest,
2526 ) -> Result<Vec<Frame>, RouterError> {
2527 let RouteOpenRequest {
2528 target,
2529 mut identity,
2530 consumer_identity,
2531 consumer_capabilities,
2532 admission_facts,
2533 } = request;
2534 let target_module_id = target_module_id(&target).to_string();
2535 debug!(
2536 connection_id = ctx.connection_id.get(),
2537 corr = frame.header.corr,
2538 module_id = %target_module_id,
2539 "handling route.open"
2540 );
2541
2542 let Some(registration) = self
2561 .registry
2562 .get_module(&target_module_id)
2563 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2564 else {
2565 if let Some((status, warming)) =
2566 self.supervisor_status(&target_module_id, frame.header.corr)?
2567 {
2568 if status.protocol == ModuleProtocol::None {
2575 return Ok(vec![self.route_open_refusal_frame(
2576 ctx,
2577 &frame,
2578 &target_module_id,
2579 "protocol_none",
2580 error_codes::MODULE_NO_PROTOCOL,
2581 format!(
2582 "module_id '{target_module_id}' is declared protocol: none; \
2583 it speaks no subc wire and serves no routes"
2584 ),
2585 )?]);
2586 }
2587 let code = if warming {
2588 "module_warming"
2589 } else {
2590 "target_unavailable"
2591 };
2592 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
2593 ctx,
2594 &frame,
2595 &target_module_id,
2596 code,
2597 &status,
2598 )?]);
2599 }
2600 if let Some(removed_ago_ms) =
2601 self.supervisor.removal_tombstone_age_ms(&target_module_id)
2602 {
2603 return Ok(vec![self.route_open_refusal_frame(
2604 ctx,
2605 &frame,
2606 &target_module_id,
2607 "removed",
2608 error_codes::MODULE_REMOVED,
2609 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
2610 )?]);
2611 }
2612 return Ok(vec![self.route_open_refusal_frame(
2613 ctx,
2614 &frame,
2615 &target_module_id,
2616 "not_registered",
2617 error_codes::UNKNOWN_MODULE,
2618 format!("module_id '{target_module_id}' is not registered"),
2619 )?]);
2620 };
2621
2622 if !registration.ready {
2626 self.counters
2627 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
2628 info!(
2629 target: "control",
2630 code = error_codes::MODULE_WARMING,
2631 module_id = ?target_module_id,
2632 connection_id = ctx.connection_id.get(),
2633 reason = "declared_not_ready",
2634 "route.open refused"
2635 );
2636 return Ok(vec![control_error_body_frame(
2637 &frame,
2638 ErrorBody {
2639 code: error_codes::MODULE_WARMING.to_string(),
2640 message: format!(
2641 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
2642 ),
2643 detail: Some(serde_json::json!({
2644 "reason": "declared_not_ready"
2645 })),
2646 },
2647 )?]);
2648 }
2649
2650 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
2669 self.counters
2670 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
2671 info!(
2672 target: "control",
2673 code = error_codes::MODULE_WARMING,
2674 module_id = ?target_module_id,
2675 connection_id = ctx.connection_id.get(),
2676 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
2677 capability = %capability,
2678 "route.open refused"
2679 );
2680 return Ok(vec![control_error_body_frame(
2681 &frame,
2682 ErrorBody {
2683 code: error_codes::MODULE_WARMING.to_string(),
2684 message: format!(
2685 "module_id '{target_module_id}' requires capability '{capability}', \
2686 which no registered module provides; retry"
2687 ),
2688 detail: Some(serde_json::json!({
2689 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
2690 "capability": capability,
2691 })),
2692 },
2693 )?]);
2694 }
2695
2696 if !target_has_required_role(&target, ®istration.manifest.provides) {
2697 return Ok(vec![self.route_open_refusal_frame(
2698 ctx,
2699 &frame,
2700 &target_module_id,
2701 "role_not_provided",
2702 "target_unavailable",
2703 format!("module_id '{target_module_id}' does not provide the requested target"),
2704 )?]);
2705 }
2706
2707 if registration.state != ChannelState::Active {
2708 return Ok(vec![self.route_open_refusal_frame(
2709 ctx,
2710 &frame,
2711 &target_module_id,
2712 "registration_not_active",
2713 "target_unavailable",
2714 format!("module_id '{target_module_id}' is not active"),
2715 )?]);
2716 }
2717
2718 if self
2719 .forwarding
2720 .module_is_draining(&target_module_id)
2721 .map_err(RouterError::Forwarding)?
2722 {
2723 return Ok(vec![self.route_open_refusal_frame(
2724 ctx,
2725 &frame,
2726 &target_module_id,
2727 "reloading",
2728 "module_reloading",
2729 format!("module_id '{target_module_id}' is reloading"),
2730 )?]);
2731 }
2732
2733 if self
2734 .process_liveness
2735 .as_ref()
2736 .and_then(|process_liveness| process_liveness.process_live(&target_module_id))
2737 == Some(false)
2738 {
2739 return Ok(vec![self.route_open_refusal_frame(
2740 ctx,
2741 &frame,
2742 &target_module_id,
2743 "supervisor_not_live",
2744 "target_unavailable",
2745 format!("module_id '{target_module_id}' is not live"),
2746 )?]);
2747 }
2748
2749 if !self
2750 .forwarding
2751 .has_live_module_connection(&target_module_id)
2752 .map_err(RouterError::Forwarding)?
2753 {
2754 return Ok(vec![self.route_open_refusal_frame(
2755 ctx,
2756 &frame,
2757 &target_module_id,
2758 "no_forwarding_connection",
2759 "target_unavailable",
2760 format!("module_id '{target_module_id}' has no live forwarding connection"),
2761 )?]);
2762 }
2763
2764 if let Some(error) =
2765 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
2766 {
2767 self.observe_route_open_refusal(
2768 ctx,
2769 &target_module_id,
2770 "op_not_allowed",
2771 "op_not_allowed",
2772 );
2773 return Ok(vec![error]);
2774 }
2775
2776 let principal = match self.route_open_principal(&frame, consumer_identity)? {
2777 Ok(principal) => principal,
2778 Err(error) => {
2779 self.observe_route_open_refusal(
2780 ctx,
2781 &target_module_id,
2782 "bad_consumer_identity",
2783 "bad_consumer_identity",
2784 );
2785 return Ok(vec![error]);
2786 }
2787 };
2788
2789 if let Principal::Reserved {
2793 module_id: opening_module_id,
2794 } = &principal
2795 {
2796 if let Some(opening_registration) = self
2797 .registry
2798 .get_module(opening_module_id)
2799 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2800 {
2801 if let Some(capability) =
2802 denied_capability(&opening_registration.manifest, ®istration.manifest)
2803 {
2804 warn!(
2805 opening_module_id,
2806 target_module_id,
2807 capability,
2808 "refusing route.open because an attested capability deny edge matches"
2809 );
2810 return Ok(vec![self.route_open_refusal_frame(
2811 ctx,
2812 &frame,
2813 &target_module_id,
2814 "capability_deny_edge",
2815 "capability_forbidden",
2816 format!(
2817 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
2818 ),
2819 )?]);
2820 }
2821 }
2822 }
2823
2824 if admission_facts.is_some() {
2825 let carrier_matches = matches!(
2826 &principal,
2827 Principal::Reserved { module_id }
2828 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
2829 );
2830 if !carrier_matches {
2831 return Ok(vec![self.route_open_refusal_frame(
2832 ctx,
2833 &frame,
2834 &target_module_id,
2835 "admission_facts_carrier_not_permitted",
2836 "admission_facts_not_permitted",
2837 "admission facts may only be carried by the configured reserved module",
2838 )?]);
2839 }
2840
2841 let target_allowed = self
2842 .admission_facts_targets
2843 .as_ref()
2844 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
2845 if !target_allowed {
2846 return Ok(vec![self.route_open_refusal_frame(
2847 ctx,
2848 &frame,
2849 &target_module_id,
2850 "admission_facts_target_not_listed",
2851 "admission_facts_target_not_allowed",
2852 format!(
2853 "admission facts are not permitted for target module_id '{target_module_id}'"
2854 ),
2855 )?]);
2856 }
2857
2858 }
2862
2863 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
2885 Ok(project_root) => project_root,
2886 Err(err) => {
2887 return Ok(vec![control_error_frame(
2888 &frame,
2889 "invalid_project_root",
2890 err.to_string(),
2891 )?])
2892 }
2893 };
2894 identity.project_root = project_root.as_path().to_path_buf();
2895
2896 let _concurrency_guard = match self
2907 .route_bind_concurrency
2908 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
2909 {
2910 Ok(guard) => guard,
2911 Err(in_flight) => {
2912 return Ok(vec![self.route_open_admission_refusal_frame(
2913 ctx,
2914 &frame,
2915 &target_module_id,
2916 format!(
2917 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
2918 ),
2919 )?]);
2920 }
2921 };
2922
2923 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
2926 RouteBindAdmission::Admitted { guard, probe } => {
2927 if probe {
2928 info!(
2929 module_id = %target_module_id,
2930 connection_id = ctx.connection_id.get(),
2931 "route.bind breaker half-open: admitting one probe"
2932 );
2933 }
2934 guard
2935 }
2936 RouteBindAdmission::Refused {
2937 consecutive_timeouts,
2938 retry_in,
2939 probe_in_flight,
2940 } => {
2941 return Ok(vec![self.route_open_breaker_refusal_frame(
2942 ctx,
2943 &frame,
2944 &target_module_id,
2945 consecutive_timeouts,
2946 retry_in,
2947 probe_in_flight,
2948 )?]);
2949 }
2950 };
2951
2952 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
2957 let relay_deadline = Instant::now() + route_bind_relay_timeout;
2958 let pending = match self
2959 .forwarding
2960 .begin_route_bind_relay_for(
2961 ctx.connection_id,
2962 ctx.egress.clone(),
2963 response_version(&frame),
2964 frame.header.corr,
2965 &target_module_id,
2966 principal.clone(),
2967 Some(project_root),
2968 relay_deadline,
2969 )
2970 .await
2971 {
2972 Ok(pending) => pending,
2973 Err(err) => {
2974 return Ok(vec![self.route_open_refusal_frame(
2975 ctx,
2976 &frame,
2977 &target_module_id,
2978 "relay_reservation_failed",
2979 forwarding_error_code(&err),
2980 err.to_string(),
2981 )?])
2982 }
2983 };
2984 let crate::forwarding::PendingRouteBindRelay {
2985 endpoint,
2986 module_sink,
2987 negotiated_ver,
2988 client_channel,
2989 client_epoch,
2990 module_channel,
2991 module_epoch,
2992 corr: relay_corr,
2993 receiver,
2994 } = pending;
2995 let mut reservation =
2996 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
2997
2998 debug!(
2999 connection_id = ctx.connection_id.get(),
3000 client_channel,
3001 client_epoch,
3002 module_channel,
3003 module_epoch,
3004 "reserved route handle pair"
3005 );
3006 let principal_label = match &principal {
3009 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
3010 Principal::Direct => "direct".to_string(),
3011 other => format!("{other:?}"),
3012 };
3013 let relay = ModuleControlRequest::RouteBind {
3014 route_channel: module_channel,
3015 epoch: module_epoch,
3016 target,
3017 identity,
3018 principal: Some(principal),
3019 consumer_capabilities,
3020 admission_facts,
3021 };
3022 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
3023 RouterError::backend(
3024 0,
3025 frame.header.corr,
3026 format!("failed to encode route.bind request: {err}"),
3027 )
3028 })?;
3029 let relay_frame = Frame::build_with_version(
3030 negotiated_ver,
3031 FrameType::Request,
3032 control_flags(),
3033 0,
3034 0,
3035 relay_corr,
3036 relay_body,
3037 )
3038 .map_err(RouterError::FrameBuild)?;
3039
3040 if let Err(err) = module_sink.send(relay_frame).await {
3041 reservation.release_and_disarm();
3042 return Ok(vec![self.route_open_refusal_frame(
3043 ctx,
3044 &frame,
3045 &target_module_id,
3046 "relay_send_failed",
3047 "target_unavailable",
3048 err.to_string(),
3049 )?]);
3050 }
3051
3052 if !self
3053 .forwarding
3054 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
3055 .map_err(RouterError::Forwarding)?
3056 {
3057 self.send_abandoned_route_bind_goodbye(
3058 &module_sink,
3059 negotiated_ver,
3060 module_channel,
3061 module_epoch,
3062 );
3063 }
3064
3065 match timeout_at(relay_deadline, receiver).await {
3066 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
3067 reservation.disarm();
3068 if breaker.record_accepted() {
3069 info!(
3070 module_id = %target_module_id,
3071 "route.bind breaker closed: the probe was accepted"
3072 );
3073 }
3074 self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
3075 Ok(Vec::new())
3076 }
3077 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
3078 reservation.release_and_disarm();
3079 breaker.record_inconclusive();
3083 self.counters
3084 .increment_route_open_refused("module_rejected");
3085 info!(
3086 target: "control",
3087 code = "module_rejected",
3088 module_code = ?body.code,
3089 module_id = ?target_module_id,
3090 connection_id = ctx.connection_id.get(),
3091 "route.open refused"
3092 );
3093 Ok(vec![control_error_body_frame(&frame, body)?])
3094 }
3095 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
3096 reservation.release_and_disarm();
3097 breaker.record_inconclusive();
3098 tracing::warn!(
3105 module_id = %target_module_id,
3106 "route.bind relay abandoned: {message}"
3107 );
3108 Ok(vec![self.route_open_refusal_frame(
3109 ctx,
3110 &frame,
3111 &target_module_id,
3112 "relay_abandoned",
3113 "target_unavailable",
3114 message,
3115 )?])
3116 }
3117 Ok(Err(_)) => {
3118 reservation.release_and_disarm();
3119 breaker.record_inconclusive();
3120 Ok(vec![self.route_open_refusal_frame(
3121 ctx,
3122 &frame,
3123 &target_module_id,
3124 "relay_waiter_canceled",
3125 "target_unavailable",
3126 "route.bind relay waiter was canceled before the module responded",
3127 )?])
3128 }
3129 Err(_) => {
3130 reservation.release_and_disarm();
3131 if let Some(opened) = breaker.record_timeout(
3135 self.route_bind_breaker_threshold,
3136 self.route_bind_breaker_cooldown,
3137 ) {
3138 warn!(
3139 module_id = %target_module_id,
3140 consecutive_timeouts = opened.consecutive_timeouts,
3141 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
3142 reopened_after_probe = opened.reopened_after_probe,
3143 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
3144 );
3145 }
3146 tracing::warn!(
3154 module_id = %target_module_id,
3155 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
3156 "route.bind relay timed out: module did not ack within budget"
3157 );
3158 Ok(vec![self.route_open_refusal_frame(
3159 ctx,
3160 &frame,
3161 &target_module_id,
3162 "relay_timed_out",
3163 "module_timeout",
3164 format!(
3165 "module_id '{target_module_id}' did not answer route.bind within {:?}",
3166 route_bind_relay_timeout
3167 ),
3168 )?])
3169 }
3170 }
3171 }
3172
3173 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3174 let response = ClientControlResponse::SupervisorSpawnSnapshot {
3175 snapshot: self.supervisor.spawn_snapshot(),
3176 };
3177 Ok(vec![control_response_body_frame(
3178 &frame,
3179 &response,
3180 "ClientControlResponse::SupervisorSpawnSnapshot",
3181 )?])
3182 }
3183
3184 fn handle_supervisor_spawn_subscribe(
3185 &self,
3186 ctx: &RouteCtx,
3187 frame: Frame,
3188 since: Option<SpawnCursor>,
3189 ) -> Result<Vec<Frame>, RouterError> {
3190 match self.supervisor.subscribe_spawns(
3191 ctx.connection_id,
3192 frame.header.corr,
3193 response_version(&frame),
3194 since,
3195 ctx.egress.clone(),
3196 ) {
3197 Ok(()) => Ok(Vec::new()),
3198 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
3199 Ok(vec![control_error_body_frame(
3200 &frame,
3201 ErrorBody {
3202 code: "spawn_cursor_incarnation_mismatch".to_string(),
3203 message: "spawn cursor belongs to a different daemon incarnation"
3204 .to_string(),
3205 detail: Some(serde_json::json!({
3206 "current_daemon_incarnation": current
3207 })),
3208 },
3209 )?])
3210 }
3211 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
3212 &frame,
3213 ErrorBody {
3214 code: "spawn_cursor_too_old".to_string(),
3215 message: "spawn cursor predates the retained event ring".to_string(),
3216 detail: Some(serde_json::json!({
3217 "oldest_retained_cursor": oldest
3218 })),
3219 },
3220 )?]),
3221 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
3222 0,
3223 frame.header.corr,
3224 format!("failed to open supervisor spawn subscription: {error}"),
3225 )),
3226 }
3227 }
3228
3229 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3230 let generation = self
3231 .registry
3232 .generation()
3233 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3234 let mut modules = Vec::new();
3235 for module in self.supervisor.list() {
3236 let status = module.status_for_control("list").map_err(|err| {
3237 RouterError::backend(
3238 0,
3239 frame.header.corr,
3240 format!("failed to read supervisor status: {err}"),
3241 )
3242 })?;
3243 let (configured, _) = module.configuration().map_err(|err| {
3244 RouterError::backend(
3245 0,
3246 frame.header.corr,
3247 format!("failed to read module configuration: {err}"),
3248 )
3249 })?;
3250 let image = module.running_image_agreement().await;
3252 let pending_reload = Some(reload_verdict(
3253 &configured.program,
3254 status.spawned_from.as_deref(),
3255 image,
3256 ));
3257 modules.push(SupervisorEntry {
3258 module_id: status.module_id,
3259 state: status.state.to_string(),
3260 enabled: status.enabled,
3261 live: status.live,
3262 protocol: status.protocol,
3263 health: status.health.status,
3264 pending_reload,
3265 last_probe_ms: status.health.last_probe_ms,
3266 last_exit_code: status.last_exit.as_ref().and_then(|e| e.code),
3267 last_exit_signal: status.last_exit.as_ref().and_then(|e| e.signal),
3268 last_exit_ms: status.last_exit.as_ref().map(|e| e.at_ms),
3269 last_exit_kind: status.last_exit.as_ref().map(|e| e.kind.into()),
3270 restart_count: Some(status.restart_count),
3271 max_restarts: Some(status.max_restarts),
3272 lifetime_restarts: Some(status.lifetime_restarts),
3273 spawn_generation: Some(status.spawn_generation),
3274 restart_window_secs: Some(status.restart_window.as_secs()),
3275 drain_timeout_ms: Some(status.drain_timeout.as_millis() as u64),
3276 restart_backoff_ms: Some(status.restart_backoff.as_millis() as u64),
3277 restart_max_backoff_ms: Some(status.restart_max_backoff.as_millis() as u64),
3278 });
3279 }
3280 let response = ClientControlResponse::SupervisorList {
3281 generation,
3282 modules,
3283 };
3284 Ok(vec![control_response_body_frame(
3285 &frame,
3286 &response,
3287 "ClientControlResponse::SupervisorList",
3288 )?])
3289 }
3290
3291 fn handle_supervisor_stderr_tail(
3292 &self,
3293 frame: Frame,
3294 module_id: String,
3295 max_lines: Option<u32>,
3296 max_bytes: Option<u32>,
3297 ) -> Result<Vec<Frame>, RouterError> {
3298 let Some(module) = self.supervisor.get(&module_id) else {
3299 return Ok(vec![control_error_frame(
3300 &frame,
3301 "unknown_module",
3302 format!("module_id '{module_id}' is not supervised"),
3303 )?]);
3304 };
3305
3306 let snapshot = module.stderr_tail(
3307 max_lines.map(|value| value as usize),
3308 max_bytes.map(|value| value as usize),
3309 );
3310
3311 let response = ClientControlResponse::SupervisorStderrTail {
3312 module_id,
3313 tail: StderrTail {
3314 capture: match snapshot.capture {
3315 CaptureState::Captured => StderrCaptureState::Captured,
3316 CaptureState::Incomplete { reason } => {
3317 StderrCaptureState::Incomplete { reason }
3318 }
3319 CaptureState::NotCaptured { reason } => {
3320 StderrCaptureState::NotCaptured { reason }
3321 }
3322 },
3323 entries: snapshot
3324 .entries
3325 .into_iter()
3326 .map(|entry| match entry {
3327 TailEntry::Line { text, truncated } => {
3328 StderrTailEntry::Line { text, truncated }
3329 }
3330 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
3331 })
3332 .collect(),
3333 dropped_lines: snapshot.dropped_lines,
3334 },
3335 };
3336 Ok(vec![control_response_body_frame(
3337 &frame,
3338 &response,
3339 "ClientControlResponse::SupervisorStderrTail",
3340 )?])
3341 }
3342
3343 async fn handle_supervisor_terminals(
3344 &self,
3345 frame: Frame,
3346 module_id: String,
3347 ) -> Result<Vec<Frame>, RouterError> {
3348 let Some(module) = self.supervisor.get(&module_id) else {
3349 return Ok(vec![control_error_frame(
3350 &frame,
3351 "unknown_module",
3352 format!("module_id '{module_id}' is not supervised"),
3353 )?]);
3354 };
3355
3356 let terminals = module
3359 .read_durable_terminal_history()
3360 .await
3361 .map_err(|error| {
3362 RouterError::backend(
3363 0,
3364 frame.header.corr,
3365 format!("failed to read terminal history: {error}"),
3366 )
3367 })?;
3368 let response = ClientControlResponse::SupervisorTerminals {
3369 module_id,
3370 terminals,
3371 };
3372 Ok(vec![control_response_body_frame(
3373 &frame,
3374 &response,
3375 "ClientControlResponse::SupervisorTerminals",
3376 )?])
3377 }
3378
3379 fn handle_supervisor_routes(
3380 &self,
3381 frame: Frame,
3382 module_id: Option<String>,
3383 ) -> Result<Vec<Frame>, RouterError> {
3384 let modules = self
3385 .forwarding
3386 .route_census(module_id.as_deref())
3387 .map_err(RouterError::Forwarding)?
3388 .into_iter()
3389 .map(|(module_id, routes)| SupervisorRouteModule {
3390 module_id,
3391 routes: routes
3392 .into_iter()
3393 .map(|route| SupervisorRoute {
3394 consumer: match route.principal {
3395 Principal::Reserved { module_id } => {
3396 SupervisorRouteConsumer::Reserved { module_id }
3397 }
3398 Principal::Direct | Principal::Unverified => {
3399 SupervisorRouteConsumer::Direct {
3400 connection_id: route.goodbye_target.connection_id.get(),
3401 }
3402 }
3403 },
3404 age_ms: Instant::now()
3405 .saturating_duration_since(route.bound_at)
3406 .as_millis()
3407 .try_into()
3408 .unwrap_or(u64::MAX),
3409 draining: route.draining,
3410 drain_reason: route.drain_reason,
3411 })
3412 .collect(),
3413 })
3414 .collect();
3415 let response = ClientControlResponse::SupervisorRoutes { modules };
3416 Ok(vec![control_response_body_frame(
3417 &frame,
3418 &response,
3419 "ClientControlResponse::SupervisorRoutes",
3420 )?])
3421 }
3422
3423 async fn handle_supervisor_provenance(
3424 &self,
3425 frame: Frame,
3426 module_id: Option<String>,
3427 ) -> Result<Vec<Frame>, RouterError> {
3428 let mut selected = if let Some(module_id) = module_id {
3429 let Some(module) = self.supervisor.get(&module_id) else {
3430 return Ok(vec![control_error_frame(
3431 &frame,
3432 "unknown_module",
3433 format!("module_id '{module_id}' is not supervised"),
3434 )?]);
3435 };
3436 vec![module]
3437 } else {
3438 self.supervisor.list()
3439 };
3440
3441 let mut modules = Vec::with_capacity(selected.len());
3442 for module in selected.drain(..) {
3443 let status = module.status().map_err(|err| {
3444 RouterError::backend(
3445 0,
3446 frame.header.corr,
3447 format!("failed to read supervisor status: {err}"),
3448 )
3449 })?;
3450 let module_declared = self
3451 .registry
3452 .get_module(&status.module_id)
3453 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3454 .and_then(|registration| registration.manifest.provenance)
3455 .map(|build| ModuleDeclaredProvenance::Reported { build })
3456 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
3457 #[cfg(test)]
3458 let running_image = match &self.provenance_probe_override {
3459 Some(result) => result.clone(),
3460 None => module.running_image_agreement().await,
3461 };
3462 #[cfg(not(test))]
3463 let running_image = module.running_image_agreement().await;
3464 modules.push(SupervisorModuleProvenance {
3465 module_id: status.module_id,
3466 module_declared,
3467 daemon_observed: SupervisorObservedProcess {
3468 pid: status.pid,
3469 spawned_at_ms: status.spawned_at_ms,
3470 spawned_from: status.spawned_from,
3471 running_image,
3472 },
3473 });
3474 }
3475 let daemon = SupervisorDaemonProvenance {
3476 daemon_build: self.daemon_provenance.build.clone(),
3477 daemon_observed: DaemonObservedProcess {
3478 pid: self.daemon_provenance.pid,
3479 started_at_ms: self
3480 .daemon_provenance
3481 .start_clock
3482 .map(|clock| clock.started_at_ms())
3483 .or(self.daemon_provenance.started_at_ms),
3484 running_image: self
3485 .daemon_provenance
3486 .probe
3487 .observe(
3488 self.daemon_provenance.pid,
3489 self.daemon_provenance.executable_path.as_deref(),
3490 self.daemon_provenance.executable_identity,
3491 self.daemon_provenance.process_start_time,
3492 )
3493 .await,
3494 },
3495 };
3496 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
3497 Ok(vec![control_response_body_frame(
3498 &frame,
3499 &response,
3500 "ClientControlResponse::SupervisorProvenance",
3501 )?])
3502 }
3503
3504 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3505 self.refresh_capability_requirements();
3506 let generation = self
3507 .registry
3508 .generation()
3509 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3510 let modules = self
3511 .supervisor
3512 .list()
3513 .into_iter()
3514 .map(|module| {
3515 let status = module.status_for_control("health").map_err(|err| {
3516 RouterError::backend(
3517 0,
3518 frame.header.corr,
3519 format!("failed to read supervisor health: {err}"),
3520 )
3521 })?;
3522 let module_id = status.module_id;
3523 let capability_detail = self
3524 .capability_evaluator
3525 .required_problem_detail(&module_id);
3526 Ok(SupervisorHealthEntry {
3527 module_id,
3528 status: status.health.status,
3529 detail: append_capability_problem_detail(
3530 status.health.detail,
3531 capability_detail,
3532 ),
3533 metrics: status.health.metrics,
3534 consecutive_failures: status.health.consecutive_failures,
3535 late_answer_count: status.health.late_answer_count,
3536 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
3537 last_action: status.health.last_action,
3538 last_action_ms: status.health.last_action_ms,
3539 last_probe_ms: status.health.last_probe_ms,
3540 })
3541 })
3542 .collect::<Result<Vec<_>, RouterError>>()?;
3543 let response = ClientControlResponse::SupervisorHealth {
3544 generation,
3545 modules,
3546 };
3547 Ok(vec![control_response_body_frame(
3548 &frame,
3549 &response,
3550 "ClientControlResponse::SupervisorHealth",
3551 )?])
3552 }
3553
3554 async fn handle_supervisor_restart(
3555 &self,
3556 frame: Frame,
3557 module_id: String,
3558 drain_timeout_ms: Option<u64>,
3559 ) -> Result<Vec<Frame>, RouterError> {
3560 let operation_lock = self.supervisor.operation_lock();
3561 let _operation_guard = operation_lock.lock().await;
3562 let Some(module) = self.supervisor.get(&module_id) else {
3563 return Ok(vec![control_error_frame(
3564 &frame,
3565 "unknown_module",
3566 format!("module_id '{module_id}' is not supervised"),
3567 )?]);
3568 };
3569
3570 if let Err(err) = module.restart(drain_timeout_ms).await {
3571 let (code, message) = match err {
3572 crate::supervise::SuperviseError::Disabled { .. } => {
3573 ("module_disabled", err.to_string())
3574 }
3575 crate::supervise::SuperviseError::SwapInProgress { .. } => {
3576 ("swap_in_progress", err.to_string())
3577 }
3578 _ => (
3579 "target_unavailable",
3580 format!("failed to restart module_id '{module_id}': {err}"),
3581 ),
3582 };
3583 return Ok(vec![control_error_frame(&frame, code, message)?]);
3584 }
3585
3586 let response = ClientControlResponse::SupervisorAck {
3587 module_id,
3588 applied: true,
3589 };
3590 Ok(vec![control_response_body_frame(
3591 &frame,
3592 &response,
3593 "ClientControlResponse::SupervisorAck",
3594 )?])
3595 }
3596
3597 async fn handle_supervisor_swap(
3601 &self,
3602 frame: Frame,
3603 module_id: String,
3604 ready_timeout_ms: Option<u64>,
3605 ) -> Result<Vec<Frame>, RouterError> {
3606 let module = {
3613 let operation_lock = self.supervisor.operation_lock();
3614 let _operation_guard = operation_lock.lock().await;
3615 self.supervisor.get(&module_id)
3616 };
3617 let Some(module) = module else {
3618 return Ok(vec![control_error_frame(
3619 &frame,
3620 "unknown_module",
3621 format!("module_id '{module_id}' is not supervised"),
3622 )?]);
3623 };
3624
3625 if let Err(err) = module
3626 .swap(ready_timeout_ms.map(Duration::from_millis))
3627 .await
3628 {
3629 use crate::supervise::SuperviseError;
3630 let message = err.to_string();
3631 let error = match err {
3632 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
3633 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
3634 code: "swap_refused".to_string(),
3635 message,
3636 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
3637 },
3638 SuperviseError::SwapFailed {
3639 arm,
3640 candidate_exit,
3641 ..
3642 } => ErrorBody {
3643 code: "swap_failed".to_string(),
3644 message,
3645 detail: Some(serde_json::json!({
3646 "arm": arm.as_str(),
3647 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
3648 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
3649 })),
3650 },
3651 _ => ErrorBody::new(
3652 "target_unavailable",
3653 format!("failed to swap module_id '{module_id}': {message}"),
3654 ),
3655 };
3656 return Ok(vec![control_error_body_frame(&frame, error)?]);
3657 }
3658
3659 let response = ClientControlResponse::SupervisorAck {
3660 module_id,
3661 applied: true,
3662 };
3663 Ok(vec![control_response_body_frame(
3664 &frame,
3665 &response,
3666 "ClientControlResponse::SupervisorAck",
3667 )?])
3668 }
3669
3670 async fn handle_supervisor_reload(
3671 &self,
3672 frame: Frame,
3673 module_id: String,
3674 ) -> Result<Vec<Frame>, RouterError> {
3675 let operation_lock = self.supervisor.operation_lock();
3676 let _operation_guard = operation_lock.lock().await;
3677 let Some(module) = self.supervisor.get(&module_id) else {
3678 return Ok(vec![control_error_frame(
3679 &frame,
3680 "unknown_module",
3681 format!("module_id '{module_id}' is not supervised"),
3682 )?]);
3683 };
3684
3685 if let Err(err) = module.reload().await {
3686 let (code, message) = match err {
3687 crate::supervise::SuperviseError::Disabled { .. } => {
3688 ("module_disabled", err.to_string())
3689 }
3690 crate::supervise::SuperviseError::SwapInProgress { .. } => {
3691 ("swap_in_progress", err.to_string())
3692 }
3693 _ => (
3694 "reload_failed",
3695 format!("failed to reload module_id '{module_id}': {err}"),
3696 ),
3697 };
3698 return Ok(vec![control_error_frame(&frame, code, message)?]);
3699 }
3700
3701 let response = ClientControlResponse::SupervisorAck {
3702 module_id,
3703 applied: true,
3704 };
3705 Ok(vec![control_response_body_frame(
3706 &frame,
3707 &response,
3708 "ClientControlResponse::SupervisorAck",
3709 )?])
3710 }
3711
3712 async fn handle_supervisor_rescan(
3713 &self,
3714 frame: Frame,
3715 preview: bool,
3716 ) -> Result<Vec<Frame>, RouterError> {
3717 let Some(context) = self.rescan.clone() else {
3718 return Ok(vec![control_error_frame(
3719 &frame,
3720 "rescan_unavailable",
3721 "the daemon was not started with a reloadable config path".to_string(),
3722 )?]);
3723 };
3724
3725 let operation_lock = self.supervisor.operation_lock();
3726 let _operation_guard = operation_lock.lock().await;
3727 let loaded = match crate::daemon_config::load(&context.config_path) {
3728 Ok(config) => config,
3729 Err(err) => {
3730 return Ok(vec![control_error_frame(
3731 &frame,
3732 "invalid_daemon_config",
3733 format!("supervisor rescan rejected daemon config: {err}"),
3734 )?])
3735 }
3736 };
3737 let Some(config) = loaded else {
3745 return Ok(vec![control_error_frame(
3746 &frame,
3747 "invalid_daemon_config",
3748 format!(
3749 "daemon config not found at {}; refusing to rescan (an absent config would \
3750 retire every supervised module)",
3751 context.config_path.display()
3752 ),
3753 )?]);
3754 };
3755 let (
3756 configured_port,
3757 storage_config,
3758 admission_facts_carrier_module_id,
3759 admission_facts_targets,
3760 modules,
3761 reserved_capabilities,
3762 ) = (
3763 config.port,
3764 config.storage,
3765 config.admission_facts_carrier_module_id,
3766 config.admission_facts_targets,
3767 config.modules,
3768 config.reserved_capabilities,
3769 );
3770
3771 let mut restart_required = Vec::new();
3780 for section in RestartRequiredSection::ALL {
3781 let changed = match section {
3782 RestartRequiredSection::Port => configured_port != context.configured_port,
3783 RestartRequiredSection::Storage => storage_config != context.storage_config,
3784 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
3785 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
3786 }
3787 RestartRequiredSection::AdmissionFactsTargets => {
3788 admission_facts_targets != context.admission_facts_targets
3789 }
3790 };
3791 if changed {
3792 restart_required.push(section.label().to_string());
3793 }
3794 }
3795 if !restart_required.is_empty() {
3796 warn!(
3797 config_path = %context.config_path.display(),
3798 sections = %restart_required.join(", "),
3799 "daemon config changed outside the modules section; restart the daemon to apply those changes"
3800 );
3801 }
3802
3803 for configured in &modules {
3804 if let Err(err) = validate_spec(&configured.module_spec()) {
3805 return Ok(vec![control_error_frame(
3806 &frame,
3807 "invalid_daemon_config",
3808 format!("supervisor rescan rejected daemon config: {err}"),
3809 )?]);
3810 }
3811 }
3812
3813 let configured_capabilities = modules
3814 .iter()
3815 .map(|module| (module.module_id.clone(), module.enabled))
3816 .collect::<Vec<_>>();
3817 let preview_capability_warnings = if preview {
3818 let (_, registrations) = self.runtime_capability_snapshot()?;
3819 let current_modules = self
3820 .supervisor
3821 .list()
3822 .into_iter()
3823 .map(|module| module.module_id().to_string())
3824 .collect::<BTreeSet<_>>();
3825 let resulting_modules = configured_capabilities.clone();
3826 let removed = current_modules
3827 .into_iter()
3828 .filter(|module_id| {
3829 !resulting_modules
3830 .iter()
3831 .any(|(configured_id, _)| configured_id == module_id)
3832 })
3833 .collect::<Vec<_>>();
3834 self.capability_evaluator.preview_removal_warnings(
3835 resulting_modules,
3836 &removed,
3837 ®istrations,
3838 )
3839 } else {
3840 Vec::new()
3841 };
3842 let result = match self
3843 .reconcile_supervised_modules(&context.supervisor, modules, preview)
3844 .await
3845 {
3846 Ok(result) => result,
3847 Err(message) => {
3848 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
3849 }
3850 };
3851 if !preview {
3852 self.capability_evaluator
3853 .configure(configured_capabilities, reserved_capabilities);
3854 self.capability_evaluator.wake_deadline_loop();
3855 self.refresh_capability_requirements();
3856 }
3857 let mut result = result;
3858 result.restart_required = restart_required;
3859 result.capability_warnings = preview_capability_warnings;
3860 let response = ClientControlResponse::SupervisorRescan { result };
3861 Ok(vec![control_response_body_frame(
3862 &frame,
3863 &response,
3864 "ClientControlResponse::SupervisorRescan",
3865 )?])
3866 }
3867
3868 async fn handle_supervisor_release_reserved(
3869 &self,
3870 frame: Frame,
3871 module_id: String,
3872 ) -> Result<Vec<Frame>, RouterError> {
3873 let Some(context) = self.rescan.clone() else {
3874 return Ok(vec![control_error_frame(
3875 &frame,
3876 "release_unavailable",
3877 "reserved-id release requires a daemon started with a reloadable config path",
3878 )?]);
3879 };
3880 let operation_lock = self.supervisor.operation_lock();
3881 let _operation_guard = operation_lock.lock().await;
3882 let loaded = match crate::daemon_config::load(&context.config_path) {
3883 Ok(Some(config)) => config,
3884 Ok(None) => {
3885 return Ok(vec![control_error_frame(
3886 &frame,
3887 "invalid_daemon_config",
3888 format!(
3889 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
3890 context.config_path.display()
3891 ),
3892 )?])
3893 }
3894 Err(err) => {
3895 return Ok(vec![control_error_frame(
3896 &frame,
3897 "invalid_daemon_config",
3898 format!("unable to verify reserved-id release against daemon config: {err}"),
3899 )?])
3900 }
3901 };
3902 if loaded
3903 .modules
3904 .iter()
3905 .any(|configured| configured.module_id == module_id)
3906 {
3907 return Ok(vec![control_error_frame(
3908 &frame,
3909 "reserved_module_configured",
3910 format!(
3911 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
3912 ),
3913 )?]);
3914 }
3915 if !self.supervisor.release_retained_reserved_gate(&module_id) {
3916 return Ok(vec![control_error_frame(
3917 &frame,
3918 "reserved_gate_not_retained",
3919 format!(
3920 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
3921 ),
3922 )?]);
3923 }
3924
3925 let response = ClientControlResponse::SupervisorAck {
3926 module_id,
3927 applied: true,
3928 };
3929 Ok(vec![control_response_body_frame(
3930 &frame,
3931 &response,
3932 "ClientControlResponse::SupervisorAck",
3933 )?])
3934 }
3935
3936 async fn reconcile_supervised_modules(
3945 &self,
3946 supervisor: &Supervisor,
3947 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
3948 preview: bool,
3949 ) -> Result<SupervisorRescanResult, String> {
3950 let mut current = BTreeMap::new();
3951 for module in self.supervisor.list() {
3952 let (spec, health) = module.configuration().map_err(|err| {
3953 format!(
3954 "failed to read configuration for module_id '{}': {err}",
3955 module.module_id()
3956 )
3957 })?;
3958 let enabled = module
3959 .status()
3960 .map_err(|err| {
3961 format!(
3962 "failed to read status for module_id '{}': {err}",
3963 module.module_id()
3964 )
3965 })?
3966 .enabled;
3967 current.insert(
3968 module.module_id().to_string(),
3969 (module, spec, health, enabled),
3970 );
3971 }
3972 let configured = configured_modules
3973 .into_iter()
3974 .map(|module| (module.module_id.clone(), module))
3975 .collect::<BTreeMap<_, _>>();
3976
3977 let added = configured
3978 .keys()
3979 .filter(|module_id| !current.contains_key(*module_id))
3980 .cloned()
3981 .collect::<Vec<_>>();
3982 let removed = current
3983 .keys()
3984 .filter(|module_id| !configured.contains_key(*module_id))
3985 .cloned()
3986 .collect::<Vec<_>>();
3987 let mut changed_pending_reload = Vec::new();
3988 let mut configuration_changes = BTreeSet::new();
3989 let mut enabled_changes = BTreeSet::new();
3990 let mut unchanged = 0_u32;
3991
3992 for (module_id, configured_module) in &configured {
3993 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
3994 else {
3995 continue;
3996 };
3997 let configuration_changed = *current_spec != configured_module.module_spec()
3998 || *current_health != configured_module.health;
3999 let enabled_changed = *current_enabled != configured_module.enabled;
4000 if configuration_changed {
4001 configuration_changes.insert(module_id.clone());
4002 changed_pending_reload.push(module_id.clone());
4003 }
4004 if enabled_changed {
4005 enabled_changes.insert(module_id.clone());
4006 }
4007 if !configuration_changed && !enabled_changed {
4008 unchanged = unchanged.saturating_add(1);
4009 }
4010 }
4011
4012 if preview {
4018 return Ok(SupervisorRescanResult {
4019 added,
4020 removed,
4021 changed_pending_reload,
4022 enabled_changes: enabled_changes.iter().cloned().collect(),
4023 unchanged,
4024 preview: true,
4025 restart_required: Vec::new(),
4029 capability_warnings: Vec::new(),
4030 });
4031 }
4032
4033 for module_id in &removed {
4034 let module = ¤t
4035 .get(module_id)
4036 .expect("removed module came from current supervisor state")
4037 .0;
4038 module.retire().await.map_err(|err| {
4039 format!("failed to retire module_id '{module_id}' during rescan: {err}")
4040 })?;
4041 self.supervisor.record_rescan_removal(module_id);
4075 self.supervisor.retire(module_id);
4076 }
4077
4078 for module_id in configured.keys() {
4079 let Some((module, _, _, _)) = current.get(module_id) else {
4080 continue;
4081 };
4082 let configured_module = configured
4083 .get(module_id)
4084 .expect("configured module id came from configured map");
4085 if configuration_changes.contains(module_id) {
4086 module
4087 .update_configuration(
4088 configured_module.module_spec(),
4089 configured_module.health,
4090 configured_module.drain_timeout_ms,
4091 )
4092 .await
4093 .map_err(|err| {
4094 format!(
4095 "failed to update module_id '{module_id}' configuration during rescan: {err}"
4096 )
4097 })?;
4098 }
4099 if enabled_changes.contains(module_id) {
4100 module
4101 .set_enabled(configured_module.enabled)
4102 .await
4103 .map_err(|err| {
4104 format!(
4105 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
4106 configured_module.enabled
4107 )
4108 })?;
4109 }
4110 }
4111
4112 for module_id in &added {
4113 let configured_module = configured
4114 .get(module_id)
4115 .expect("added module id came from configured map");
4116 supervisor
4117 .supervise_configured_with_health(
4118 configured_module.module_spec(),
4119 configured_module.enabled,
4120 configured_module.health,
4121 configured_module.drain_timeout_ms,
4122 configured_module.restart,
4123 )
4124 .map_err(|err| {
4125 format!("failed to add module_id '{module_id}' during rescan: {err}")
4126 })?;
4127 }
4128
4129 Ok(SupervisorRescanResult {
4130 added,
4131 removed,
4132 changed_pending_reload,
4133 enabled_changes: enabled_changes.iter().cloned().collect(),
4134 unchanged,
4135 preview: false,
4136 restart_required: Vec::new(),
4139 capability_warnings: Vec::new(),
4140 })
4141 }
4142
4143 async fn handle_supervisor_set_enabled(
4144 &self,
4145 frame: Frame,
4146 module_id: String,
4147 enabled: bool,
4148 ) -> Result<Vec<Frame>, RouterError> {
4149 let operation_lock = self.supervisor.operation_lock();
4150 let _operation_guard = operation_lock.lock().await;
4151 let Some(module) = self.supervisor.get(&module_id) else {
4152 return Ok(vec![control_error_frame(
4153 &frame,
4154 "unknown_module",
4155 format!("module_id '{module_id}' is not supervised"),
4156 )?]);
4157 };
4158
4159 let applied = match module.set_enabled(enabled).await {
4160 Ok(applied) => applied,
4161 Err(err) => {
4162 return Ok(vec![control_error_frame(
4163 &frame,
4164 "target_unavailable",
4165 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
4166 )?])
4167 }
4168 };
4169
4170 self.capability_evaluator.wake_deadline_loop();
4171 self.refresh_capability_requirements();
4172 let response = ClientControlResponse::SupervisorAck { module_id, applied };
4173 Ok(vec![control_response_body_frame(
4174 &frame,
4175 &response,
4176 "ClientControlResponse::SupervisorAck",
4177 )?])
4178 }
4179
4180 async fn handle_supervisor_health_probe(
4181 &self,
4182 frame: Frame,
4183 module_id: String,
4184 ) -> Result<Vec<Frame>, RouterError> {
4185 self.refresh_capability_requirements();
4186 let Some(registration) = self
4187 .registry
4188 .get_module(&module_id)
4189 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4190 else {
4191 return Ok(vec![control_error_frame(
4192 &frame,
4193 "unknown_module",
4194 format!("module_id '{module_id}' is not registered"),
4195 )?]);
4196 };
4197
4198 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
4211 {
4212 return Ok(vec![control_error_frame(
4213 &frame,
4214 "health_not_advertised",
4215 format!("module_id '{module_id}' did not advertise health.check"),
4216 )?]);
4217 }
4218
4219 let deadline = Instant::now() + self.health_probe_timeout;
4220 let pending = match self.forwarding.begin_module_control_rpc_for(
4221 &module_id,
4222 MODULE_CONTROL_OP_HEALTH_CHECK,
4223 deadline,
4224 ) {
4225 Ok(pending) => pending,
4226 Err(err) => {
4227 return Ok(vec![control_error_frame(
4228 &frame,
4229 forwarding_error_code(&err),
4230 err.to_string(),
4231 )?])
4232 }
4233 };
4234
4235 let PendingModuleControlRpc {
4236 endpoint,
4237 module_sink,
4238 negotiated_ver,
4239 corr: probe_corr,
4240 receiver,
4241 } = pending;
4242 let mut guard =
4243 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
4244 let probe_body =
4245 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
4246 RouterError::backend(
4247 0,
4248 frame.header.corr,
4249 format!("failed to encode health.check request: {err}"),
4250 )
4251 })?;
4252 let probe_frame = Frame::build_with_version(
4253 negotiated_ver,
4254 FrameType::Request,
4255 control_flags(),
4256 0,
4257 0,
4258 probe_corr,
4259 probe_body,
4260 )
4261 .map_err(RouterError::FrameBuild)?;
4262
4263 if let Err(err) = module_sink.send(probe_frame).await {
4264 return Ok(vec![control_error_frame(
4265 &frame,
4266 "target_unavailable",
4267 err.to_string(),
4268 )?]);
4269 }
4270
4271 match timeout_at(deadline, receiver).await {
4272 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
4273 guard.disarm();
4274 let Some(report) = response.health_report() else {
4275 return Ok(vec![control_error_frame(
4276 &frame,
4277 "invalid_control_body",
4278 "health.check RPC returned a non-health response",
4279 )?]);
4280 };
4281 let HealthReport {
4286 status,
4287 detail,
4288 metrics,
4289 } = report;
4290 let capability_detail = self
4291 .capability_evaluator
4292 .required_problem_detail(&module_id);
4293 let response = ClientControlResponse::SupervisorHealthProbe {
4294 module_id,
4295 status,
4296 detail: append_capability_problem_detail(detail, capability_detail),
4297 metrics,
4298 };
4299 Ok(vec![control_response_body_frame(
4300 &frame,
4301 &response,
4302 "ClientControlResponse::SupervisorHealthProbe",
4303 )?])
4304 }
4305 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
4306 guard.disarm();
4307 Ok(vec![control_error_body_frame(&frame, body)?])
4308 }
4309 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
4310 guard.disarm();
4311 Ok(vec![control_error_frame(
4312 &frame,
4313 "target_unavailable",
4314 message,
4315 )?])
4316 }
4317 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
4318 guard.disarm();
4319 Ok(vec![control_error_frame(
4320 &frame,
4321 "invalid_control_body",
4322 message,
4323 )?])
4324 }
4325 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
4326 guard.disarm();
4327 Ok(vec![control_error_frame(
4328 &frame,
4329 "invalid_control_body",
4330 format!("expected module-control op '{expected}', got '{actual}'"),
4331 )?])
4332 }
4333 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
4334 guard.disarm();
4335 Ok(vec![control_error_frame(
4336 &frame,
4337 "module_timeout",
4338 format!(
4339 "module_id '{module_id}' answered health.check after {:?}",
4340 self.health_probe_timeout
4341 ),
4342 )?])
4343 }
4344 Ok(Err(_)) => Ok(vec![control_error_frame(
4345 &frame,
4346 "target_unavailable",
4347 "health.check waiter was canceled before the module responded",
4348 )?]),
4349 Err(_) => Ok(vec![control_error_frame(
4350 &frame,
4351 "module_timeout",
4352 format!(
4353 "module_id '{module_id}' did not answer health.check within {:?}",
4354 self.health_probe_timeout
4355 ),
4356 )?]),
4357 }
4358 }
4359
4360 fn supervisor_status(
4361 &self,
4362 module_id: &str,
4363 corr: u64,
4364 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
4365 self.supervisor
4366 .get(module_id)
4367 .map(|module| {
4368 let warming = module.is_warming_for_control("status").map_err(|err| {
4369 RouterError::backend(
4370 0,
4371 corr,
4372 format!(
4373 "failed to read supervisor warming state for module_id '{module_id}': {err}"
4374 ),
4375 )
4376 })?;
4377 module.status_for_control("status").map_err(|err| {
4378 RouterError::backend(
4379 0,
4380 corr,
4381 format!(
4382 "failed to read supervisor status for module_id '{module_id}': {err}"
4383 ),
4384 )
4385 }).map(|status| (status, warming))
4386 })
4387 .transpose()
4388 }
4389
4390 fn guard_module_control_op(
4391 &self,
4392 frame: &Frame,
4393 module_id: &str,
4394 op: &str,
4395 ) -> Result<Option<Frame>, RouterError> {
4396 if self.module_grants_op(module_id, op, frame.header.corr)? {
4397 return Ok(None);
4398 }
4399
4400 Ok(Some(control_error_frame(
4401 frame,
4402 "op_not_allowed",
4403 format!("module_id '{module_id}' did not grant control op '{op}'"),
4404 )?))
4405 }
4406
4407 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
4408 let Some(registration) = self
4409 .registry
4410 .get_module(module_id)
4411 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
4412 else {
4413 return Ok(false);
4414 };
4415 Ok(module_registration_grants_op(®istration.control_ops, op))
4416 }
4417
4418 fn handle_status_update(
4419 &self,
4420 endpoint: ModuleEndpointId,
4421 frame: Frame,
4422 ) -> Result<Vec<Frame>, RouterError> {
4423 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
4424 Ok(update) => update,
4425 Err(err) => {
4426 if is_known_module_push_op(&frame.body) {
4431 return Ok(vec![control_error_frame(
4432 &frame,
4433 "invalid_control_body",
4434 format!("malformed module control push body: {err}"),
4435 )?]);
4436 }
4437 return Ok(Vec::new());
4438 }
4439 };
4440
4441 match update {
4442 ModuleControlPush::RouteStatus {
4443 route_channel,
4444 route_epoch,
4445 status,
4446 } => {
4447 self.forwarding
4448 .cache_status(endpoint, route_channel, route_epoch, status)
4449 .map_err(RouterError::Forwarding)?;
4450 }
4451 }
4452 Ok(Vec::new())
4453 }
4454
4455 fn handle_route_poll(
4456 &self,
4457 ctx: &RouteCtx,
4458 frame: Frame,
4459 route_channel: u16,
4460 route_epoch: u32,
4461 kind: PollKind,
4462 ) -> Result<Vec<Frame>, RouterError> {
4463 let snapshot = self
4464 .forwarding
4465 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
4466 .map_err(RouterError::Forwarding)?;
4467 let response = match (kind, snapshot) {
4468 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
4469 ClientControlResponse::RoutePoll {
4470 route_channel,
4471 route_epoch,
4472 status,
4473 live: None,
4474 }
4475 }
4476 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
4477 route_channel,
4478 route_epoch,
4479 status: None,
4480 live: None,
4481 },
4482 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
4483 let live = self
4509 .process_liveness
4510 .as_ref()
4511 .and_then(|source| source.process_live(&module_id))
4512 .unwrap_or(true);
4513 ClientControlResponse::RoutePoll {
4514 route_channel,
4515 route_epoch,
4516 status: None,
4517 live: Some(live),
4518 }
4519 }
4520 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
4521 route_channel,
4522 route_epoch,
4523 status: None,
4524 live: Some(false),
4525 },
4526 };
4527
4528 Ok(vec![control_response_body_frame(
4529 &frame,
4530 &response,
4531 "ClientControlResponse::RoutePoll",
4532 )?])
4533 }
4534
4535 pub(crate) fn observe_module_control_completion(
4536 &self,
4537 completion: ModuleControlRpcCompletion,
4538 ) -> bool {
4539 match completion {
4540 ModuleControlRpcCompletion::Unknown => false,
4541 ModuleControlRpcCompletion::Settled => true,
4542 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
4543 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
4544 info!(
4545 module_id = %module_id,
4546 latency_ms,
4547 "late health.check answer proves the module is alive"
4548 );
4549 match self
4550 .supervisor
4551 .record_late_health_answer(&module_id, latency_ms)
4552 {
4553 Ok(true) => {}
4554 Ok(false) => debug!(
4555 module_id = %module_id,
4556 latency_ms,
4557 "late health.check answer has no active supervisor snapshot"
4558 ),
4559 Err(err) => warn!(
4560 module_id = %module_id,
4561 latency_ms,
4562 error = %err,
4563 "failed to record late health.check answer"
4564 ),
4565 }
4566 true
4567 }
4568 }
4569 }
4570
4571 fn refuse_to_end_module_connection_for_a_client(
4592 &self,
4593 module_connection_id: ConnectionId,
4594 corr: u64,
4595 err: ForwardingError,
4596 ) -> Result<(), RouterError> {
4597 if let ForwardingError::ConnectionClosing { connection_id } = err {
4598 if connection_id != module_connection_id {
4599 warn!(
4600 module_connection_id = module_connection_id.get(),
4601 client_connection_id = connection_id.get(),
4602 corr,
4603 "dropping a route.bind response for a closing client; the module connection keeps serving"
4604 );
4605 return Ok(());
4606 }
4607 }
4608 Err(RouterError::Forwarding(err))
4609 }
4610
4611 fn handle_module_relay_response(
4612 &self,
4613 connection_id: ConnectionId,
4614 frame: Frame,
4615 ) -> Result<Vec<Frame>, RouterError> {
4616 let mut secondary_error = None;
4617 let outcome = match frame.header.ty {
4618 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
4619 Ok(probe) if probe.op == "route.bind" => {
4620 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
4621 Ok(ModuleControlResponse::RouteBindAck {}) => {
4622 RouteBindRelayOutcome::Accepted
4623 }
4624 Ok(other) => {
4625 let message =
4626 format!("route.bind response carried unexpected body: {other:?}");
4627 secondary_error = Some(control_error_frame(
4628 &frame,
4629 "invalid_control_body",
4630 message.clone(),
4631 )?);
4632 RouteBindRelayOutcome::ModuleGone(message)
4633 }
4634 Err(err) => {
4635 let message = format!("malformed route.bind response body: {err}");
4636 secondary_error = Some(control_error_frame(
4637 &frame,
4638 "invalid_control_body",
4639 message.clone(),
4640 )?);
4641 RouteBindRelayOutcome::ModuleGone(message)
4642 }
4643 }
4644 }
4645 Ok(probe) => {
4646 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
4647 {
4648 Ok(response) => ModuleControlRpcOutcome::Response(response),
4649 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
4650 "malformed {} response body: {err}",
4651 probe.op
4652 )),
4653 };
4654 let completion = self
4655 .forwarding
4656 .complete_module_control_rpc(
4657 connection_id,
4658 frame.header.corr,
4659 Some(&probe.op),
4660 outcome,
4661 )
4662 .map_err(RouterError::Forwarding)?;
4663 if !self.observe_module_control_completion(completion) {
4664 debug!(
4665 connection_id = connection_id.get(),
4666 corr = frame.header.corr,
4667 op = %probe.op,
4668 "dropping late or unknown module-control RPC response"
4669 );
4670 }
4671 return Ok(Vec::new());
4672 }
4673 Err(err) => {
4674 if let Some(expected_op) = self
4675 .forwarding
4676 .pending_module_control_op(connection_id, frame.header.corr)
4677 .map_err(RouterError::Forwarding)?
4678 {
4679 let completion = self
4680 .forwarding
4681 .complete_module_control_rpc(
4682 connection_id,
4683 frame.header.corr,
4684 None,
4685 ModuleControlRpcOutcome::MalformedResponse(format!(
4686 "malformed {expected_op} response body: {err}"
4687 )),
4688 )
4689 .map_err(RouterError::Forwarding)?;
4690 if !self.observe_module_control_completion(completion) {
4691 debug!(
4692 connection_id = connection_id.get(),
4693 corr = frame.header.corr,
4694 "dropping late malformed module-control RPC response"
4695 );
4696 }
4697 return Ok(Vec::new());
4698 }
4699 let message = format!("malformed route.bind response body: {err}");
4700 secondary_error = Some(control_error_frame(
4701 &frame,
4702 "invalid_control_body",
4703 message.clone(),
4704 )?);
4705 RouteBindRelayOutcome::ModuleGone(message)
4706 }
4707 },
4708 FrameType::Error => {
4709 if self
4710 .forwarding
4711 .pending_module_control_op(connection_id, frame.header.corr)
4712 .map_err(RouterError::Forwarding)?
4713 .is_some()
4714 {
4715 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
4716 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
4717 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
4718 "malformed module-control ERROR body: {err}"
4719 )),
4720 };
4721 let completion = self
4722 .forwarding
4723 .complete_module_control_rpc(
4724 connection_id,
4725 frame.header.corr,
4726 None,
4727 outcome,
4728 )
4729 .map_err(RouterError::Forwarding)?;
4730 if !self.observe_module_control_completion(completion) {
4731 debug!(
4732 connection_id = connection_id.get(),
4733 corr = frame.header.corr,
4734 "dropping late or unknown module-control RPC error"
4735 );
4736 }
4737 return Ok(Vec::new());
4738 }
4739 match serde_json::from_slice::<ErrorBody>(&frame.body) {
4740 Ok(body) => RouteBindRelayOutcome::Rejected(body),
4741 Err(err) => {
4742 let message = format!("malformed route.bind ERROR body: {err}");
4743 secondary_error = Some(control_error_frame(
4744 &frame,
4745 "invalid_control_body",
4746 message.clone(),
4747 )?);
4748 RouteBindRelayOutcome::ModuleGone(message)
4749 }
4750 }
4751 }
4752 ty => {
4753 return Ok(vec![control_error_frame(
4754 &frame,
4755 "unsupported_control_frame",
4756 format!("unsupported module channel-0 frame {ty:?}"),
4757 )?])
4758 }
4759 };
4760
4761 let settled =
4762 self.forwarding
4763 .complete_pending_relay(connection_id, frame.header.corr, outcome);
4764 let completion = match settled {
4765 Ok(completion) => completion,
4766 Err(err) => {
4767 self.refuse_to_end_module_connection_for_a_client(
4768 connection_id,
4769 frame.header.corr,
4770 err,
4771 )?;
4772 return Ok(secondary_error.into_iter().collect());
4773 }
4774 };
4775 if let Some(target) = completion.abandoned.as_ref() {
4776 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
4777 }
4778 if !completion.settled {
4779 debug!(
4780 connection_id = connection_id.get(),
4781 corr = frame.header.corr,
4782 frame_type = ?frame.header.ty,
4783 "dropping late or unknown route.bind relay response"
4784 );
4785 }
4786 Ok(secondary_error.into_iter().collect())
4787 }
4788
4789 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
4790 debug!(connection_id = connection_id.get(), "handling GOODBYE");
4791 let registrations = self
4792 .deregister_connection(connection_id)
4793 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
4794 let released_routes = self
4795 .forwarding
4796 .cleanup_connection(connection_id)
4797 .map_err(RouterError::Forwarding)?;
4798 self.emit_route_goodbyes(released_routes);
4799 if !registrations.is_empty() {
4801 crate::supervise::notify_registration_release();
4802 }
4803 Ok(Vec::new())
4804 }
4805}
4806
4807impl Default for ControlHandler {
4808 fn default() -> Self {
4809 Self::new(Arc::new(Registry::default()))
4810 }
4811}
4812
4813impl crate::supervise::SwapPromotionObserver for ControlHandler {
4814 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
4815 self.apply_registration_capabilities(registration);
4816 }
4817}
4818
4819fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
4820 CapabilityRequirementStatus {
4821 consumer: status.consumer,
4822 capability: status.capability,
4823 need: match status.need {
4824 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
4825 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
4826 },
4827 verdict: status.verdict.as_str().to_string(),
4828 episode_seq: status.episode_seq,
4829 config_satisfiable: status.config_satisfiable,
4830 runtime_available: status.runtime_available,
4831 detail: status.detail,
4832 }
4833}
4834
4835fn append_capability_problem_detail(
4836 detail: Option<String>,
4837 capability_detail: Option<String>,
4838) -> Option<String> {
4839 match (detail, capability_detail) {
4840 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
4841 (Some(detail), None) => Some(detail),
4842 (None, Some(capability_detail)) => Some(capability_detail),
4843 (None, None) => None,
4844 }
4845}
4846
4847fn subc_ops() -> Vec<String> {
4848 SUBC_CONTROL_OPS
4849 .iter()
4850 .map(|op| (*op).to_string())
4851 .collect()
4852}
4853
4854fn module_subc_ops() -> Vec<String> {
4855 SUBC_CONTROL_OPS
4856 .iter()
4857 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
4858 .map(|op| (*op).to_string())
4859 .collect()
4860}
4861
4862#[cfg(test)]
4863fn module_baseline_control_ops() -> Vec<String> {
4864 MODULE_BASELINE_CONTROL_OPS
4865 .iter()
4866 .map(|op| (*op).to_string())
4867 .collect()
4868}
4869
4870fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
4871 let mut seen = HashSet::new();
4872 let mut effective = Vec::new();
4873 for op in MODULE_BASELINE_CONTROL_OPS {
4874 if seen.insert((*op).to_string()) {
4875 effective.push((*op).to_string());
4876 }
4877 }
4878 for op in declared.unwrap_or_default() {
4879 if seen.insert(op.clone()) {
4880 effective.push(op);
4881 }
4882 }
4883 effective
4884}
4885
4886fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
4887 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
4888}
4889
4890fn target_module_id(target: &RouteTarget) -> &str {
4891 match target {
4892 RouteTarget::ToolProvider { module_id }
4893 | RouteTarget::ManagementSurface { module_id }
4894 | RouteTarget::InternalService { module_id, .. } => module_id,
4895 }
4896}
4897
4898fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
4899 roles.iter().any(|role| match (target, role) {
4900 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
4901 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
4902 (
4903 RouteTarget::InternalService { service_id, .. },
4904 ProviderRole::InternalService {
4905 service_id: provided,
4906 ..
4907 },
4908 ) => service_id == provided,
4909 _ => false,
4910 })
4911}
4912
4913fn is_routable_role(role: &ProviderRole) -> bool {
4914 matches!(
4915 role,
4916 ProviderRole::ToolProvider { .. }
4917 | ProviderRole::ManagementSurface { .. }
4918 | ProviderRole::InternalService { .. }
4919 )
4920}
4921
4922#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4923enum ControlRequestBodyError {
4924 UnknownOp,
4925 InvalidBody,
4926}
4927
4928#[derive(Debug, Deserialize)]
4929struct ControlOpProbe {
4930 op: String,
4931}
4932
4933const MODULE_PUSH_OPS: &[&str] = &["route.status"];
4936
4937fn is_known_module_push_op(body: &[u8]) -> bool {
4938 serde_json::from_slice::<ControlOpProbe>(body)
4939 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
4940 .unwrap_or(false)
4941}
4942
4943fn is_known_module_request_op(body: &[u8]) -> bool {
4944 serde_json::from_slice::<ControlOpProbe>(body)
4945 .map(|probe| MODULE_TO_SUBC_CONTROL_OPS.contains(&probe.op.as_str()))
4946 .unwrap_or(false)
4947}
4948
4949fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
4950 debug!(
4951 op = %op,
4952 connection_id = connection_id.get(),
4953 corr,
4954 "control dispatch"
4955 );
4956}
4957
4958fn log_slow_control_dispatch(
4959 dispatch_started_at: Option<StdInstant>,
4960 op: &'static str,
4961 connection_id: ConnectionId,
4962 corr: u64,
4963) {
4964 let Some(dispatch_started_at) = dispatch_started_at else {
4965 return;
4966 };
4967 let elapsed = dispatch_started_at.elapsed();
4968 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
4969 warn!(
4970 op = %op,
4971 connection_id = connection_id.get(),
4972 corr,
4973 elapsed_ms = elapsed.as_millis() as u64,
4974 "slow control dispatch"
4975 );
4976 }
4977}
4978
4979fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
4980 match request {
4981 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
4982 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
4983 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
4984 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
4985 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
4986 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
4987 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
4988 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
4989 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
4990 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
4991 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
4992 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
4993 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
4994 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
4995 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
4996 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
4997 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
4998 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
4999 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
5000 }
5001}
5002
5003fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
5004 match request {
5005 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
5006 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
5007 }
5008}
5009
5010fn parse_client_control_request(
5011 body: &[u8],
5012) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
5013 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
5014 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5015 Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
5016 ControlRequestBodyError::InvalidBody
5017 }
5018 Ok(_) => ControlRequestBodyError::UnknownOp,
5019 Err(_) => ControlRequestBodyError::InvalidBody,
5020 };
5021 (err, classification)
5022 })
5023}
5024
5025fn parse_module_control_request_from_module(
5026 body: &[u8],
5027) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
5028 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
5029 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5030 Ok(probe) if MODULE_TO_SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
5031 ControlRequestBodyError::InvalidBody
5032 }
5033 Ok(_) => ControlRequestBodyError::UnknownOp,
5034 Err(_) => ControlRequestBodyError::InvalidBody,
5035 };
5036 (err, classification)
5037 })
5038}
5039
5040#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
5041enum ProviderRoleKind {
5042 ToolProvider,
5043 PipelineStage,
5044 ManagementSurface,
5045 InternalService,
5046}
5047
5048fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
5049 match role {
5050 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
5051 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
5052 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
5053 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
5054 }
5055}
5056
5057fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
5058 roles.iter().map(provider_role_kind).collect()
5059}
5060
5061fn capability_census_trigger(
5065 old: Option<&CapabilityDeclarations>,
5066 new: Option<&CapabilityDeclarations>,
5067) -> bool {
5068 let old_provides = old
5069 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
5070 .unwrap_or_default();
5071 let old_denies = old
5072 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
5073 .unwrap_or_default();
5074 let new = new.cloned().unwrap_or(CapabilityDeclarations {
5075 provides: Vec::new(),
5076 requires: Vec::new(),
5077 must_never_reach: Vec::new(),
5078 });
5079
5080 new.provides
5081 .iter()
5082 .any(|capability| !old_provides.contains(capability))
5083 || new
5084 .must_never_reach
5085 .iter()
5086 .any(|capability| !old_denies.contains(capability))
5087}
5088
5089fn denied_capability<'a>(
5092 opening_manifest: &'a ModuleManifest,
5093 target_manifest: &ModuleManifest,
5094) -> Option<&'a str> {
5095 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
5096 let target_capabilities = target_manifest.capabilities.as_ref()?;
5097 opening_capabilities
5098 .must_never_reach
5099 .iter()
5100 .find(|denied| {
5101 target_capabilities
5102 .provides
5103 .iter()
5104 .any(|provided| provided == *denied)
5105 })
5106 .map(String::as_str)
5107}
5108
5109fn catalog_update_frozen_field_message(
5110 registered: &ModuleManifest,
5111 provides: &[ProviderRole],
5112) -> Option<String> {
5113 let old_has_provides = !registered.provides.is_empty();
5114 let new_has_provides = !provides.is_empty();
5115 if old_has_provides != new_has_provides {
5116 return Some(format!(
5117 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
5118 registered.module_id
5119 ));
5120 }
5121
5122 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
5123 return Some(format!(
5124 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
5125 registered.module_id
5126 ));
5127 }
5128
5129 let registered_concurrency = manifest_concurrency(registered);
5130 let mut candidate = registered.clone();
5131 candidate.provides = provides.to_vec();
5132 let candidate_concurrency = manifest_concurrency(&candidate);
5133 if candidate_concurrency != registered_concurrency {
5134 return Some(format!(
5135 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
5136 registered.module_id, registered_concurrency, candidate_concurrency
5137 ));
5138 }
5139
5140 None
5143}
5144
5145fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
5146 manifest.provides.iter().any(is_routable_role)
5147}
5148
5149fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
5155 manifest
5156 .provides
5157 .iter()
5158 .find_map(|provider| match provider {
5159 ProviderRole::ToolProvider { concurrency, .. }
5160 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
5161 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
5162 })
5163 .unwrap_or(Concurrency::ModuleManaged)
5164}
5165
5166fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
5173 let has_management_surface = manifest
5174 .provides
5175 .iter()
5176 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
5177 if !has_management_surface {
5178 return false;
5179 }
5180 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
5181 return false;
5182 };
5183 let Some(provides) = raw
5184 .get("manifest")
5185 .and_then(|manifest| manifest.get("provides"))
5186 .and_then(serde_json::Value::as_array)
5187 else {
5188 return false;
5189 };
5190 provides.iter().any(|role| {
5195 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
5196 && role.get("concurrency").is_none()
5197 })
5198}
5199
5200fn negotiate_version(peer_version: u8) -> Result<u8, String> {
5201 if peer_version != PROTOCOL_VERSION {
5202 return Err(format!(
5203 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
5204 ));
5205 }
5206 Ok(PROTOCOL_VERSION)
5207}
5208
5209fn pong(frame: &Frame) -> Result<Frame, RouterError> {
5210 Frame::build_with_version(
5211 response_version(frame),
5212 FrameType::Pong,
5213 frame.header.flags,
5214 0,
5215 0,
5216 frame.header.corr,
5217 Vec::new(),
5218 )
5219 .map_err(RouterError::FrameBuild)
5220}
5221
5222fn control_error_frame(
5223 frame: &Frame,
5224 code: &'static str,
5225 message: impl Into<String>,
5226) -> Result<Frame, RouterError> {
5227 control_error_body_frame(
5228 frame,
5229 ErrorBody {
5230 code: code.to_string(),
5231 message: message.into(),
5232 detail: None,
5233 },
5234 )
5235}
5236
5237fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
5238 let body = serde_json::to_vec(&error).map_err(|err| {
5239 RouterError::backend(
5240 0,
5241 frame.header.corr,
5242 format!("failed to encode control ERROR: {err}"),
5243 )
5244 })?;
5245
5246 Frame::build_with_version(
5247 response_version(frame),
5248 FrameType::Error,
5249 control_flags(),
5250 0,
5251 0,
5252 frame.header.corr,
5253 body,
5254 )
5255 .map_err(RouterError::FrameBuild)
5256}
5257
5258fn control_response_body_frame<T: Serialize>(
5259 frame: &Frame,
5260 reply: &T,
5261 label: &'static str,
5262) -> Result<Frame, RouterError> {
5263 let body = serde_json::to_vec(reply).map_err(|err| {
5264 RouterError::backend(
5265 0,
5266 frame.header.corr,
5267 format!("failed to encode {label}: {err}"),
5268 )
5269 })?;
5270
5271 Frame::build_with_version(
5272 response_version(frame),
5273 FrameType::Response,
5274 control_flags(),
5275 0,
5276 0,
5277 frame.header.corr,
5278 body,
5279 )
5280 .map_err(RouterError::FrameBuild)
5281}
5282
5283fn forwarding_error_code(err: &ForwardingError) -> &'static str {
5313 match err {
5314 ForwardingError::NoModuleConnection => "target_unavailable",
5315 ForwardingError::ModuleReloading { .. } => "module_reloading",
5316 ForwardingError::ClientRouteChannelExhausted { .. }
5317 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
5318 ForwardingError::StaleModuleEndpoint
5319 | ForwardingError::UnknownReservation { .. }
5320 | ForwardingError::ConnectionClosing { .. }
5321 | ForwardingError::ClientEgressClosed { .. }
5322 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
5323 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
5326 ForwardingError::RelayCorrelationExhausted
5327 | ForwardingError::RouteOpenBuild(_)
5328 | ForwardingError::Poisoned => "forwarding_error",
5329 }
5330}
5331
5332fn response_version(frame: &Frame) -> u8 {
5333 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
5334 frame.header.ver
5335 } else {
5336 PROTOCOL_VERSION
5337 }
5338}
5339
5340fn control_flags() -> Flags {
5341 Flags::new(false, Priority::Passive, false)
5342}
5343
5344fn send_goodbye_target_best_effort(
5349 counters: &DaemonCounters,
5350 target: &GoodbyeTarget,
5351 context: &'static str,
5352) {
5353 let Ok(frame) = Frame::build_with_version(
5354 target.negotiated_ver,
5355 FrameType::Goodbye,
5356 control_flags(),
5357 target.channel,
5358 target.epoch,
5359 0,
5360 Vec::new(),
5361 ) else {
5362 return;
5363 };
5364 crate::forwarding::send_module_route_goodbye(
5365 counters,
5366 &target.sink,
5367 frame,
5368 target.module_id.as_deref(),
5369 context,
5370 );
5371}
5372
5373pub(crate) fn send_route_control_pushes(
5374 forwarding: &ForwardingTable,
5375 routes: Vec<EndpointRoute>,
5376 push: ClientControlPush,
5377) {
5378 let body = match serde_json::to_vec(&push) {
5379 Ok(body) => body,
5380 Err(err) => {
5381 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
5382 return;
5383 }
5384 };
5385 let mut targets = Vec::new();
5386 for route in routes {
5387 let target = route.goodbye_target;
5388 if let Some(existing) = targets
5389 .iter()
5390 .find(|existing: &&GoodbyeTarget| existing.connection_id == target.connection_id)
5391 {
5392 debug_assert_eq!(
5393 existing.negotiated_ver, target.negotiated_ver,
5394 "one connection cannot negotiate multiple frame versions"
5395 );
5396 continue;
5397 }
5398 targets.push(target);
5399 }
5400 for target in targets {
5401 let frame = match Frame::build_with_version(
5402 target.negotiated_ver,
5403 FrameType::Push,
5404 control_flags(),
5405 0,
5406 0,
5407 0,
5408 body.clone(),
5409 ) {
5410 Ok(frame) => frame,
5411 Err(err) => {
5412 warn!(
5413 route_channel = target.channel,
5414 error = %err,
5415 "failed to build route lifecycle control PUSH frame"
5416 );
5417 continue;
5418 }
5419 };
5420 if let Err(err) = target.sink.try_send(frame) {
5421 if target.close_on_delivery_failure() {
5422 warn!(
5423 target_connection_id = target.connection_id.get(),
5424 route_channel = target.channel,
5425 error = %err,
5426 "route lifecycle control PUSH was not delivered to client; closing target connection"
5427 );
5428 let _ = forwarding.escalate_client_delivery_failure(
5429 target.connection_id,
5430 target.channel,
5431 target.epoch,
5432 CloseReason::new(
5433 "route_lifecycle_push_delivery_failed",
5434 format!(
5435 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
5436 target.channel
5437 ),
5438 ),
5439 crate::forwarding::UndeliveredFrame {
5440 module_id: target.module_id.as_deref(),
5441 sink: &target.sink,
5442 },
5443 );
5444 }
5445 }
5446 }
5447}
5448
5449#[cfg(test)]
5450mod tests {
5451 use std::{
5452 collections::BTreeMap,
5453 fmt,
5454 path::PathBuf,
5455 sync::{Arc, Mutex},
5456 time::Duration,
5457 };
5458
5459 use serde_json::{json, Value};
5460 use subc_protocol::{
5461 manifest::{
5462 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
5463 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
5464 },
5465 session::HealthStatus,
5466 FrameType,
5467 };
5468
5469 use super::*;
5470 use crate::{
5471 forwarding::{DataRoute, DataRouteState},
5472 registry::ChannelState,
5473 router::FrameSink,
5474 stderr_tail::DEFAULT_MAX_LINE_BYTES,
5475 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
5476 test_support::TestTempDir,
5477 RouteCtx, Router,
5478 };
5479 use tokio::{
5480 sync::mpsc,
5481 time::{sleep, Instant},
5482 };
5483 use tracing::{
5484 field::{Field, Visit},
5485 Event, Subscriber,
5486 };
5487 use tracing_subscriber::{layer::Context, prelude::*, Layer};
5488
5489 fn fake_aft_stub_path() -> PathBuf {
5508 let mut path = std::env::current_exe().expect("current_exe available in tests");
5509 path.pop(); path.pop(); path.push(if cfg!(windows) {
5512 "fake-aft-stub.exe"
5513 } else {
5514 "fake-aft-stub"
5515 });
5516 assert!(
5517 path.exists(),
5518 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
5519 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
5520 path.display()
5521 );
5522 path
5523 }
5524
5525 fn client_retries(code: &str) -> bool {
5530 subc_protocol::error_codes::is_retryable_route_open(code)
5531 }
5532
5533 #[test]
5544 fn retryability_of_forwarding_codes_matches_the_failure() {
5545 let transient = [
5548 ForwardingError::NoModuleConnection,
5549 ForwardingError::ModuleReloading {
5550 module_id: "m".into(),
5551 },
5552 ForwardingError::StaleModuleEndpoint,
5553 ForwardingError::UnknownReservation {
5554 client_channel: 1,
5555 module_channel: 1,
5556 },
5557 ForwardingError::ConnectionClosing {
5558 connection_id: ConnectionId::new(1),
5559 },
5560 ForwardingError::ClientEgressClosed {
5561 connection_id: ConnectionId::new(1),
5562 },
5563 ForwardingError::ModuleEgressUnavailable {
5564 connection_id: ConnectionId::new(1),
5565 },
5566 ];
5567 for err in transient {
5568 let code = forwarding_error_code(&err);
5569 assert!(
5570 client_retries(code),
5571 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
5572 );
5573 }
5574
5575 let permanent = [
5580 ForwardingError::ClientRouteChannelExhausted {
5581 connection_id: ConnectionId::new(1),
5582 },
5583 ForwardingError::ModuleRouteChannelExhausted {
5584 endpoint: ModuleEndpointId {
5585 connection_id: ConnectionId::new(1),
5586 generation: 1,
5587 },
5588 },
5589 ForwardingError::RelayCorrelationExhausted,
5590 ForwardingError::RouteOpenBuild("x".into()),
5591 ForwardingError::Poisoned,
5592 ];
5593 for err in permanent {
5594 let code = forwarding_error_code(&err);
5595 assert!(
5596 !client_retries(code),
5597 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
5598 );
5599 }
5600 }
5601
5602 #[tokio::test]
5630 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
5631 let handler = ControlHandler::default();
5632 let frame =
5633 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
5634
5635 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
5639 assert_eq!(
5640 stamped,
5641 Principal::Direct,
5642 "a caller that proved nothing must not be stamped as a supervised module"
5643 );
5644
5645 let forged = handler
5650 .route_open_principal(
5651 &frame,
5652 Some(ConsumerIdentity {
5653 module_id: "aft".to_string(),
5654 launch_nonce: "not-a-real-nonce".to_string(),
5655 }),
5656 )
5657 .unwrap();
5658 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
5659 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
5660 }
5661
5662 #[test]
5679 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
5680 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
5681 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
5682 let ClientControlRequest::RouteOpen {
5683 consumer_identity, ..
5684 } = parsed
5685 else {
5686 panic!("route.open body must parse as RouteOpen");
5687 };
5688 assert_eq!(
5689 consumer_identity,
5690 Some(ConsumerIdentity {
5691 module_id: "aft".to_string(),
5692 launch_nonce: "n".to_string(),
5693 }),
5694 "the wire field name must reach the value route_open_principal reads"
5695 );
5696 }
5697
5698 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
5699 ModuleManifest::builder(module_id, "0.1.0")
5700 .protocol_ver(protocol_ver)
5701 .provides(vec![ProviderRole::ToolProvider {
5702 tools: vec![Tool {
5703 name: "read".to_string(),
5704 description: None,
5705 execution_mode: ExecutionMode::Pure,
5706 schema: json!({"type": "object"}),
5707 }],
5708 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
5709 concurrency: Concurrency::ModuleManaged,
5710 emits_push: true,
5711 sub_supervises: true,
5712 }])
5713 .build()
5714 }
5715
5716 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
5717 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
5718 }
5719
5720 fn hello_frame_with_control_ops(
5721 module_id: &str,
5722 protocol_ver: u8,
5723 corr: u64,
5724 control_ops: Option<Vec<String>>,
5725 ) -> Frame {
5726 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
5727 }
5728
5729 fn hello_frame_with_nonce(
5730 module_id: &str,
5731 protocol_ver: u8,
5732 corr: u64,
5733 launch_nonce: Option<&str>,
5734 ) -> Frame {
5735 hello_frame_full(
5736 module_id,
5737 protocol_ver,
5738 corr,
5739 None,
5740 launch_nonce.map(ToOwned::to_owned),
5741 )
5742 }
5743
5744 fn hello_frame_full(
5745 module_id: &str,
5746 protocol_ver: u8,
5747 corr: u64,
5748 control_ops: Option<Vec<String>>,
5749 launch_nonce: Option<String>,
5750 ) -> Frame {
5751 let body = serde_json::to_vec(&ModuleHelloBody {
5752 manifest: manifest(module_id, protocol_ver),
5753 protocol_ver,
5754 control_ops,
5755 launch_nonce,
5756 })
5757 .unwrap();
5758 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
5759 }
5760
5761 fn non_routable_hello_frame_with_control_ops(
5762 module_id: &str,
5763 corr: u64,
5764 control_ops: Option<Vec<String>>,
5765 ) -> Frame {
5766 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
5767 manifest.provides.clear();
5768 let body = serde_json::to_vec(&ModuleHelloBody {
5769 manifest,
5770 protocol_ver: PROTOCOL_VERSION,
5771 control_ops,
5772 launch_nonce: None,
5773 })
5774 .unwrap();
5775 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
5776 }
5777
5778 fn capability_grammar_hello_frame(
5779 capabilities: Value,
5780 runtime_computed: Option<Value>,
5781 corr: u64,
5782 ) -> Frame {
5783 let mut body = serde_json::to_value(ModuleHelloBody {
5784 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
5785 protocol_ver: PROTOCOL_VERSION,
5786 control_ops: None,
5787 launch_nonce: None,
5788 })
5789 .expect("HELLO body serializes");
5790 body["manifest"]["capabilities"] = capabilities;
5791 if let Some(runtime_computed) = runtime_computed {
5792 body["runtime_computed"] = runtime_computed;
5793 }
5794 Frame::build(
5795 FrameType::Hello,
5796 control_flags(),
5797 0,
5798 0,
5799 corr,
5800 serde_json::to_vec(&body).expect("HELLO body reserializes"),
5801 )
5802 .expect("HELLO frame builds")
5803 }
5804
5805 fn channel_request(channel: u16, corr: u64) -> Frame {
5806 Frame::build(
5807 FrameType::Request,
5808 Flags::new(true, Priority::Interactive, false),
5809 channel,
5810 0,
5811 corr,
5812 b"opaque".to_vec(),
5813 )
5814 .unwrap()
5815 }
5816
5817 fn route_ctx(
5818 connection_id: ConnectionId,
5819 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
5820 let (tx, rx) = mpsc::channel(8);
5821 (
5822 RouteCtx {
5823 connection_id,
5824 egress: FrameSink::new(tx),
5825 },
5826 rx,
5827 )
5828 }
5829
5830 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
5831 serde_json::from_slice(&frame.body).unwrap()
5832 }
5833
5834 async fn hello_via_sink(
5839 handler: &ControlHandler,
5840 ctx: &RouteCtx,
5841 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
5842 hello: Frame,
5843 ) -> Frame {
5844 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
5845 assert!(
5846 replies.is_empty(),
5847 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
5848 );
5849 let ack = rx
5850 .try_recv()
5851 .expect("HELLO_ACK is queued on the module sink")
5852 .frame;
5853 assert_eq!(ack.header.ty, FrameType::HelloAck);
5854 ack
5855 }
5856
5857 fn parse_error(frame: &Frame) -> Value {
5858 serde_json::from_slice(&frame.body).unwrap()
5859 }
5860
5861 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
5862 serde_json::from_slice(&frame.body).unwrap()
5863 }
5864
5865 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
5866 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
5867 route_channel,
5868 route_epoch: 0,
5869 kind,
5870 })
5871 .unwrap();
5872 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
5873 }
5874
5875 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
5876 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
5877 module_id: module_id.to_string(),
5878 })
5879 .unwrap();
5880 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
5881 }
5882
5883 fn route_open_frame(corr: u64, module_id: &str, project_root: TestTempDir) -> Frame {
5884 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
5885 }
5886
5887 fn route_open_frame_with_consumer_capabilities(
5888 corr: u64,
5889 module_id: &str,
5890 project_root: TestTempDir,
5891 consumer_capabilities: Option<Vec<String>>,
5892 ) -> Frame {
5893 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
5894 target: RouteTarget::ToolProvider {
5895 module_id: module_id.to_string(),
5896 },
5897 identity: BindIdentity::new(
5898 project_root.path().to_path_buf(),
5899 "unit".to_string(),
5900 "session".to_string(),
5901 ),
5902 consumer_identity: None,
5903 consumer_capabilities,
5904 admission_facts: None,
5905 })
5906 .unwrap();
5907 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
5908 }
5909
5910 fn route_open_frame_with_admission_facts(
5911 corr: u64,
5912 module_id: &str,
5913 project_root: TestTempDir,
5914 consumer_identity: Option<subc_control::ConsumerIdentity>,
5915 facts: Option<Value>,
5916 ) -> Frame {
5917 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
5918 target: RouteTarget::ToolProvider {
5919 module_id: module_id.to_string(),
5920 },
5921 identity: BindIdentity::new(
5922 project_root.path().to_path_buf(),
5923 "unit".to_string(),
5924 format!("session-{corr}"),
5925 ),
5926 consumer_identity,
5927 consumer_capabilities: None,
5928 admission_facts: facts,
5929 })
5930 .unwrap();
5931 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
5932 }
5933
5934 #[derive(Clone, Default)]
5935 struct EventCapture {
5936 events: Arc<Mutex<Vec<CapturedEvent>>>,
5937 }
5938
5939 #[derive(Clone, Debug)]
5940 struct CapturedEvent {
5941 target: String,
5942 fields: BTreeMap<String, String>,
5943 }
5944
5945 impl EventCapture {
5946 fn events(&self) -> Vec<CapturedEvent> {
5947 self.events.lock().unwrap().clone()
5948 }
5949 }
5950
5951 impl<S> Layer<S> for EventCapture
5952 where
5953 S: Subscriber,
5954 {
5955 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
5956 let mut visitor = EventFieldVisitor::default();
5957 event.record(&mut visitor);
5958 self.events.lock().unwrap().push(CapturedEvent {
5959 target: event.metadata().target().to_string(),
5960 fields: visitor.fields,
5961 });
5962 }
5963 }
5964
5965 #[derive(Default)]
5966 struct EventFieldVisitor {
5967 fields: BTreeMap<String, String>,
5968 }
5969
5970 impl Visit for EventFieldVisitor {
5971 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
5972 self.fields
5973 .insert(field.name().to_string(), format!("{value:?}"));
5974 }
5975 }
5976
5977 fn health_response(corr: u64, status: HealthStatus) -> Frame {
5978 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
5979 status,
5980 detail: Some("warming".to_string()),
5981 metrics: Some(json!({"queue_depth": 3})),
5982 })
5983 .unwrap();
5984 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
5985 }
5986
5987 fn route_bind_ack(corr: u64) -> Frame {
5988 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
5989 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
5990 }
5991
5992 fn unique_project_root(label: &str) -> TestTempDir {
5993 TestTempDir::new(label)
5994 }
5995
5996 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
5997 match parse_route_poll(frame) {
5998 ClientControlResponse::RoutePoll {
5999 status: None,
6000 live: Some(live),
6001 ..
6002 } => assert_eq!(live, expected_live),
6003 other => panic!("unexpected route.poll response: {other:?}"),
6004 }
6005 }
6006
6007 fn bind_liveness_route(
6008 registry: &Registry,
6009 forwarding: &ForwardingTable,
6010 module_id: &str,
6011 ) -> (RouteCtx, u16, u32) {
6012 let module_connection = ConnectionId::new(101);
6013 let client_connection = ConnectionId::new(202);
6014 let registration = registry
6015 .register_with_control_ops(
6016 manifest(module_id, PROTOCOL_VERSION),
6017 PROTOCOL_VERSION,
6018 module_connection,
6019 module_baseline_control_ops(),
6020 )
6021 .unwrap();
6022 let (module_tx, _module_rx) = mpsc::channel(8);
6023 let endpoint = forwarding
6024 .register_module_connection(
6025 module_connection,
6026 module_id.to_string(),
6027 PROTOCOL_VERSION,
6028 manifest_concurrency(®istration.manifest),
6029 FrameSink::new(module_tx),
6030 )
6031 .unwrap();
6032 let (client_ctx, _client_rx) = route_ctx(client_connection);
6033 let pending = forwarding
6034 .begin_route_bind_relay_for_test(
6035 client_connection,
6036 client_ctx.egress.clone(),
6037 1,
6038 module_id,
6039 )
6040 .unwrap();
6041 assert_eq!(pending.endpoint, endpoint);
6042 let route_channel = pending.client_channel;
6043 let route_epoch = pending.client_epoch;
6044 forwarding
6045 .complete_pending_relay(
6046 module_connection,
6047 pending.corr,
6048 RouteBindRelayOutcome::Accepted,
6049 )
6050 .unwrap();
6051 (client_ctx, route_channel, route_epoch)
6052 }
6053
6054 struct FakeProcessLiveness {
6055 live: Option<bool>,
6056 }
6057
6058 impl ModuleProcessLiveness for FakeProcessLiveness {
6059 fn process_live(&self, _module_id: &str) -> Option<bool> {
6060 self.live
6061 }
6062 }
6063
6064 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6065 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
6066 {
6067 let registry = Arc::new(Registry::default());
6068 let supervisor_handle = SupervisorHandle::new();
6069 let supervisor = Supervisor::new(
6070 Arc::clone(®istry),
6071 RestartPolicy::new(1, Duration::from_millis(10)),
6072 )
6073 .with_handle(supervisor_handle.clone());
6074 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
6075 let module = supervisor
6076 .spawn(ModuleSpec {
6077 module_id: "stderr-tail-wire".to_string(),
6078 program: fake_aft_stub_path(),
6079 args: Vec::new(),
6080 env: vec![
6081 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
6082 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
6083 ],
6084 reserved: false,
6085 reserved_prefixes: Vec::new(),
6086 protocol: ModuleProtocol::Subc,
6087 overlap: Default::default(),
6088 })
6089 .unwrap();
6090
6091 let deadline = Instant::now() + Duration::from_secs(5);
6092 loop {
6093 let tail = module.stderr_tail(None, None);
6094 if tail
6095 .entries
6096 .iter()
6097 .any(|entry| matches!(entry, TailEntry::ProcessStart))
6098 && tail.entries.iter().any(|entry| {
6099 matches!(
6100 entry,
6101 TailEntry::Line {
6102 truncated: true,
6103 ..
6104 }
6105 )
6106 })
6107 {
6108 break;
6109 }
6110 assert!(
6111 Instant::now() < deadline,
6112 "module did not produce a truncated line and restart boundary: {tail:?}"
6113 );
6114 sleep(Duration::from_millis(10)).await;
6115 }
6116
6117 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
6118 let request = ClientControlRequest::SupervisorStderrTail {
6119 module_id: "stderr-tail-wire".to_string(),
6120 max_lines: None,
6121 max_bytes: None,
6122 };
6123 let frame = Frame::build(
6124 FrameType::Request,
6125 control_flags(),
6126 0,
6127 0,
6128 1,
6129 serde_json::to_vec(&request).unwrap(),
6130 )
6131 .unwrap();
6132 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6133 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
6134 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
6135 serde_json::from_slice(&responses[0].body).unwrap()
6136 else {
6137 panic!("expected supervisor.stderr_tail response");
6138 };
6139
6140 assert!(
6141 tail.entries
6142 .iter()
6143 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
6144 "the control response lost the restart boundary"
6145 );
6146 let Some(StderrTailEntry::Line { text, truncated }) = tail.entries.iter().find(|entry| {
6147 matches!(
6148 entry,
6149 StderrTailEntry::Line {
6150 truncated: true,
6151 ..
6152 }
6153 )
6154 }) else {
6155 panic!("the control response lost the truncated line");
6156 };
6157 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
6158 assert!(*truncated);
6159 }
6160
6161 #[tokio::test(flavor = "current_thread")]
6166 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
6167 let dir = TestTempDir::new("terminals-off-worker");
6168 let journal_path = dir.join("terminals.jsonl");
6169 let registry = Arc::new(Registry::default());
6170 let supervisor_handle = SupervisorHandle::new();
6171 let supervisor =
6172 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
6173 .with_handle(supervisor_handle.clone())
6174 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
6175 let module = supervisor
6176 .spawn(ModuleSpec {
6177 module_id: "terminal-off-worker".to_string(),
6178 program: fake_aft_stub_path(),
6179 args: Vec::new(),
6180 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
6181 reserved: false,
6182 reserved_prefixes: Vec::new(),
6183 protocol: ModuleProtocol::Subc,
6184 overlap: Default::default(),
6185 })
6186 .unwrap();
6187 let deadline = Instant::now() + Duration::from_secs(5);
6188 while module.terminal_history().entries.len() != 2 {
6189 assert!(Instant::now() < deadline, "module did not record two exits");
6190 sleep(Duration::from_millis(10)).await;
6191 }
6192
6193 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
6194 let handler =
6195 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
6196 let frame = Frame::build(
6197 FrameType::Request,
6198 control_flags(),
6199 0,
6200 0,
6201 1,
6202 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
6203 module_id: "terminal-off-worker".to_string(),
6204 })
6205 .unwrap(),
6206 )
6207 .unwrap();
6208 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6209 let spawned_at = std::time::Instant::now();
6210 let read = tokio::spawn({
6211 let handler = Arc::clone(&handler);
6212 async move { handler.handle_control_frame(&ctx, frame).await }
6213 });
6214 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
6217 .await
6218 .unwrap()
6219 .expect("the history read reached its pause");
6220 let elapsed = spawned_at.elapsed();
6221 assert!(
6222 elapsed < Duration::from_secs(2) && !read.is_finished(),
6223 "this task could not run while the history read was paused \
6224 (resumed after {elapsed:?}, read finished: {})",
6225 read.is_finished()
6226 );
6227
6228 drop(release);
6229 let responses = read.await.unwrap().unwrap();
6230 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
6231 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
6232 panic!("expected supervisor.terminals response");
6233 };
6234 assert_eq!(terminals.entries.len(), 2);
6235 assert_eq!(terminals.journal_skipped_lines, 0);
6236 assert_eq!(terminals.journal_read_errors, 0);
6237 }
6238
6239 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6240 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
6241 let registry = Arc::new(Registry::default());
6242 let supervisor_handle = SupervisorHandle::new();
6243 let supervisor =
6244 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
6245 .with_handle(supervisor_handle.clone());
6246 let module = supervisor
6247 .spawn(ModuleSpec {
6248 module_id: "terminal-golden".to_string(),
6249 program: fake_aft_stub_path(),
6250 args: Vec::new(),
6251 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
6252 reserved: false,
6253 reserved_prefixes: Vec::new(),
6254 protocol: ModuleProtocol::Subc,
6255 overlap: Default::default(),
6256 })
6257 .unwrap();
6258
6259 let deadline = Instant::now() + Duration::from_secs(5);
6260 while module.terminal_history().entries.len() != 2 {
6261 assert!(
6262 Instant::now() < deadline,
6263 "module did not retain two terminal exits: {:?}",
6264 module.terminal_history()
6265 );
6266 sleep(Duration::from_millis(10)).await;
6267 }
6268
6269 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
6270 let request = ClientControlRequest::SupervisorTerminals {
6271 module_id: "terminal-golden".to_string(),
6272 };
6273 let frame = Frame::build(
6274 FrameType::Request,
6275 control_flags(),
6276 0,
6277 0,
6278 1,
6279 serde_json::to_vec(&request).unwrap(),
6280 )
6281 .unwrap();
6282 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6283 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
6284 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
6285 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
6286 panic!("expected supervisor.terminals response");
6287 };
6288 assert_eq!(terminals.entries.len(), 2);
6289 assert_eq!(terminals.dropped, 0);
6290
6291 let mut rendered = serde_json::to_value(response).unwrap();
6292 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
6295 for (index, entry) in rendered["entries"]
6296 .as_array_mut()
6297 .expect("terminal response entries array")
6298 .iter_mut()
6299 .enumerate()
6300 {
6301 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
6302 }
6303
6304 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
6305 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
6306 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
6307 if std::env::var_os("UPDATE_GOLDEN").is_some() {
6308 std::fs::write(&golden_path, &serialized).unwrap();
6309 }
6310 let expected: Value =
6311 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
6312 assert_eq!(rendered, expected);
6313 }
6314
6315 #[test]
6316 fn hello_registers_manifest_and_returns_ack() {
6317 let registry = Arc::new(Registry::default());
6318 let handler = ControlHandler::new(Arc::clone(®istry));
6319 let conn = ConnectionId::new(1);
6320
6321 let responses = handler
6322 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
6323 .unwrap();
6324
6325 assert_eq!(responses.len(), 1);
6326 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
6327 assert_eq!(responses[0].header.channel, 0);
6328 assert_eq!(responses[0].header.corr, 7);
6329 let ack = parse_ack(&responses[0]);
6330 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
6331 assert!(ack
6332 .subc_capabilities
6333 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
6334 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
6335 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
6336 assert!(ack
6337 .subc_ops
6338 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
6339 assert!(ack
6340 .subc_ops
6341 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
6342
6343 let registration = registry.get_module("aft").unwrap().unwrap();
6344 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
6345 assert_eq!(registration.state, ChannelState::Active);
6346 assert_eq!(registration.connection_id, conn);
6347 assert_eq!(registration.control_ops, module_baseline_control_ops());
6348 }
6349
6350 #[test]
6351 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
6352 let invalid_identifiers = [
6353 ("case_change", "credentials-Provider/v1"),
6354 ("leading_zero", "credentials-provider/v01"),
6355 ("trailing_hyphen", "credentials-provider-/v1"),
6356 ("consecutive_hyphens", "credentials--provider/v1"),
6357 ("uppercase", "Credentials-provider/v1"),
6358 ("missing_v", "credentials-provider/1"),
6359 ("whitespace", "credentials provider/v1"),
6360 ("zero_version", "credentials-provider/v0"),
6361 ("out_of_range_version", "credentials-provider/v4294967296"),
6362 (
6363 "overlength_name",
6364 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
6365 ),
6366 ];
6367 let mut cases = invalid_identifiers
6368 .into_iter()
6369 .map(|(name, identifier)| {
6370 (
6371 format!("identifier_{name}"),
6372 "capabilities.provides[0]".to_string(),
6373 identifier.to_string(),
6374 json!({ "provides": [identifier] }),
6375 None,
6376 )
6377 })
6378 .collect::<Vec<_>>();
6379 cases.extend([
6380 (
6381 "unknown_need".to_string(),
6382 "capabilities.requires[0].need".to_string(),
6383 "deferred".to_string(),
6384 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
6385 None,
6386 ),
6387 (
6388 "duplicate_provides".to_string(),
6389 "capabilities.provides[1]".to_string(),
6390 "credentials-provider/v1".to_string(),
6391 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
6392 None,
6393 ),
6394 (
6395 "duplicate_must_never_reach".to_string(),
6396 "capabilities.must_never_reach[1]".to_string(),
6397 "credentials-provider/v1".to_string(),
6398 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
6399 None,
6400 ),
6401 (
6402 "duplicate_requires_same_need".to_string(),
6403 "capabilities.requires[1]".to_string(),
6404 "credentials-provider/v1".to_string(),
6405 json!({ "requires": [
6406 { "capability": "credentials-provider/v1", "need": "required" },
6407 { "capability": "credentials-provider/v1", "need": "required" }
6408 ] }),
6409 None,
6410 ),
6411 (
6412 "duplicate_requires_conflicting_need".to_string(),
6413 "capabilities.requires[1]".to_string(),
6414 "credentials-provider/v1".to_string(),
6415 json!({ "requires": [
6416 { "capability": "credentials-provider/v1", "need": "required" },
6417 { "capability": "credentials-provider/v1", "need": "optional" }
6418 ] }),
6419 None,
6420 ),
6421 (
6422 "capabilities_root_pointer".to_string(),
6423 "runtime_computed[0]".to_string(),
6424 "/capabilities".to_string(),
6425 json!({}),
6426 Some(json!(["/capabilities"])),
6427 ),
6428 (
6429 "capabilities_descendant_pointer".to_string(),
6430 "runtime_computed[0]".to_string(),
6431 "/capabilities/provides".to_string(),
6432 json!({}),
6433 Some(json!(["/capabilities/provides"])),
6434 ),
6435 (
6436 "malformed_pointer_without_leading_slash".to_string(),
6437 "runtime_computed[0]".to_string(),
6438 "capabilities".to_string(),
6439 json!({}),
6440 Some(json!(["capabilities"])),
6441 ),
6442 (
6443 "malformed_pointer_escape".to_string(),
6444 "runtime_computed[0]".to_string(),
6445 "/roles/~2/tools".to_string(),
6446 json!({}),
6447 Some(json!(["/roles/~2/tools"])),
6448 ),
6449 (
6450 "unknown_capabilities_field".to_string(),
6451 "capabilities.future".to_string(),
6452 "<array>".to_string(),
6453 json!({ "future": [] }),
6454 None,
6455 ),
6456 ]);
6457
6458 for (index, (name, field, value, capabilities, runtime_computed)) in
6459 cases.into_iter().enumerate()
6460 {
6461 let registry = Arc::new(Registry::default());
6462 let handler = ControlHandler::new(Arc::clone(®istry));
6463 let response = handler
6464 .handle_control(
6465 ConnectionId::new((index + 1) as u64),
6466 capability_grammar_hello_frame(
6467 capabilities,
6468 runtime_computed,
6469 index as u64 + 1,
6470 ),
6471 )
6472 .expect("invalid HELLO returns a refusal");
6473
6474 assert_eq!(response.len(), 1, "{name} must emit one refusal");
6475 let error = parse_error(&response[0]);
6476 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
6477 let message = error["message"]
6478 .as_str()
6479 .expect("error message is a string");
6480 assert!(
6481 message.contains(&field),
6482 "{name}: field missing from {message}"
6483 );
6484 assert!(
6485 message.contains(&value),
6486 "{name}: value missing from {message}"
6487 );
6488 assert_eq!(
6489 registry
6490 .active_registration_count()
6491 .expect("registry reads"),
6492 0,
6493 "{name}: refused HELLO must not create a catalog entry"
6494 );
6495 }
6496 }
6497
6498 #[test]
6499 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
6500 let registry = Arc::new(Registry::default());
6501 let handler = ControlHandler::new(Arc::clone(®istry));
6502 let capabilities = json!({
6503 "provides": ["credentials-provider/v1"],
6504 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
6505 "must_never_reach": ["federation-transport/v1"]
6506 });
6507 let response = handler
6508 .handle_control(
6509 ConnectionId::new(99),
6510 capability_grammar_hello_frame(
6511 capabilities.clone(),
6512 Some(json!(["/roles/0/tools"])),
6513 99,
6514 ),
6515 )
6516 .expect("valid HELLO registers");
6517 assert_eq!(response[0].header.ty, FrameType::HelloAck);
6518
6519 let request = Frame::build(
6520 FrameType::Request,
6521 control_flags(),
6522 0,
6523 0,
6524 100,
6525 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
6526 .expect("catalog request serializes"),
6527 )
6528 .expect("catalog request frame builds");
6529 let response = handler
6530 .handle_catalog_list(request, None)
6531 .expect("catalog list succeeds");
6532 let ClientControlResponse::CatalogList { modules, .. } =
6533 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
6534 else {
6535 panic!("catalog request must return catalog.list");
6536 };
6537 assert_eq!(modules.len(), 1);
6538 assert_eq!(
6539 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
6540 capabilities
6541 );
6542 }
6543
6544 #[test]
6545 fn catalog_list_mirrors_management_operation_description() {
6546 let registry = Arc::new(Registry::default());
6547 let handler = ControlHandler::new(Arc::clone(®istry));
6548 let description = "List managed records and return their identifiers and metadata.";
6549 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
6550 manifest.provides = vec![ProviderRole::ManagementSurface {
6551 operations: vec![ManagementOperation {
6552 name: "records.list".to_string(),
6553 kind: ManagementOperationKind::Query,
6554 description: Some(description.to_string()),
6555 }],
6556 config_schema: json!({"type": "object"}),
6557 observability: vec![ObservabilitySurface {
6558 name: "records.stats".to_string(),
6559 kind: ObservabilityKind::Snapshot,
6560 }],
6561 identity_scope: vec![IdentityScope::Project],
6562 concurrency: Concurrency::ModuleManaged,
6563 }];
6564 registry
6565 .register_with_control_ops(
6566 manifest,
6567 PROTOCOL_VERSION,
6568 ConnectionId::new(99),
6569 Vec::new(),
6570 )
6571 .expect("described management manifest registers");
6572
6573 let request = Frame::build(
6574 FrameType::Request,
6575 control_flags(),
6576 0,
6577 0,
6578 100,
6579 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
6580 .expect("catalog request serializes"),
6581 )
6582 .expect("catalog request frame builds");
6583 let response = handler
6584 .handle_catalog_list(request, None)
6585 .expect("catalog list succeeds");
6586 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
6587 assert_eq!(
6588 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
6589 "catalog.list must preserve the declared operation description verbatim"
6590 );
6591 }
6592
6593 #[test]
6594 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
6595 let registry = Arc::new(Registry::default());
6596 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
6597 [("vault".to_string(), true), ("squatter".to_string(), true)],
6598 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
6599 );
6600 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
6601 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
6602 provides: vec!["credentials-provider/v1".to_string()],
6603 requires: Vec::new(),
6604 must_never_reach: Vec::new(),
6605 });
6606 let frame = Frame::build(
6607 FrameType::Hello,
6608 control_flags(),
6609 0,
6610 0,
6611 77,
6612 serde_json::to_vec(&ModuleHelloBody {
6613 manifest: squatter,
6614 protocol_ver: PROTOCOL_VERSION,
6615 control_ops: None,
6616 launch_nonce: None,
6617 })
6618 .expect("HELLO serializes"),
6619 )
6620 .expect("HELLO frame builds");
6621 let response = handler
6622 .handle_control(ConnectionId::new(77), frame)
6623 .expect("reserved claim receives a typed refusal");
6624 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
6625 assert_eq!(
6626 registry
6627 .active_registration_count()
6628 .expect("registry reads"),
6629 0,
6630 "a reserved capability refusal must not leave a catalog entry"
6631 );
6632 }
6633
6634 #[test]
6635 fn server_describe_surfaces_required_capability_verdict_fields() {
6636 let registry = Arc::new(Registry::default());
6637 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
6638 [
6639 ("consumer".to_string(), true),
6640 ("provider".to_string(), false),
6641 ],
6642 BTreeMap::new(),
6643 );
6644 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
6645 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
6646 provides: Vec::new(),
6647 requires: vec![subc_protocol::manifest::CapabilityRequirement {
6648 capability: "credentials-provider/v1".to_string(),
6649 need: subc_protocol::manifest::CapabilityNeed::Required,
6650 }],
6651 must_never_reach: Vec::new(),
6652 });
6653 let hello = Frame::build(
6654 FrameType::Hello,
6655 control_flags(),
6656 0,
6657 0,
6658 78,
6659 serde_json::to_vec(&ModuleHelloBody {
6660 manifest: consumer,
6661 protocol_ver: PROTOCOL_VERSION,
6662 control_ops: None,
6663 launch_nonce: None,
6664 })
6665 .expect("HELLO serializes"),
6666 )
6667 .expect("HELLO frame builds");
6668 handler
6669 .handle_control(ConnectionId::new(78), hello)
6670 .expect("consumer registers");
6671 let describe = Frame::build(
6672 FrameType::Request,
6673 control_flags(),
6674 0,
6675 0,
6676 79,
6677 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
6678 .expect("request serializes"),
6679 )
6680 .expect("describe frame builds");
6681 let response = handler
6682 .handle_server_describe(describe)
6683 .expect("server.describe succeeds");
6684 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
6685 let requirement = &rendered["capability_requirements"][0];
6686 assert_eq!(requirement["consumer"], "consumer");
6687 assert_eq!(requirement["verdict"], "never_provided");
6688 assert_eq!(requirement["episode_seq"], 1);
6689 assert_eq!(requirement["config_satisfiable"], false);
6690 assert_eq!(requirement["runtime_available"], false);
6691 assert!(requirement["detail"]
6692 .as_str()
6693 .expect("detail string")
6694 .contains("credentials-provider/v1"));
6695 }
6696
6697 #[test]
6698 fn catalog_list_omits_capabilities_for_legacy_manifest() {
6699 let registry = Arc::new(Registry::default());
6700 let handler = ControlHandler::new(Arc::clone(®istry));
6701 let hello = handler
6702 .handle_control(
6703 ConnectionId::new(101),
6704 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
6705 )
6706 .expect("legacy HELLO registers");
6707 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
6708
6709 let request = Frame::build(
6710 FrameType::Request,
6711 control_flags(),
6712 0,
6713 0,
6714 102,
6715 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
6716 .expect("catalog request serializes"),
6717 )
6718 .expect("catalog request frame builds");
6719 let response = handler
6720 .handle_catalog_list(request, None)
6721 .expect("catalog list succeeds");
6722 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
6723 assert!(
6724 body["modules"][0].get("capabilities").is_none(),
6725 "legacy manifest must retain an absent capabilities field on catalog.list"
6726 );
6727 }
6728
6729 #[test]
6730 fn hello_ack_omits_storage_when_no_storage_config() {
6731 let registry = Arc::new(Registry::default());
6732 let handler = ControlHandler::new(Arc::clone(®istry));
6733 let responses = handler
6734 .handle_control(
6735 ConnectionId::new(1),
6736 hello_frame("aft", PROTOCOL_VERSION, 7),
6737 )
6738 .unwrap();
6739 let ack = parse_ack(&responses[0]);
6740 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
6741 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
6742 }
6743
6744 #[tokio::test]
6745 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
6746 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
6747 let registry = Arc::new(Registry::default());
6748 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
6749 let responses = handler
6750 .handle_control(
6751 ConnectionId::new(1),
6752 hello_frame("aft", PROTOCOL_VERSION, 7),
6753 )
6754 .unwrap();
6755 let ack = parse_ack(&responses[0]);
6756 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
6757
6758 let described = handler
6759 .handle_control_frame(
6760 &route_ctx(ConnectionId::new(2)).0,
6761 Frame::build(
6762 FrameType::Request,
6763 control_flags(),
6764 0,
6765 0,
6766 9,
6767 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
6768 )
6769 .unwrap(),
6770 )
6771 .await
6772 .unwrap();
6773 let ClientControlResponse::ServerDescribe { machine_id, .. } =
6774 serde_json::from_slice(&described[0].body).unwrap()
6775 else {
6776 panic!("server.describe answered with another shape");
6777 };
6778 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
6779 }
6780
6781 #[test]
6782 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
6783 let registry = Arc::new(Registry::default());
6786 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
6787 crate::daemon_config::StorageConfig::Sqlite {
6788 data_home: std::path::PathBuf::from("/data"),
6789 },
6790 ));
6791
6792 let responses = handler
6793 .handle_control(
6794 ConnectionId::new(1),
6795 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
6796 )
6797 .unwrap();
6798 let ack = parse_ack(&responses[0]);
6799 assert_eq!(
6800 ack.storage,
6801 Some(serde_json::json!({
6802 "module_id": "alfonso-routing",
6803 "storage_namespace": "default",
6804 "isolation": { "kind": "module" },
6805 "backend": {
6806 "backend": "sqlite",
6807 "path": "/data/cortexkit/alfonso-routing/store.db"
6808 }
6809 })),
6810 "the delivered descriptor is the module's own sqlite store path"
6811 );
6812 }
6813
6814 #[test]
6815 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
6816 let registry = Arc::new(Registry::default());
6817 let handler = ControlHandler::new(Arc::clone(®istry));
6818 let conn = ConnectionId::new(1);
6819 let responses = handler
6820 .handle_control(
6821 conn,
6822 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
6823 )
6824 .unwrap();
6825 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
6826 let registration = registry.get_module("aft").unwrap().unwrap();
6827 assert_eq!(registration.control_ops, module_baseline_control_ops());
6828
6829 let frame =
6830 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
6831 assert!(handler
6832 .guard_module_control_op(&frame, "aft", "route.bind")
6833 .unwrap()
6834 .is_none());
6835 let error = handler
6836 .guard_module_control_op(&frame, "aft", "test.synthetic")
6837 .unwrap()
6838 .expect("synthetic ungranted op should be rejected");
6839 assert_eq!(error.header.ty, FrameType::Error);
6840 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
6841 }
6842
6843 #[test]
6844 fn hello_control_ops_some_adds_optional_grants() {
6845 let registry = Arc::new(Registry::default());
6846 let handler = ControlHandler::new(Arc::clone(®istry));
6847 handler
6848 .handle_control(
6849 ConnectionId::new(1),
6850 hello_frame_with_control_ops(
6851 "aft",
6852 PROTOCOL_VERSION,
6853 7,
6854 Some(vec![
6855 "future.synthetic".to_string(),
6856 "route.bind".to_string(),
6857 ]),
6858 ),
6859 )
6860 .unwrap();
6861 let registration = registry.get_module("aft").unwrap().unwrap();
6862 assert_eq!(
6863 registration.control_ops,
6864 vec![
6865 "route.bind".to_string(),
6866 "route.status".to_string(),
6867 "future.synthetic".to_string(),
6868 ]
6869 );
6870 let frame =
6871 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
6872 assert!(handler
6873 .guard_module_control_op(&frame, "aft", "future.synthetic")
6874 .unwrap()
6875 .is_none());
6876 }
6877
6878 #[tokio::test]
6879 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
6880 let registry = Arc::new(Registry::default());
6881 let forwarding = Arc::new(ForwardingTable::default());
6882 let handler =
6883 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
6884 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
6885 hello_via_sink(
6886 &handler,
6887 &module_ctx,
6888 &mut module_rx,
6889 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
6890 )
6891 .await;
6892
6893 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
6894 let responses = handler
6895 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
6896 .await
6897 .unwrap();
6898 assert_eq!(responses.len(), 1);
6899 assert_eq!(responses[0].header.ty, FrameType::Error);
6900 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
6901 assert!(module_rx.try_recv().is_err());
6902 }
6903
6904 #[tokio::test]
6905 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
6906 let registry = Arc::new(Registry::default());
6907 let forwarding = Arc::new(ForwardingTable::default());
6908 let handler =
6909 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
6910 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
6911 hello_via_sink(
6912 &handler,
6913 &module_ctx,
6914 &mut module_rx,
6915 hello_frame_with_control_ops(
6916 "aft",
6917 PROTOCOL_VERSION,
6918 7,
6919 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
6920 ),
6921 )
6922 .await;
6923
6924 let project_root = unique_project_root("demux");
6925 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
6926 let route_handler = handler.clone();
6927 let route_task = tokio::spawn(async move {
6928 route_handler
6929 .handle_control_frame(
6930 &route_client_ctx,
6931 route_open_frame(100, "aft", project_root),
6932 )
6933 .await
6934 .unwrap()
6935 });
6936 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
6937 .await
6938 .unwrap()
6939 .unwrap();
6940 assert!(matches!(
6941 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
6942 ModuleControlRequest::RouteBind { .. }
6943 ));
6944
6945 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
6946 let health_handler = handler.clone();
6947 let health_task = tokio::spawn(async move {
6948 health_handler
6949 .handle_control_frame(
6950 &health_client_ctx,
6951 supervisor_health_probe_frame(101, "aft"),
6952 )
6953 .await
6954 .unwrap()
6955 });
6956 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
6957 .await
6958 .unwrap()
6959 .unwrap();
6960 assert_eq!(
6961 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
6962 ModuleControlRequest::HealthCheck {}
6963 );
6964
6965 handler
6966 .handle_control_frame(
6967 &module_ctx,
6968 health_response(health_frame.header.corr, HealthStatus::Degraded),
6969 )
6970 .await
6971 .unwrap();
6972 let health_response = health_task.await.unwrap();
6973 assert_eq!(health_response.len(), 1);
6974 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
6975 ClientControlResponse::SupervisorHealthProbe {
6976 module_id,
6977 status,
6978 detail,
6979 metrics,
6980 } => {
6981 assert_eq!(module_id, "aft");
6982 assert_eq!(status, HealthStatus::Degraded);
6983 assert_eq!(detail.as_deref(), Some("warming"));
6984 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
6985 }
6986 other => panic!("unexpected health response: {other:?}"),
6987 }
6988
6989 handler
6990 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
6991 .await
6992 .unwrap();
6993 let route_response = route_task.await.unwrap();
6994 assert!(route_response.is_empty());
6995 let published = route_client_rx.recv().await.unwrap();
6996 assert!(matches!(
6997 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
6998 ClientControlResponse::RouteOpen { .. }
6999 ));
7000 }
7001
7002 async fn relay_route_open(
7007 handler: &ControlHandler,
7008 client_connection: ConnectionId,
7009 client_egress: &FrameSink,
7010 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
7011 corr: u64,
7012 module_id: &str,
7013 project_root_label: &str,
7014 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
7015 let ctx = RouteCtx {
7016 connection_id: client_connection,
7017 egress: client_egress.clone(),
7018 };
7019 let handler = handler.clone();
7020 let project_root = unique_project_root(project_root_label);
7021 let module_id = module_id.to_string();
7022 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
7023 let task = tokio::spawn(async move {
7024 let _guard = tracing::dispatcher::set_default(&dispatch);
7025 handler
7026 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
7027 .await
7028 .unwrap()
7029 });
7030 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
7031 .await
7032 .expect("module receives the relayed route.bind")
7033 .expect("module egress is open");
7034 (task, bind.frame)
7035 }
7036
7037 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
7038 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
7039 ModuleControlRequest::RouteBind {
7040 route_channel,
7041 epoch,
7042 ..
7043 } => (route_channel, epoch),
7044 other => panic!("expected a route.bind request, got {other:?}"),
7045 }
7046 }
7047
7048 fn published_route(frame: &Frame) -> (u16, u32) {
7049 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
7050 ClientControlResponse::RouteOpen {
7051 route_channel,
7052 route_epoch,
7053 } => (route_channel, route_epoch),
7054 other => panic!("expected a route.open response, got {other:?}"),
7055 }
7056 }
7057
7058 #[tokio::test]
7078 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
7079 let registry = Arc::new(Registry::default());
7080 let forwarding = Arc::new(ForwardingTable::default());
7081 let handler =
7082 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
7083
7084 let module_connection = ConnectionId::new(30);
7085 let (module_ctx, mut module_rx) = route_ctx(module_connection);
7086 hello_via_sink(
7087 &handler,
7088 &module_ctx,
7089 &mut module_rx,
7090 hello_frame("aft", PROTOCOL_VERSION, 7),
7091 )
7092 .await;
7093
7094 let dying_client = ConnectionId::new(31);
7095 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
7096
7097 let (first_task, first_bind) = relay_route_open(
7100 &handler,
7101 dying_client,
7102 &dying_ctx.egress,
7103 &mut module_rx,
7104 100,
7105 "aft",
7106 "closing-first",
7107 )
7108 .await;
7109 handler
7110 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
7111 .await
7112 .unwrap();
7113 assert!(first_task.await.unwrap().is_empty());
7114 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
7115
7116 let (second_task, second_bind) = relay_route_open(
7118 &handler,
7119 dying_client,
7120 &dying_ctx.egress,
7121 &mut module_rx,
7122 101,
7123 "aft",
7124 "closing-second",
7125 )
7126 .await;
7127 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
7128
7129 assert!(forwarding
7132 .escalate_client_delivery_failure(
7133 dying_client,
7134 first_channel,
7135 first_epoch,
7136 CloseReason::new(
7137 "module_to_client_delivery_failed",
7138 "client egress refused a module frame",
7139 ),
7140 crate::forwarding::UndeliveredFrame {
7141 module_id: None,
7142 sink: &dying_ctx.egress,
7143 },
7144 )
7145 .unwrap());
7146 assert!(!dying_ctx.egress.is_closed());
7147
7148 let ack = handler
7150 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
7151 .await;
7152 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
7153 if module_loop_error.is_some() {
7154 handler.cleanup_connection(module_connection).unwrap();
7158 }
7159 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7163 .await
7164 .ok()
7165 .flatten();
7166
7167 assert!(
7169 registry
7170 .get_module_by_connection(module_connection)
7171 .unwrap()
7172 .is_some(),
7173 "one client's closing connection ended the shared module connection: \
7174 {module_loop_error:?}"
7175 );
7176 let cotenant = ConnectionId::new(32);
7178 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
7179 let (cotenant_task, cotenant_bind) = relay_route_open(
7180 &handler,
7181 cotenant,
7182 &cotenant_ctx.egress,
7183 &mut module_rx,
7184 102,
7185 "aft",
7186 "closing-cotenant",
7187 )
7188 .await;
7189 handler
7190 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
7191 .await
7192 .unwrap();
7193 assert!(cotenant_task.await.unwrap().is_empty());
7194 let (cotenant_channel, cotenant_epoch) =
7195 published_route(&cotenant_rx.recv().await.unwrap());
7196 assert!(matches!(
7197 forwarding
7198 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
7199 .unwrap(),
7200 DataRoute::Client(DataRouteState::Bound(_))
7201 ));
7202
7203 let goodbye = post_ack_module_frame
7206 .expect("module receives a GOODBYE for the abandoned route channel");
7207 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
7208 assert_eq!(goodbye.header.channel, abandoned_channel);
7209 assert_eq!(goodbye.header.epoch, abandoned_epoch);
7210
7211 assert!(dying_rx.try_recv().is_err());
7215 let second_response = second_task.await.unwrap();
7216 assert_eq!(second_response.len(), 1);
7217 assert_eq!(
7218 parse_error(&second_response[0])["code"],
7219 "target_unavailable"
7220 );
7221 }
7222
7223 #[test]
7230 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
7231 let handler = ControlHandler::default();
7232 let module_connection = ConnectionId::new(30);
7233 let client_connection = ConnectionId::new(31);
7234
7235 handler
7236 .refuse_to_end_module_connection_for_a_client(
7237 module_connection,
7238 77,
7239 ForwardingError::ConnectionClosing {
7240 connection_id: client_connection,
7241 },
7242 )
7243 .expect("a closing client must never end the module connection");
7244
7245 assert!(matches!(
7246 handler.refuse_to_end_module_connection_for_a_client(
7247 module_connection,
7248 78,
7249 ForwardingError::ConnectionClosing {
7250 connection_id: module_connection,
7251 },
7252 ),
7253 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
7254 connection_id
7255 })) if connection_id == module_connection
7256 ));
7257 assert!(matches!(
7258 handler.refuse_to_end_module_connection_for_a_client(
7259 module_connection,
7260 79,
7261 ForwardingError::Poisoned,
7262 ),
7263 Err(RouterError::Forwarding(ForwardingError::Poisoned))
7264 ));
7265 assert!(matches!(
7266 handler.refuse_to_end_module_connection_for_a_client(
7267 module_connection,
7268 80,
7269 ForwardingError::StaleModuleEndpoint,
7270 ),
7271 Err(RouterError::Forwarding(
7272 ForwardingError::StaleModuleEndpoint
7273 ))
7274 ));
7275 }
7276
7277 #[tokio::test]
7290 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
7291 let registry = Arc::new(Registry::default());
7292 let forwarding = Arc::new(ForwardingTable::default());
7293 let supervisor = SupervisorHandle::new();
7294 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7295 let handler =
7296 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
7297 .with_supervisor(supervisor);
7298
7299 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
7300 hello_via_sink(
7301 &handler,
7302 &target_ctx,
7303 &mut target_rx,
7304 hello_frame("target", PROTOCOL_VERSION, 1),
7305 )
7306 .await;
7307
7308 let wrong_nonce = handler
7312 .handle_control_frame(
7313 &route_ctx(ConnectionId::new(91)).0,
7314 route_open_frame_with_admission_facts(
7315 20,
7316 "target",
7317 unique_project_root("admission-facts"),
7318 Some(subc_control::ConsumerIdentity {
7319 module_id: "fed".to_string(),
7320 launch_nonce: "not-the-real-nonce".to_string(),
7321 }),
7322 None,
7323 ),
7324 )
7325 .await
7326 .unwrap();
7327 assert_eq!(
7328 parse_error(&wrong_nonce[0])["code"],
7329 "bad_consumer_identity",
7330 "a mismatched launch nonce must be refused, not stamped Reserved"
7331 );
7332
7333 let never_spawned = handler
7337 .handle_control_frame(
7338 &route_ctx(ConnectionId::new(92)).0,
7339 route_open_frame_with_admission_facts(
7340 21,
7341 "target",
7342 unique_project_root("admission-facts"),
7343 Some(subc_control::ConsumerIdentity {
7344 module_id: "never-spawned".to_string(),
7345 launch_nonce: "any-nonce".to_string(),
7346 }),
7347 None,
7348 ),
7349 )
7350 .await
7351 .unwrap();
7352 assert_eq!(
7353 parse_error(&never_spawned[0])["code"],
7354 "bad_consumer_identity",
7355 "an unspawned module_id must be refused rather than accepted for lack of a record"
7356 );
7357 }
7358
7359 #[tokio::test]
7382 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
7383 let registry = Arc::new(Registry::default());
7384 let forwarding = Arc::new(ForwardingTable::default());
7385 let supervisor = SupervisorHandle::new();
7386 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7387 let handler =
7388 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
7389 .with_supervisor(supervisor);
7390
7391 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
7392 hello_via_sink(
7393 &handler,
7394 &target_ctx,
7395 &mut target_rx,
7396 hello_frame("target", PROTOCOL_VERSION, 1),
7397 )
7398 .await;
7399
7400 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
7401 let route_handler = handler.clone();
7402 let route_task = tokio::spawn(async move {
7403 route_handler
7404 .handle_control_frame(
7405 &client_ctx,
7406 route_open_frame_with_admission_facts(
7407 30,
7408 "target",
7409 unique_project_root("admission-facts"),
7410 Some(subc_control::ConsumerIdentity {
7411 module_id: "fed".to_string(),
7412 launch_nonce: "fed-nonce".to_string(),
7413 }),
7414 None,
7415 ),
7416 )
7417 .await
7418 .unwrap()
7419 });
7420
7421 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7428 .await
7429 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
7430 .expect("module control channel closed before route.bind");
7431 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
7432 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
7433 panic!("expected route.bind")
7434 };
7435 assert_eq!(
7436 principal,
7437 Some(Principal::Reserved {
7438 module_id: "fed".to_string()
7439 }),
7440 "a correctly attested consumer must be stamped Reserved for its own id"
7441 );
7442
7443 handler
7444 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
7445 .await
7446 .unwrap();
7447 assert!(route_task.await.unwrap().is_empty());
7448 assert!(
7449 matches!(
7450 serde_json::from_slice::<ClientControlResponse>(
7451 &client_rx.recv().await.unwrap().body
7452 )
7453 .unwrap(),
7454 ClientControlResponse::RouteOpen { .. }
7455 ),
7456 "the route must actually open, not merely avoid an error"
7457 );
7458 }
7459
7460 #[tokio::test(start_paused = true)]
7461 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
7462 let registry = Arc::new(Registry::default());
7463 let forwarding = Arc::new(ForwardingTable::default());
7464 let supervisor = SupervisorHandle::new();
7465 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7466 let handler =
7467 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
7468 .with_supervisor(supervisor);
7469
7470 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
7471 hello_via_sink(
7472 &handler,
7473 &target_ctx,
7474 &mut target_rx,
7475 hello_frame("target", PROTOCOL_VERSION, 1),
7476 )
7477 .await;
7478
7479 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
7480 let direct_handler = handler.clone();
7481 let direct_open = tokio::spawn(async move {
7482 direct_handler
7483 .handle_control_frame(
7484 &direct_ctx,
7485 route_open_frame(2, "target", unique_project_root("route-census-direct")),
7486 )
7487 .await
7488 .unwrap()
7489 });
7490 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7491 .await
7492 .expect("no direct route.bind within 5s")
7493 .expect("target control channel closed before direct route.bind");
7494 handler
7495 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
7496 .await
7497 .unwrap();
7498 assert!(direct_open.await.unwrap().is_empty());
7499 let _ = direct_rx.recv().await.unwrap();
7500
7501 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
7502 let reserved_handler = handler.clone();
7503 let reserved_open = tokio::spawn(async move {
7504 reserved_handler
7505 .handle_control_frame(
7506 &reserved_ctx,
7507 route_open_frame_with_admission_facts(
7508 3,
7509 "target",
7510 unique_project_root("admission-facts"),
7511 Some(ConsumerIdentity {
7512 module_id: "fed".to_string(),
7513 launch_nonce: "fed-nonce".to_string(),
7514 }),
7515 None,
7516 ),
7517 )
7518 .await
7519 .unwrap()
7520 });
7521 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7522 .await
7523 .expect("no reserved route.bind within 5s")
7524 .expect("target control channel closed before reserved route.bind");
7525 handler
7526 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
7527 .await
7528 .unwrap();
7529 assert!(reserved_open.await.unwrap().is_empty());
7530 let _ = reserved_rx.recv().await.unwrap();
7531
7532 forwarding
7533 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
7534 .unwrap();
7535 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
7536 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
7537 module_id: Some("target".to_string()),
7538 })
7539 .unwrap();
7540 let census_frame =
7541 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
7542 let response = handler
7543 .handle_control_frame(&census_ctx, census_frame)
7544 .await
7545 .unwrap()
7546 .pop()
7547 .unwrap();
7548 let actual: Value = serde_json::from_slice(&response.body).unwrap();
7549 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
7550 assert!(matches!(
7551 decoded,
7552 ClientControlResponse::SupervisorRoutes { .. }
7553 ));
7554 let routes = actual["modules"][0]["routes"].as_array().unwrap();
7555 assert_eq!(routes.len(), 2);
7556 assert!(routes.iter().all(|route| route["draining"] == true));
7557 assert!(
7560 routes.iter().all(|route| route["drain_reason"] == "reload"),
7561 "draining routes must name the drain's reason: {routes:?}"
7562 );
7563 assert!(routes.iter().any(|route| {
7564 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
7565 }));
7566 assert!(routes.iter().any(|route| {
7567 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
7568 }));
7569
7570 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
7571 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
7572 if std::env::var_os("UPDATE_GOLDEN").is_some() {
7573 std::fs::write(
7574 &golden_path,
7575 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
7576 )
7577 .unwrap();
7578 }
7579 let expected: Value =
7580 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
7581 assert_eq!(actual, expected);
7582 }
7583
7584 async fn query_live_roots(
7585 handler: &ControlHandler,
7586 module_ctx: &RouteCtx,
7587 ) -> ModuleControlResponseToModule {
7588 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
7589 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
7590 let response = handler
7591 .handle_control_frame(module_ctx, frame)
7592 .await
7593 .unwrap()
7594 .pop()
7595 .unwrap();
7596 serde_json::from_slice(&response.body).unwrap()
7597 }
7598
7599 #[tokio::test(start_paused = true)]
7600 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
7601 let registry = Arc::new(Registry::default());
7602 let forwarding = Arc::new(ForwardingTable::default());
7603 let handler = ControlHandler::with_forwarding(registry, forwarding);
7604 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
7605 hello_via_sink(
7606 &handler,
7607 &target_ctx,
7608 &mut target_rx,
7609 hello_frame("target", PROTOCOL_VERSION, 1),
7610 )
7611 .await;
7612 let root = unique_project_root("live-roots-known");
7613 let path = ProjectRootId::from_path_allowing_missing(root.path())
7614 .unwrap()
7615 .as_path()
7616 .to_path_buf();
7617 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
7618 let open_handler = handler.clone();
7619 let opened = tokio::spawn(async move {
7620 open_handler
7621 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
7622 .await
7623 .unwrap()
7624 });
7625 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7626 .await
7627 .unwrap()
7628 .unwrap();
7629 handler
7630 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
7631 .await
7632 .unwrap();
7633 assert!(opened.await.unwrap().is_empty());
7634 let _ = client_rx.recv().await.unwrap();
7635
7636 let root = unique_project_root("live-roots-pending");
7637 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
7638 .unwrap()
7639 .as_path()
7640 .to_path_buf();
7641 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
7642 let open_handler = handler.clone();
7643 let pending = tokio::spawn(async move {
7644 open_handler
7645 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
7646 .await
7647 .unwrap()
7648 });
7649 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7650 .await
7651 .unwrap()
7652 .unwrap();
7653 let actual = query_live_roots(&handler, &target_ctx).await;
7654 let ModuleControlResponseToModule::LiveRoots {
7655 roots,
7656 unknown_root_bindings,
7657 total_bindings,
7658 } = actual
7659 else {
7660 panic!("expected live roots")
7661 };
7662 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
7663 assert_eq!(unknown_root_bindings, 0);
7664 assert_eq!(
7665 roots.len(),
7666 2,
7667 "root-known arm must retain each canonical root"
7668 );
7669 assert_eq!(
7670 total_bindings,
7671 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
7672 );
7673 let counts = roots
7674 .iter()
7675 .map(|root| (root.project_root.clone(), root.bound, root.pending))
7676 .collect::<Vec<_>>();
7677 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
7678 expected.sort_by(|a, b| a.0.cmp(&b.0));
7679 assert_eq!(
7680 counts, expected,
7681 "roots must sort by path and count pending separately"
7682 );
7683 handler
7684 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
7685 .await
7686 .unwrap();
7687 assert!(pending.await.unwrap().is_empty());
7688 }
7689
7690 #[tokio::test(start_paused = true)]
7691 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
7692 let forwarding = Arc::new(ForwardingTable::default());
7693 let handler =
7694 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
7695 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
7696 hello_via_sink(
7697 &handler,
7698 &target_ctx,
7699 &mut target_rx,
7700 hello_frame("target", PROTOCOL_VERSION, 1),
7701 )
7702 .await;
7703 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
7704 let pending = forwarding
7705 .begin_route_bind_relay_for_test(
7706 client_ctx.connection_id,
7707 client_ctx.egress.clone(),
7708 2,
7709 "target",
7710 )
7711 .unwrap();
7712 forwarding
7713 .complete_pending_relay(
7714 target_ctx.connection_id,
7715 pending.corr,
7716 RouteBindRelayOutcome::Accepted,
7717 )
7718 .unwrap();
7719 let actual = query_live_roots(&handler, &target_ctx).await;
7720 let ModuleControlResponseToModule::LiveRoots {
7721 roots,
7722 unknown_root_bindings,
7723 total_bindings,
7724 } = actual
7725 else {
7726 panic!("expected live roots")
7727 };
7728 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
7729 assert_eq!(
7730 unknown_root_bindings, 1,
7731 "unknown-root arm must not read as no bindings"
7732 );
7733 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
7734 assert_eq!(
7735 total_bindings,
7736 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
7737 );
7738 }
7739
7740 #[tokio::test(start_paused = true)]
7748 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
7749 let forwarding = Arc::new(ForwardingTable::default());
7750 let handler =
7751 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
7752 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
7753 let replies = handler
7754 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
7755 .await
7756 .unwrap();
7757 let queued_by_hello = module_rx.len();
7758
7759 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
7760 let open_handler = handler.clone();
7761 let open = tokio::spawn(async move {
7762 open_handler
7763 .handle_control_frame(
7764 &client_ctx,
7765 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
7766 )
7767 .await
7768 .unwrap()
7769 });
7770 let mut spins = 0;
7772 while module_rx.len() == queued_by_hello {
7773 spins += 1;
7774 assert!(spins < 10_000, "route.open never queued a route.bind");
7775 tokio::task::yield_now().await;
7776 }
7777
7778 for reply in replies {
7780 module_ctx.egress.send(reply).await.unwrap();
7781 }
7782
7783 let first = module_rx.recv().await.unwrap().frame;
7784 assert_eq!(
7785 first.header.ty,
7786 FrameType::HelloAck,
7787 "the first frame a registering module reads must be its HELLO_ACK"
7788 );
7789 assert_eq!(first.header.corr, 7);
7790 let second = module_rx.recv().await.unwrap().frame;
7791 assert_eq!(second.header.ty, FrameType::Request);
7792 assert!(
7793 matches!(
7794 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
7795 ModuleControlRequest::RouteBind { .. }
7796 ),
7797 "the route.bind follows the ack"
7798 );
7799 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
7800
7801 handler
7802 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
7803 .await
7804 .unwrap();
7805 assert!(open.await.unwrap().is_empty());
7806 let _ = client_rx.recv().await.unwrap();
7807 }
7808
7809 #[tokio::test(start_paused = true)]
7810 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
7811 let handler = ControlHandler::with_forwarding(
7812 Arc::new(Registry::default()),
7813 Arc::new(ForwardingTable::default()),
7814 );
7815 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
7816 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
7817 hello_via_sink(
7818 &handler,
7819 &first_ctx,
7820 &mut first_rx,
7821 hello_frame("first", PROTOCOL_VERSION, 1),
7822 )
7823 .await;
7824 hello_via_sink(
7825 &handler,
7826 &second_ctx,
7827 &mut second_rx,
7828 hello_frame("second", PROTOCOL_VERSION, 2),
7829 )
7830 .await;
7831 let root = unique_project_root("second-only");
7832 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
7833 let cloned = handler.clone();
7834 let open = tokio::spawn(async move {
7835 cloned
7836 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
7837 .await
7838 .unwrap()
7839 });
7840 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
7841 .await
7842 .unwrap()
7843 .unwrap();
7844 let first = query_live_roots(&handler, &first_ctx).await;
7845 let second = query_live_roots(&handler, &second_ctx).await;
7846 assert!(
7847 matches!(
7848 first,
7849 ModuleControlResponseToModule::LiveRoots {
7850 total_bindings: 0,
7851 ..
7852 }
7853 ),
7854 "cross-module scope must not expose another module's roots"
7855 );
7856 assert!(
7857 matches!(
7858 second,
7859 ModuleControlResponseToModule::LiveRoots {
7860 total_bindings: 1,
7861 ..
7862 }
7863 ),
7864 "second module must see its pending route"
7865 );
7866 handler
7867 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
7868 .await
7869 .unwrap();
7870 assert!(open.await.unwrap().is_empty());
7871 }
7872
7873 #[tokio::test(start_paused = true)]
7874 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
7875 let handler = ControlHandler::with_forwarding(
7876 Arc::new(Registry::default()),
7877 Arc::new(ForwardingTable::default()),
7878 );
7879 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
7880 hello_via_sink(
7881 &handler,
7882 &target_ctx,
7883 &mut target_rx,
7884 hello_frame("target", PROTOCOL_VERSION, 1),
7885 )
7886 .await;
7887 let actual = query_live_roots(&handler, &target_ctx).await;
7888 let ModuleControlResponseToModule::LiveRoots {
7889 roots,
7890 unknown_root_bindings,
7891 total_bindings,
7892 } = actual
7893 else {
7894 panic!("expected live roots")
7895 };
7896 assert!(roots.is_empty());
7897 assert_eq!(unknown_root_bindings, 0);
7898 assert_eq!(total_bindings, 0);
7899 assert_eq!(
7900 total_bindings,
7901 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
7902 );
7903 }
7904
7905 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
7912 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
7913 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
7914 let text = std::fs::read_to_string(&path)
7915 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
7916 let vectors: Vec<(String, Value)> = text
7917 .lines()
7918 .filter(|line| !line.trim().is_empty())
7919 .map(|line| {
7920 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
7921 let id = entry["corpus_id"]
7922 .as_str()
7923 .expect("every vector carries a corpus_id")
7924 .to_string();
7925 (id, entry["package"].clone())
7926 })
7927 .collect();
7928 assert_eq!(
7932 vectors.len(),
7933 3,
7934 "vendored fed corpus changed size; re-sync from subc-federation"
7935 );
7936
7937 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
7952 let richest = vectors
7953 .iter()
7954 .filter_map(|(_, package)| package.as_object())
7955 .map(|object| {
7956 object
7957 .keys()
7958 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
7959 .count()
7960 })
7961 .max()
7962 .unwrap_or(0);
7963 assert!(
7964 richest >= 2,
7965 "vendored corpus no longer carries a package with unmodelled fields, \
7966 so the relay test can no longer distinguish a verbatim relay from a lossy one"
7967 );
7968
7969 vectors
7970 }
7971
7972 #[tokio::test]
7980 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
7981 for (corpus_id, package) in fed_admission_facts_vectors() {
7982 let registry = Arc::new(Registry::default());
7983 let forwarding = Arc::new(ForwardingTable::default());
7984 let supervisor = SupervisorHandle::new();
7985 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7986 let handler =
7987 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
7988 .with_supervisor(supervisor)
7989 .with_admission_facts_config(
7990 Some("fed".to_string()),
7991 Some(vec!["target".to_string()]),
7992 );
7993
7994 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
7995 hello_via_sink(
7996 &handler,
7997 &target_ctx,
7998 &mut target_rx,
7999 hello_frame("target", PROTOCOL_VERSION, 1),
8000 )
8001 .await;
8002
8003 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
8004 let route_handler = handler.clone();
8005 let expected = package.clone();
8006 let route_task = tokio::spawn(async move {
8007 route_handler
8008 .handle_control_frame(
8009 &client_ctx,
8010 route_open_frame_with_admission_facts(
8011 20,
8012 "target",
8013 unique_project_root("admission-facts"),
8014 Some(subc_control::ConsumerIdentity {
8015 module_id: "fed".to_string(),
8016 launch_nonce: "fed-nonce".to_string(),
8017 }),
8018 Some(package),
8019 ),
8020 )
8021 .await
8022 .unwrap()
8023 });
8024
8025 let bind_frame = target_rx.recv().await.unwrap();
8026 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8027 let ModuleControlRequest::RouteBind {
8028 admission_facts, ..
8029 } = bind
8030 else {
8031 panic!("{corpus_id}: expected route.bind")
8032 };
8033 assert_eq!(
8034 admission_facts,
8035 Some(expected),
8036 "{corpus_id}: relay must not add, drop or reshape any field"
8037 );
8038
8039 handler
8040 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8041 .await
8042 .unwrap();
8043 route_task.await.unwrap();
8044 }
8045 }
8046
8047 #[tokio::test]
8048 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
8049 let registry = Arc::new(Registry::default());
8050 let forwarding = Arc::new(ForwardingTable::default());
8051 let supervisor = SupervisorHandle::new();
8052 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8053 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
8054 let handler =
8055 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8056 .with_supervisor(supervisor)
8057 .with_admission_facts_config(
8058 Some("fed".to_string()),
8059 Some(vec!["target".to_string()]),
8060 );
8061
8062 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
8063 hello_via_sink(
8064 &handler,
8065 &target_ctx,
8066 &mut target_rx,
8067 hello_frame("target", PROTOCOL_VERSION, 1),
8068 )
8069 .await;
8070 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
8071 hello_via_sink(
8072 &handler,
8073 &other_ctx,
8074 &mut other_rx,
8075 hello_frame("other", PROTOCOL_VERSION, 2),
8076 )
8077 .await;
8078
8079 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
8080 let expected_facts = facts.clone();
8081 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
8082 let route_handler = handler.clone();
8083 let route_task = tokio::spawn(async move {
8084 route_handler
8085 .handle_control_frame(
8086 &client_ctx,
8087 route_open_frame_with_admission_facts(
8088 10,
8089 "target",
8090 unique_project_root("admission-facts"),
8091 Some(subc_control::ConsumerIdentity {
8092 module_id: "fed".to_string(),
8093 launch_nonce: "fed-nonce".to_string(),
8094 }),
8095 Some(facts.clone()),
8096 ),
8097 )
8098 .await
8099 .unwrap()
8100 });
8101 let bind_frame = target_rx.recv().await.unwrap();
8102 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8103 let ModuleControlRequest::RouteBind {
8104 admission_facts, ..
8105 } = bind
8106 else {
8107 panic!("expected route.bind")
8108 };
8109 assert_eq!(admission_facts, Some(expected_facts));
8110 handler
8111 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8112 .await
8113 .unwrap();
8114 assert!(route_task.await.unwrap().is_empty());
8115 assert!(matches!(
8116 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
8117 .unwrap(),
8118 ClientControlResponse::RouteOpen { .. }
8119 ));
8120
8121 let direct = handler
8122 .handle_control_frame(
8123 &route_ctx(ConnectionId::new(73)).0,
8124 route_open_frame_with_admission_facts(
8125 11,
8126 "target",
8127 unique_project_root("admission-facts"),
8128 None,
8129 Some(json!({"x": 1})),
8130 ),
8131 )
8132 .await
8133 .unwrap();
8134 assert_eq!(
8135 parse_error(&direct[0])["code"],
8136 "admission_facts_not_permitted"
8137 );
8138
8139 let different_reserved = handler
8140 .handle_control_frame(
8141 &route_ctx(ConnectionId::new(77)).0,
8142 route_open_frame_with_admission_facts(
8143 15,
8144 "target",
8145 unique_project_root("admission-facts"),
8146 Some(subc_control::ConsumerIdentity {
8147 module_id: "other".to_string(),
8148 launch_nonce: "other-nonce".to_string(),
8149 }),
8150 Some(json!({"x": 1})),
8151 ),
8152 )
8153 .await
8154 .unwrap();
8155 assert_eq!(
8156 parse_error(&different_reserved[0])["code"],
8157 "admission_facts_not_permitted"
8158 );
8159
8160 let other_target = handler
8161 .handle_control_frame(
8162 &route_ctx(ConnectionId::new(74)).0,
8163 route_open_frame_with_admission_facts(
8164 12,
8165 "other",
8166 unique_project_root("admission-facts"),
8167 Some(subc_control::ConsumerIdentity {
8168 module_id: "fed".to_string(),
8169 launch_nonce: "fed-nonce".to_string(),
8170 }),
8171 Some(json!({"x": 1})),
8172 ),
8173 )
8174 .await
8175 .unwrap();
8176 assert_eq!(
8177 parse_error(&other_target[0])["code"],
8178 "admission_facts_target_not_allowed"
8179 );
8180
8181 let nonexistent = handler
8182 .handle_control_frame(
8183 &route_ctx(ConnectionId::new(75)).0,
8184 route_open_frame_with_admission_facts(
8185 13,
8186 "missing",
8187 unique_project_root("admission-facts"),
8188 None,
8189 Some(json!({"x": 1})),
8190 ),
8191 )
8192 .await
8193 .unwrap();
8194 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
8195
8196 let described = handler
8197 .handle_control_frame(
8198 &route_ctx(ConnectionId::new(76)).0,
8199 Frame::build(
8200 FrameType::Request,
8201 control_flags(),
8202 0,
8203 0,
8204 14,
8205 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
8206 )
8207 .unwrap(),
8208 )
8209 .await
8210 .unwrap();
8211 let ClientControlResponse::ServerDescribe { capabilities, .. } =
8212 serde_json::from_slice(&described[0].body).unwrap()
8213 else {
8214 panic!("expected server.describe response")
8215 };
8216 assert!(capabilities
8217 .iter()
8218 .any(|cap| cap == "admission_facts_relay_v1"));
8219 }
8220
8221 #[tokio::test]
8222 async fn admission_facts_without_configured_carrier_are_rejected() {
8223 let registry = Arc::new(Registry::default());
8224 let forwarding = Arc::new(ForwardingTable::default());
8225 let handler = ControlHandler::with_forwarding(registry, forwarding);
8226 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
8227 hello_via_sink(
8228 &handler,
8229 &target_ctx,
8230 &mut target_rx,
8231 hello_frame("target", PROTOCOL_VERSION, 1),
8232 )
8233 .await;
8234
8235 let responses = handler
8236 .handle_control_frame(
8237 &route_ctx(ConnectionId::new(79)).0,
8238 route_open_frame_with_admission_facts(
8239 16,
8240 "target",
8241 unique_project_root("admission-facts"),
8242 None,
8243 Some(json!({"x": 1})),
8244 ),
8245 )
8246 .await
8247 .unwrap();
8248 assert_eq!(
8249 parse_error(&responses[0])["code"],
8250 "admission_facts_not_permitted"
8251 );
8252 }
8253
8254 #[tokio::test]
8255 async fn route_open_relays_consumer_capabilities_verbatim() {
8256 let registry = Arc::new(Registry::default());
8257 let forwarding = Arc::new(ForwardingTable::default());
8258 let handler =
8259 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8260 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
8261 hello_via_sink(
8262 &handler,
8263 &module_ctx,
8264 &mut module_rx,
8265 hello_frame("aft", PROTOCOL_VERSION, 7),
8266 )
8267 .await;
8268
8269 let expected = vec!["elicitation".to_string(), "roots".to_string()];
8270 let expected_for_request = expected.clone();
8271 let project_root = unique_project_root("consumer-capabilities-present");
8272 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
8273 let route_handler = handler.clone();
8274 let route_task = tokio::spawn(async move {
8275 route_handler
8276 .handle_control_frame(
8277 &client_ctx,
8278 route_open_frame_with_consumer_capabilities(
8279 401,
8280 "aft",
8281 project_root,
8282 Some(expected_for_request),
8283 ),
8284 )
8285 .await
8286 .unwrap()
8287 });
8288 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8289 .await
8290 .unwrap()
8291 .unwrap();
8292 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8293 let ModuleControlRequest::RouteBind {
8294 consumer_capabilities,
8295 ..
8296 } = bind
8297 else {
8298 panic!("expected route.bind request, got {bind:?}");
8299 };
8300 assert_eq!(consumer_capabilities, Some(expected.clone()));
8301
8302 handler
8303 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8304 .await
8305 .unwrap();
8306 let route_response = route_task.await.unwrap();
8307 assert!(route_response.is_empty());
8308 let published = client_rx.recv().await.unwrap();
8309 assert!(matches!(
8310 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8311 ClientControlResponse::RouteOpen { .. }
8312 ));
8313 }
8314
8315 #[tokio::test]
8316 async fn route_open_without_consumer_capabilities_relays_none() {
8317 let registry = Arc::new(Registry::default());
8318 let forwarding = Arc::new(ForwardingTable::default());
8319 let handler =
8320 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8321 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
8322 hello_via_sink(
8323 &handler,
8324 &module_ctx,
8325 &mut module_rx,
8326 hello_frame("aft", PROTOCOL_VERSION, 7),
8327 )
8328 .await;
8329
8330 let project_root = unique_project_root("consumer-capabilities-absent");
8331 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
8332 let route_handler = handler.clone();
8333 let route_task = tokio::spawn(async move {
8334 route_handler
8335 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
8336 .await
8337 .unwrap()
8338 });
8339 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8340 .await
8341 .unwrap()
8342 .unwrap();
8343 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8344 let ModuleControlRequest::RouteBind {
8345 consumer_capabilities,
8346 ..
8347 } = bind
8348 else {
8349 panic!("expected route.bind request, got {bind:?}");
8350 };
8351 assert_eq!(consumer_capabilities, None);
8352
8353 handler
8354 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8355 .await
8356 .unwrap();
8357 let route_response = route_task.await.unwrap();
8358 assert!(route_response.is_empty());
8359 let published = client_rx.recv().await.unwrap();
8360 assert!(matches!(
8361 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8362 ClientControlResponse::RouteOpen { .. }
8363 ));
8364 }
8365
8366 #[tokio::test]
8367 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
8368 let registry = Arc::new(Registry::default());
8369 let forwarding = Arc::new(ForwardingTable::default());
8370 let handler =
8371 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8372 .with_health_probe_timeout(Duration::from_secs(5));
8373 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
8374 hello_via_sink(
8375 &handler,
8376 &module_ctx,
8377 &mut module_rx,
8378 non_routable_hello_frame_with_control_ops(
8379 "mcp",
8380 300,
8381 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
8382 ),
8383 )
8384 .await;
8385 assert!(registry
8386 .get_module("mcp")
8387 .unwrap()
8388 .unwrap()
8389 .manifest
8390 .provides
8391 .is_empty());
8392
8393 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
8394 let route_response = handler
8395 .handle_control_frame(
8396 &route_client_ctx,
8397 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
8398 )
8399 .await
8400 .unwrap();
8401 assert_eq!(route_response[0].header.ty, FrameType::Error);
8402 assert_eq!(
8403 parse_error(&route_response[0])["code"],
8404 "target_unavailable"
8405 );
8406 assert!(parse_error(&route_response[0])["message"]
8407 .as_str()
8408 .unwrap()
8409 .contains("does not provide the requested target"));
8410 assert!(module_rx.try_recv().is_err());
8411
8412 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
8413 let health_handler = handler.clone();
8414 let health_task = tokio::spawn(async move {
8415 health_handler
8416 .handle_control_frame(
8417 &health_client_ctx,
8418 supervisor_health_probe_frame(302, "mcp"),
8419 )
8420 .await
8421 .unwrap()
8422 });
8423 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8424 .await
8425 .unwrap()
8426 .unwrap();
8427 assert_eq!(
8428 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
8429 ModuleControlRequest::HealthCheck {}
8430 );
8431 handler
8432 .handle_control_frame(
8433 &module_ctx,
8434 health_response(health_frame.header.corr, HealthStatus::Ok),
8435 )
8436 .await
8437 .unwrap();
8438 let health_response = health_task.await.unwrap();
8439 assert_eq!(health_response[0].header.ty, FrameType::Response);
8440 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
8441 ClientControlResponse::SupervisorHealthProbe {
8442 module_id, status, ..
8443 } => {
8444 assert_eq!(module_id, "mcp");
8445 assert_eq!(status, HealthStatus::Ok);
8446 }
8447 other => panic!("unexpected health response: {other:?}"),
8448 }
8449
8450 forwarding
8455 .cleanup_connection(module_ctx.connection_id)
8456 .unwrap();
8457 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
8458 let cleanup_response = tokio::time::timeout(
8459 Duration::from_millis(200),
8460 handler.handle_control_frame(
8461 &cleanup_probe_ctx,
8462 supervisor_health_probe_frame(303, "mcp"),
8463 ),
8464 )
8465 .await
8466 .expect("probe should fail immediately when the control lane is gone")
8467 .unwrap();
8468 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
8469 assert_eq!(
8470 parse_error(&cleanup_response[0])["code"],
8471 "target_unavailable"
8472 );
8473 assert!(parse_error(&cleanup_response[0])["message"]
8474 .as_str()
8475 .unwrap()
8476 .contains("no module connection"));
8477
8478 handler
8479 .cleanup_connection(module_ctx.connection_id)
8480 .unwrap();
8481 }
8482
8483 #[tokio::test]
8484 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
8485 let registry = Arc::new(Registry::default());
8486 let supervisor_handle = SupervisorHandle::new();
8487 let supervisor =
8488 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
8489 .with_handle(supervisor_handle.clone())
8490 .with_connection_file_path(
8491 std::env::temp_dir()
8492 .join(format!("subc-route-open-warming-{}", std::process::id())),
8493 );
8494 let module = supervisor
8495 .supervise_configured(
8496 ModuleSpec {
8497 module_id: "warming".to_string(),
8498 program: fake_aft_stub_path(),
8499 args: Vec::new(),
8500 env: Vec::new(),
8501 reserved: false,
8502 reserved_prefixes: Vec::new(),
8503 protocol: ModuleProtocol::Subc,
8504 overlap: Default::default(),
8505 },
8506 true,
8507 )
8508 .unwrap();
8509 assert_eq!(module.state().unwrap(), ModuleState::Running);
8510
8511 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8512 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
8513 let response = handler
8514 .handle_control_frame(
8515 &ctx,
8516 route_open_frame(304, "warming", unique_project_root("warming")),
8517 )
8518 .await
8519 .unwrap();
8520 module.stop().await.unwrap();
8521
8522 assert_eq!(response[0].header.ty, FrameType::Error);
8523 let error = parse_error(&response[0]);
8524 assert_eq!(error["code"], "module_warming");
8525 assert!(error["message"]
8526 .as_str()
8527 .unwrap()
8528 .contains("state=running, enabled=true, live=false"));
8529 }
8530
8531 #[tokio::test]
8536 async fn route_open_refusal_names_the_check_that_refused() {
8537 let handler = ControlHandler::new(Arc::new(Registry::default()));
8538 let capture = EventCapture::default();
8539 let _subscriber =
8540 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8541 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
8542 let response = handler
8543 .handle_control_frame(
8544 &ctx,
8545 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
8546 )
8547 .await
8548 .unwrap();
8549
8550 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
8551 let event = capture
8552 .events()
8553 .into_iter()
8554 .find(|event| {
8555 event.target == "control"
8556 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
8557 })
8558 .expect("route.open refusal event");
8559 assert_eq!(
8560 event.fields.get("reason"),
8561 Some(&"\"not_registered\"".to_string())
8562 );
8563 }
8564
8565 #[tokio::test]
8566 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
8567 let registry = Arc::new(Registry::default());
8568 let supervisor_handle = SupervisorHandle::new();
8569 let supervisor =
8570 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
8571 .with_handle(supervisor_handle.clone())
8572 .with_connection_file_path(std::env::temp_dir().join(format!(
8573 "subc-route-open-refusal-info-{}",
8574 std::process::id()
8575 )));
8576 let module = supervisor
8577 .supervise_configured(
8578 ModuleSpec {
8579 module_id: "warming".to_string(),
8580 program: fake_aft_stub_path(),
8581 args: Vec::new(),
8582 env: Vec::new(),
8583 reserved: false,
8584 reserved_prefixes: Vec::new(),
8585 protocol: ModuleProtocol::Subc,
8586 overlap: Default::default(),
8587 },
8588 true,
8589 )
8590 .unwrap();
8591 assert_eq!(module.state().unwrap(), ModuleState::Running);
8592
8593 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8594 assert!(handler
8595 .counters()
8596 .snapshot()
8597 .get("route_open_refused_by_code")
8598 .is_none());
8599 let capture = EventCapture::default();
8600 let _subscriber =
8601 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8602 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
8603 let response = handler
8604 .handle_control_frame(
8605 &ctx,
8606 route_open_frame(394, "warming", unique_project_root("refusal-info")),
8607 )
8608 .await
8609 .unwrap();
8610 module.stop().await.unwrap();
8611
8612 assert_eq!(parse_error(&response[0])["code"], "module_warming");
8613 let event = capture
8614 .events()
8615 .into_iter()
8616 .find(|event| {
8617 event.target == "control"
8618 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
8619 })
8620 .expect("route.open refusal event");
8621 assert_eq!(
8622 event.fields.get("module_id"),
8623 Some(&"\"warming\"".to_string())
8624 );
8625 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
8626 assert_eq!(
8627 event.fields.get("reason"),
8628 Some(&"\"supervised_not_registered\"".to_string())
8629 );
8630 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
8631 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
8632 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
8633 assert_eq!(
8634 handler.counters().snapshot()["route_open_refused_by_code"],
8635 json!({ "module_warming": 1 })
8636 );
8637 }
8638
8639 #[tokio::test(flavor = "current_thread")]
8640 async fn route_open_unknown_module_escapes_target_module_id() {
8641 let handler = ControlHandler::new(Arc::new(Registry::default()));
8642 let capture = EventCapture::default();
8643 let _subscriber =
8644 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8645 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
8646 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
8647 let response = handler
8648 .handle_control_frame(
8649 &ctx,
8650 route_open_frame(
8651 395,
8652 hostile_module_id,
8653 unique_project_root("hostile-target-module-id"),
8654 ),
8655 )
8656 .await
8657 .unwrap();
8658
8659 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
8660 let event = capture
8661 .events()
8662 .into_iter()
8663 .find(|event| {
8664 event.target == "control"
8665 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
8666 })
8667 .expect("route.open unknown-module refusal event");
8668 let logged = event.fields.get("module_id").expect("module_id field");
8669 assert!(!logged.bytes().any(|byte| byte < 0x20));
8670 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
8671 }
8672
8673 #[tokio::test(flavor = "current_thread")]
8674 async fn route_open_module_rejection_uses_daemon_counter_key() {
8675 let registry = Arc::new(Registry::default());
8676 let forwarding = Arc::new(ForwardingTable::default());
8677 let handler =
8678 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8679 let module_connection = ConnectionId::new(95);
8680 let (module_ctx, mut module_rx) = route_ctx(module_connection);
8681 hello_via_sink(
8682 &handler,
8683 &module_ctx,
8684 &mut module_rx,
8685 hello_frame("aft", PROTOCOL_VERSION, 395),
8686 )
8687 .await;
8688
8689 let client_connection = ConnectionId::new(96);
8690 let (client_ctx, _client_rx) = route_ctx(client_connection);
8691 let capture = EventCapture::default();
8692 let _subscriber =
8693 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8694 let (route_task, bind) = relay_route_open(
8695 &handler,
8696 client_connection,
8697 &client_ctx.egress,
8698 &mut module_rx,
8699 396,
8700 "aft",
8701 "hostile-module-code",
8702 )
8703 .await;
8704 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
8705 let rejection = Frame::build(
8706 FrameType::Error,
8707 control_flags(),
8708 0,
8709 0,
8710 bind.header.corr,
8711 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
8712 )
8713 .unwrap();
8714 handler
8715 .handle_control_frame(&module_ctx, rejection)
8716 .await
8717 .unwrap();
8718
8719 let response = route_task.await.unwrap();
8720 assert_eq!(parse_error(&response[0])["code"], hostile_code);
8721 let counters = handler.counters().snapshot();
8722 assert_eq!(
8723 counters["route_open_refused_by_code"],
8724 json!({ "module_rejected": 1 })
8725 );
8726 assert!(counters["route_open_refused_by_code"]
8727 .get(hostile_code)
8728 .is_none());
8729
8730 let event = capture
8731 .events()
8732 .into_iter()
8733 .find(|event| {
8734 event.target == "control"
8735 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
8736 })
8737 .expect("route.open module-rejection refusal event");
8738 let logged = event.fields.get("module_code").expect("module_code field");
8739 assert!(!logged.bytes().any(|byte| byte < 0x20));
8740 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
8741 }
8742
8743 #[tokio::test]
8744 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
8745 let registry = Arc::new(Registry::default());
8746 let supervisor_handle = SupervisorHandle::new();
8747 let missing_program = std::env::temp_dir().join(format!(
8748 "subc-route-open-missing-program-{}",
8749 std::process::id()
8750 ));
8751 let supervisor =
8752 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
8753 .with_handle(supervisor_handle.clone());
8754 let module = supervisor
8755 .supervise_configured(
8756 ModuleSpec {
8757 module_id: "failed".to_string(),
8758 program: missing_program,
8759 args: Vec::new(),
8760 env: Vec::new(),
8761 reserved: false,
8762 reserved_prefixes: Vec::new(),
8763 protocol: ModuleProtocol::Subc,
8764 overlap: Default::default(),
8765 },
8766 true,
8767 )
8768 .unwrap();
8769 assert_eq!(module.state().unwrap(), ModuleState::Failed);
8770
8771 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8772 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
8773 let response = handler
8774 .handle_control_frame(
8775 &ctx,
8776 route_open_frame(305, "failed", unique_project_root("failed")),
8777 )
8778 .await
8779 .unwrap();
8780
8781 assert_eq!(response[0].header.ty, FrameType::Error);
8782 let error = parse_error(&response[0]);
8783 assert_eq!(error["code"], "target_unavailable");
8784 assert!(error["message"]
8785 .as_str()
8786 .unwrap()
8787 .contains("state=failed, enabled=true, live=false"));
8788 }
8789
8790 #[tokio::test]
8791 async fn route_open_role_mismatch_remains_target_unavailable() {
8792 let registry = Arc::new(Registry::default());
8793 let handler = ControlHandler::new(Arc::clone(®istry));
8794 handler
8795 .handle_control(
8796 ConnectionId::new(41),
8797 non_routable_hello_frame_with_control_ops("health-only", 306, None),
8798 )
8799 .unwrap();
8800
8801 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
8802 let response = handler
8803 .handle_control_frame(
8804 &ctx,
8805 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
8806 )
8807 .await
8808 .unwrap();
8809
8810 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
8811 assert!(parse_error(&response[0])["message"]
8812 .as_str()
8813 .unwrap()
8814 .contains("does not provide the requested target"));
8815 }
8816
8817 #[tokio::test]
8818 async fn route_open_inactive_registration_remains_target_unavailable() {
8819 let registry = Arc::new(Registry::default());
8820 let handler = ControlHandler::new(Arc::clone(®istry));
8821 handler
8822 .handle_control(
8823 ConnectionId::new(43),
8824 hello_frame("inactive", PROTOCOL_VERSION, 308),
8825 )
8826 .unwrap();
8827 assert!(registry
8828 .set_module_state_for_test("inactive", ChannelState::Closed)
8829 .unwrap());
8830
8831 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
8832 let response = handler
8833 .handle_control_frame(
8834 &ctx,
8835 route_open_frame(309, "inactive", unique_project_root("inactive")),
8836 )
8837 .await
8838 .unwrap();
8839
8840 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
8841 assert!(parse_error(&response[0])["message"]
8842 .as_str()
8843 .unwrap()
8844 .contains("is not active"));
8845 }
8846
8847 #[tokio::test]
8848 async fn late_health_reply_is_recorded_through_the_module_response_path() {
8849 let registry = Arc::new(Registry::default());
8850 let forwarding = Arc::new(ForwardingTable::default());
8851 let supervisor_handle = SupervisorHandle::new();
8852 let supervisor = Supervisor::new(Arc::clone(®istry), crate::RestartPolicy::default())
8853 .with_forwarding(Arc::clone(&forwarding))
8854 .with_handle(supervisor_handle.clone());
8855 let module = supervisor
8856 .supervise_configured(
8857 crate::ModuleSpec {
8858 module_id: "late-health-response".to_string(),
8859 program: PathBuf::from("disabled-module"),
8860 args: Vec::new(),
8861 env: Vec::new(),
8862 reserved: false,
8863 reserved_prefixes: Vec::new(),
8864 protocol: ModuleProtocol::Subc,
8865 overlap: Default::default(),
8866 },
8867 false,
8868 )
8869 .unwrap();
8870 let handler =
8871 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8872 .with_supervisor(supervisor_handle);
8873 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
8874 handler
8875 .handle_control_frame(
8876 &module_ctx,
8877 hello_frame_with_control_ops(
8878 "late-health-response",
8879 PROTOCOL_VERSION,
8880 7,
8881 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
8882 ),
8883 )
8884 .await
8885 .unwrap();
8886 let probe_started_at = Instant::now() - Duration::from_millis(80);
8887 let pending = forwarding
8888 .begin_health_probe_rpc_for(
8889 "late-health-response",
8890 MODULE_CONTROL_OP_HEALTH_CHECK,
8891 probe_started_at,
8892 Instant::now() - Duration::from_millis(1),
8893 )
8894 .unwrap();
8895 assert!(forwarding
8896 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
8897 .unwrap());
8898
8899 let responses = handler
8900 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
8901 .await
8902 .unwrap();
8903
8904 assert!(responses.is_empty());
8905 let health = module.status().unwrap().health;
8906 assert_eq!(health.late_answer_count, 1);
8907 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
8908 }
8909
8910 #[tokio::test]
8911 async fn health_probe_timeout_and_module_death_are_typed() {
8912 let registry = Arc::new(Registry::default());
8913 let forwarding = Arc::new(ForwardingTable::default());
8914 let handler =
8915 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8916 .with_health_probe_timeout(Duration::from_millis(50));
8917 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
8918 hello_via_sink(
8919 &handler,
8920 &module_ctx,
8921 &mut module_rx,
8922 hello_frame_with_control_ops(
8923 "aft",
8924 PROTOCOL_VERSION,
8925 7,
8926 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
8927 ),
8928 )
8929 .await;
8930
8931 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
8932 let responses = handler
8933 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
8934 .await
8935 .unwrap();
8936 assert_eq!(responses[0].header.ty, FrameType::Error);
8937 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
8938 let _ = module_rx.try_recv();
8939
8940 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
8941 let health_handler = handler.clone();
8942 let death_task = tokio::spawn(async move {
8943 health_handler
8944 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
8945 .await
8946 .unwrap()
8947 });
8948 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8949 .await
8950 .unwrap()
8951 .unwrap();
8952 handler
8953 .cleanup_connection(module_ctx.connection_id)
8954 .unwrap();
8955 let responses = death_task.await.unwrap();
8956 assert_eq!(responses[0].header.ty, FrameType::Error);
8957 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
8958 }
8959
8960 #[test]
8961 fn hello_requires_exact_protocol_version() {
8962 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
8963 let registry = Arc::new(Registry::default());
8964 let handler = ControlHandler::new(Arc::clone(®istry));
8965 let responses = handler
8966 .handle_control(
8967 ConnectionId::new(connection),
8968 hello_frame("aft", offered, 9),
8969 )
8970 .unwrap();
8971
8972 assert_eq!(responses.len(), 1);
8973 assert_eq!(responses[0].header.ty, FrameType::Error);
8974 let error = parse_error(&responses[0]);
8975 assert_eq!(error["code"], "version_unsupported");
8976 assert!(registry.get_module("aft").unwrap().is_none());
8977 assert_eq!(registry.active_registration_count().unwrap(), 0);
8978 }
8979 }
8980
8981 #[test]
8982 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
8983 let registry = Arc::new(Registry::default());
8984 let forwarding = Arc::new(ForwardingTable::default());
8985 let handler =
8986 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8987 let module_connection = ConnectionId::new(301);
8988 let registration = registry
8989 .register_with_control_ops(
8990 manifest("aft-push", PROTOCOL_VERSION),
8991 PROTOCOL_VERSION,
8992 module_connection,
8993 module_baseline_control_ops(),
8994 )
8995 .unwrap();
8996 let (module_tx, _module_rx) = mpsc::channel(8);
8997 let endpoint = forwarding
8998 .register_module_connection(
8999 module_connection,
9000 "aft-push".to_string(),
9001 PROTOCOL_VERSION,
9002 manifest_concurrency(®istration.manifest),
9003 FrameSink::new(module_tx),
9004 )
9005 .unwrap();
9006
9007 let unknown = Frame::build(
9009 FrameType::Push,
9010 control_flags(),
9011 0,
9012 0,
9013 5,
9014 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
9015 )
9016 .unwrap();
9017 let out = handler.handle_status_update(endpoint, unknown).unwrap();
9018 assert!(
9019 out.is_empty(),
9020 "unknown push op must be ignored, got {out:?}"
9021 );
9022
9023 let malformed = Frame::build(
9025 FrameType::Push,
9026 control_flags(),
9027 0,
9028 0,
9029 6,
9030 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
9031 )
9032 .unwrap();
9033 let out = handler.handle_status_update(endpoint, malformed).unwrap();
9034 assert_eq!(out.len(), 1);
9035 assert_eq!(out[0].header.ty, FrameType::Error);
9036 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
9037 }
9038
9039 #[test]
9040 fn hello_rejected_when_connection_already_owns_client_routes() {
9041 let registry = Arc::new(Registry::default());
9042 let forwarding = Arc::new(ForwardingTable::default());
9043 let handler =
9044 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9045 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
9047 let client_connection = ConnectionId::new(202);
9048
9049 let responses = handler
9052 .handle_control(
9053 client_connection,
9054 hello_frame("aft-second", PROTOCOL_VERSION, 9),
9055 )
9056 .unwrap();
9057 assert_eq!(responses[0].header.ty, FrameType::Error);
9058 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
9059 assert!(registry.get_module("aft-second").unwrap().is_none());
9060 }
9061
9062 #[test]
9063 fn reserved_module_hello_requires_matching_launch_nonce() {
9064 let registry = Arc::new(Registry::default());
9065 let supervisor = SupervisorHandle::new();
9066 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
9069 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9070
9071 let no_nonce = handler
9073 .handle_control(
9074 ConnectionId::new(1),
9075 hello_frame("vault", PROTOCOL_VERSION, 1),
9076 )
9077 .unwrap();
9078 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
9079 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
9080 assert!(registry.get_module("vault").unwrap().is_none());
9081
9082 let wrong = handler
9084 .handle_control(
9085 ConnectionId::new(2),
9086 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
9087 )
9088 .unwrap();
9089 assert_eq!(wrong[0].header.ty, FrameType::Error);
9090 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
9091 assert!(registry.get_module("vault").unwrap().is_none());
9092
9093 let ok = handler
9095 .handle_control(
9096 ConnectionId::new(3),
9097 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
9098 )
9099 .unwrap();
9100 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
9101 assert!(registry.get_module("vault").unwrap().is_some());
9102 }
9103
9104 #[test]
9105 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
9106 let registry = Arc::new(Registry::default());
9107 let supervisor = SupervisorHandle::new();
9108 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
9109 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
9110 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9111
9112 let squat = handler
9113 .handle_control(
9114 ConnectionId::new(1),
9115 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
9116 )
9117 .unwrap();
9118 assert_eq!(squat[0].header.ty, FrameType::Error);
9119 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
9120 assert!(parse_error(&squat[0])["message"]
9121 .as_str()
9122 .unwrap()
9123 .contains("fed:"));
9124
9125 let accepted_peer = handler
9126 .handle_control(
9127 ConnectionId::new(2),
9128 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
9129 )
9130 .unwrap();
9131 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
9132
9133 let accepted_short = handler
9134 .handle_control(
9135 ConnectionId::new(3),
9136 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
9137 )
9138 .unwrap();
9139 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
9140
9141 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
9142 let response = handler
9143 .handle_control(
9144 ConnectionId::new(conn),
9145 hello_frame(module_id, PROTOCOL_VERSION, conn),
9146 )
9147 .unwrap();
9148 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
9149 }
9150 }
9151
9152 #[test]
9153 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
9154 let registry = Arc::new(Registry::default());
9155 let supervisor = SupervisorHandle::new();
9156 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
9157 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
9158 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
9159 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9160
9161 let owner_nonce = handler
9162 .handle_control(
9163 ConnectionId::new(1),
9164 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
9165 )
9166 .unwrap();
9167 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
9168 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
9169 assert!(registry.get_module("fed:special").unwrap().is_none());
9170
9171 let exact_nonce = handler
9172 .handle_control(
9173 ConnectionId::new(2),
9174 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
9175 )
9176 .unwrap();
9177 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
9178 assert!(registry.get_module("fed:special").unwrap().is_some());
9179 }
9180
9181 #[test]
9182 fn non_reserved_module_ignores_launch_nonce() {
9183 let registry = Arc::new(Registry::default());
9184 let handler = ControlHandler::new(Arc::clone(®istry));
9187 let no_nonce = handler
9188 .handle_control(
9189 ConnectionId::new(1),
9190 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
9191 )
9192 .unwrap();
9193 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
9194 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
9195
9196 let echoed_nonce = handler
9197 .handle_control(
9198 ConnectionId::new(2),
9199 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
9200 )
9201 .unwrap();
9202 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
9203 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
9204 }
9205
9206 #[test]
9207 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
9208 let handler = ControlHandler::default();
9209 let conn = ConnectionId::new(1);
9210 let malformed = Frame::build(
9211 FrameType::Hello,
9212 control_flags(),
9213 0,
9214 0,
9215 3,
9216 b"{not json".to_vec(),
9217 )
9218 .unwrap();
9219
9220 let error = handler.handle_control(conn, malformed).unwrap();
9221 assert_eq!(error[0].header.ty, FrameType::Error);
9222 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
9223
9224 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
9225 let pong = handler.handle_control(conn, ping).unwrap();
9226 assert_eq!(pong[0].header.ty, FrameType::Pong);
9227 assert_eq!(pong[0].header.corr, 4);
9228 }
9229
9230 #[test]
9231 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
9232 let registry = Arc::new(Registry::default());
9233 let handler = ControlHandler::new(Arc::clone(®istry));
9234
9235 handler
9236 .handle_control(
9237 ConnectionId::new(1),
9238 hello_frame("aft", PROTOCOL_VERSION, 1),
9239 )
9240 .unwrap();
9241 let duplicate = handler
9242 .handle_control(
9243 ConnectionId::new(2),
9244 hello_frame("aft", PROTOCOL_VERSION, 2),
9245 )
9246 .unwrap();
9247
9248 assert_eq!(duplicate[0].header.ty, FrameType::Error);
9249 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
9250 let registration = registry.get_module("aft").unwrap().unwrap();
9251 assert_eq!(registration.connection_id, ConnectionId::new(1));
9252 }
9253
9254 #[test]
9255 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
9256 let registry = Arc::new(Registry::default());
9257 let forwarding = Arc::new(ForwardingTable::default());
9258 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
9259 let handler =
9260 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9261 .with_process_liveness(process_liveness);
9262 let (ctx, route_channel, route_epoch) =
9263 bind_liveness_route(®istry, &forwarding, "aft-dead");
9264 let responses = handler
9265 .handle_route_poll(
9266 &ctx,
9267 route_poll_frame(41, PollKind::Liveness, route_channel),
9268 route_channel,
9269 route_epoch,
9270 PollKind::Liveness,
9271 )
9272 .unwrap();
9273
9274 assert_eq!(responses.len(), 1);
9275 assert_eq!(responses[0].header.ty, FrameType::Response);
9276 assert_route_poll_liveness(&responses[0], false);
9277 }
9278
9279 #[test]
9280 fn liveness_poll_without_process_source_uses_bound_route() {
9281 let registry = Arc::new(Registry::default());
9282 let forwarding = Arc::new(ForwardingTable::default());
9283 let handler =
9284 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9285 let (ctx, route_channel, route_epoch) =
9286 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
9287 let responses = handler
9288 .handle_route_poll(
9289 &ctx,
9290 route_poll_frame(42, PollKind::Liveness, route_channel),
9291 route_channel,
9292 route_epoch,
9293 PollKind::Liveness,
9294 )
9295 .unwrap();
9296
9297 assert_route_poll_liveness(&responses[0], true);
9298 }
9299
9300 #[test]
9301 fn liveness_poll_untracked_process_source_uses_bound_route() {
9302 let registry = Arc::new(Registry::default());
9303 let forwarding = Arc::new(ForwardingTable::default());
9304 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
9305 let handler =
9306 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9307 .with_process_liveness(process_liveness);
9308 let (ctx, route_channel, route_epoch) =
9309 bind_liveness_route(®istry, &forwarding, "aft-untracked");
9310 let responses = handler
9311 .handle_route_poll(
9312 &ctx,
9313 route_poll_frame(43, PollKind::Liveness, route_channel),
9314 route_channel,
9315 route_epoch,
9316 PollKind::Liveness,
9317 )
9318 .unwrap();
9319
9320 assert_route_poll_liveness(&responses[0], true);
9321 }
9322
9323 #[tokio::test]
9324 async fn unknown_op_returns_unknown_control_op() {
9325 let handler = ControlHandler::default();
9326 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
9327 let request = Frame::build(
9328 FrameType::Request,
9329 control_flags(),
9330 0,
9331 0,
9332 55,
9333 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
9334 )
9335 .unwrap();
9336
9337 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
9338
9339 assert_eq!(response.len(), 1);
9340 assert_eq!(response[0].header.ty, FrameType::Error);
9341 assert_eq!(response[0].header.corr, 55);
9342 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
9343 }
9344
9345 #[tokio::test]
9346 async fn supervisor_provenance_rejects_unknown_exact_module() {
9347 let handler = ControlHandler::default();
9348 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
9349 let request = Frame::build(
9350 FrameType::Request,
9351 control_flags(),
9352 0,
9353 0,
9354 57,
9355 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
9356 )
9357 .unwrap();
9358
9359 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
9360
9361 assert_eq!(response.len(), 1);
9362 assert_eq!(response[0].header.ty, FrameType::Error);
9363 assert_eq!(response[0].header.corr, 57);
9364 let error = parse_error(&response[0]);
9365 assert_eq!(error["code"], "unknown_module");
9366 assert_eq!(error["message"], "module_id 'missing' is not supervised");
9367 }
9368
9369 #[test]
9370 fn provenance_probe_override_keeps_handler_tests_deterministic() {
9371 let expected = subc_control::RunningImageAgreement::Unavailable {
9372 reason: subc_control::RunningImageUnavailableReason::HashFailed,
9373 };
9374 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
9375 assert_eq!(handler.provenance_probe_override, Some(expected));
9376 }
9377
9378 #[test]
9379 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
9380 let verdict = reload_verdict(
9381 std::path::Path::new("/bin/new"),
9382 Some(std::path::Path::new("/bin/old")),
9383 subc_control::RunningImageAgreement::Unavailable {
9384 reason: subc_control::RunningImageUnavailableReason::HashFailed,
9385 },
9386 );
9387 assert!(matches!(
9388 verdict.path,
9389 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
9390 if configured == std::path::Path::new("/bin/new")
9391 && spawned_from == std::path::Path::new("/bin/old")
9392 ));
9393 }
9394
9395 #[test]
9396 fn reload_verdict_detects_replaced_image_at_same_path() {
9397 let image = subc_control::RunningImageAgreement::Mismatch {
9398 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
9399 digest: "old".into(),
9400 },
9401 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
9402 digest: "new".into(),
9403 },
9404 };
9405 let verdict = reload_verdict(
9406 std::path::Path::new("/bin/same"),
9407 Some(std::path::Path::new("/bin/same")),
9408 image.clone(),
9409 );
9410 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
9411 assert_eq!(verdict.image, image);
9412 }
9413
9414 #[test]
9415 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
9416 let image = subc_control::RunningImageAgreement::Unavailable {
9417 reason: subc_control::RunningImageUnavailableReason::NotRunning,
9418 };
9419 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
9420 assert_eq!(
9421 verdict.path,
9422 subc_control::ReloadPathAgreement::Unavailable {
9423 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
9424 }
9425 );
9426 assert_eq!(verdict.image, image);
9427
9428 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
9429 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
9430 };
9431 let verdict = reload_verdict(
9432 std::path::Path::new("/bin/same"),
9433 Some(std::path::Path::new("/bin/same")),
9434 unconfirmed.clone(),
9435 );
9436 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
9437 assert_eq!(verdict.image, unconfirmed);
9438 }
9439
9440 #[test]
9441 fn reload_verdict_preserves_each_image_unavailability_reason() {
9442 use subc_control::RunningImageUnavailableReason as Reason;
9443
9444 for reason in [
9445 Reason::NotRunning,
9446 Reason::UnsupportedPlatform,
9447 Reason::RunningExecutableUnreadable,
9448 Reason::SpawnedPathUnreadable,
9449 Reason::HashFailed,
9450 Reason::ProcessIdentityUnconfirmed,
9451 Reason::Unknown("future_probe_reason".to_string()),
9452 ] {
9453 let image = subc_control::RunningImageAgreement::Unavailable {
9454 reason: reason.clone(),
9455 };
9456 let verdict = reload_verdict(
9457 std::path::Path::new("/bin/same"),
9458 Some(std::path::Path::new("/bin/same")),
9459 image.clone(),
9460 );
9461 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
9462 assert_eq!(verdict.image, image, "{reason:?}");
9463 }
9464 }
9465
9466 #[tokio::test]
9467 async fn malformed_control_bodies_return_invalid_control_body() {
9468 let handler = ControlHandler::default();
9469 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
9470
9471 for (corr, body) in [
9472 (56, br#"{"route_channel":1}"#.as_slice()),
9473 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
9474 (
9475 58,
9476 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
9477 ),
9478 ] {
9479 let request = Frame::build(
9480 FrameType::Request,
9481 control_flags(),
9482 0,
9483 0,
9484 corr,
9485 body.to_vec(),
9486 )
9487 .unwrap();
9488 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
9489
9490 assert_eq!(response.len(), 1);
9491 assert_eq!(response[0].header.ty, FrameType::Error);
9492 assert_eq!(response[0].header.corr, corr);
9493 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
9494 }
9495 }
9496
9497 #[tokio::test]
9498 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
9499 let registry = Arc::new(Registry::default());
9500 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
9501 let router = Router::with_control_handler(Arc::clone(&control));
9502 let connection = router.begin_connection();
9503 let (ctx, mut rx) = route_ctx(connection.id());
9504
9505 router
9506 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
9507 .await
9508 .unwrap();
9509 let response = rx.recv().await.unwrap();
9510 let ack = parse_ack(&response);
9511 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9512 let channel = 1;
9513
9514 let goodbye =
9515 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
9516 router.route_for_connection(&ctx, goodbye).await.unwrap();
9517 assert!(rx.try_recv().is_err());
9518 assert!(registry.get_module("aft").unwrap().is_none());
9519
9520 router
9521 .route_for_connection(&ctx, channel_request(channel, 13))
9522 .await
9523 .unwrap();
9524 let error_frame = rx.recv().await.unwrap();
9525 assert_eq!(error_frame.header.ty, FrameType::Error);
9526 assert_eq!(error_frame.header.channel, channel);
9527 }
9528
9529 #[tokio::test]
9530 async fn dropping_router_connection_releases_registration() {
9531 let registry = Arc::new(Registry::default());
9532 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
9533 let router = Router::with_control_handler(control);
9534 let connection = router.begin_connection();
9535 let (ctx, mut rx) = route_ctx(connection.id());
9536
9537 router
9538 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
9539 .await
9540 .unwrap();
9541 let response = rx.recv().await.unwrap();
9542 let ack = parse_ack(&response);
9543 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9544 assert!(registry.get_module("aft").unwrap().is_some());
9545
9546 drop(connection);
9547
9548 assert!(registry.get_module("aft").unwrap().is_none());
9549 assert_eq!(registry.active_registration_count().unwrap(), 0);
9550 }
9551
9552 fn capability_manifest(
9553 module_id: &str,
9554 provides: &[&str],
9555 must_never_reach: &[&str],
9556 ) -> ModuleManifest {
9557 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
9558 manifest.capabilities = Some(CapabilityDeclarations {
9559 provides: provides
9560 .iter()
9561 .map(|capability| (*capability).to_string())
9562 .collect(),
9563 requires: Vec::new(),
9564 must_never_reach: must_never_reach
9565 .iter()
9566 .map(|capability| (*capability).to_string())
9567 .collect(),
9568 });
9569 manifest
9570 }
9571
9572 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
9573 Frame::build(
9574 FrameType::Hello,
9575 control_flags(),
9576 0,
9577 0,
9578 corr,
9579 serde_json::to_vec(&ModuleHelloBody {
9580 protocol_ver: manifest.protocol_ver,
9581 manifest,
9582 control_ops: None,
9583 launch_nonce: None,
9584 })
9585 .expect("capability test HELLO serializes"),
9586 )
9587 .expect("capability test HELLO frame builds")
9588 }
9589
9590 fn catalog_update_with_capabilities_frame(
9591 corr: u64,
9592 capabilities: CapabilityDeclarations,
9593 ) -> Frame {
9594 Frame::build(
9595 FrameType::Request,
9596 control_flags(),
9597 0,
9598 0,
9599 corr,
9600 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
9601 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
9602 capabilities: Some(capabilities),
9603 ready: None,
9604 })
9605 .expect("capability catalog.update serializes"),
9606 )
9607 .expect("capability catalog.update frame builds")
9608 }
9609
9610 async fn register_capability_manifest(
9611 handler: &ControlHandler,
9612 ctx: &RouteCtx,
9613 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9614 manifest: ModuleManifest,
9615 corr: u64,
9616 ) {
9617 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
9618 }
9619
9620 async fn open_route_for_capability_test(
9621 handler: &ControlHandler,
9622 target_ctx: &RouteCtx,
9623 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9624 client_connection_id: u64,
9625 corr: u64,
9626 target_module_id: &str,
9627 consumer_identity: Option<ConsumerIdentity>,
9628 ) -> (
9629 mpsc::Receiver<crate::router::OutboundFrame>,
9630 ModuleControlRequest,
9631 ) {
9632 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
9633 let route_handler = handler.clone();
9634 let target_module_id = target_module_id.to_string();
9635 let route_task = tokio::spawn(async move {
9636 route_handler
9637 .handle_control_frame(
9638 &client_ctx,
9639 route_open_frame_with_admission_facts(
9640 corr,
9641 &target_module_id,
9642 unique_project_root("admission-facts"),
9643 consumer_identity,
9644 None,
9645 ),
9646 )
9647 .await
9648 .expect("capability test route.open succeeds")
9649 });
9650 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
9651 .await
9652 .expect("capability test route.open must reach route.bind")
9653 .expect("target control receiver stays open");
9654 let bind_request: ModuleControlRequest =
9655 serde_json::from_slice(&bind.body).expect("route.bind decodes");
9656 handler
9657 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
9658 .await
9659 .expect("capability test route.bind ACK succeeds");
9660 assert!(route_task.await.expect("route.open task joins").is_empty());
9661 let opened = client_rx
9662 .recv()
9663 .await
9664 .expect("successful route.open publishes a response");
9665 assert!(matches!(
9666 serde_json::from_slice::<ClientControlResponse>(&opened.body),
9667 Ok(ClientControlResponse::RouteOpen { .. })
9668 ));
9669 (client_rx, bind_request)
9670 }
9671
9672 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
9673 assert_eq!(frame.header.ty, FrameType::Push);
9674 assert_eq!(frame.header.channel, 0);
9675 assert_eq!(
9676 serde_json::from_slice::<ClientControlPush>(&frame.body)
9677 .expect("route.closed control push decodes"),
9678 ClientControlPush::RouteClosed {
9679 module_id: target_module_id.to_string(),
9680 reason: RouteCloseReason::CapabilityDenied,
9681 drained: false,
9682 abandoned: 0,
9683 excluded_subscriptions: 0,
9684 terminal: Some(false),
9685 }
9686 );
9687 }
9688
9689 #[tokio::test]
9690 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
9691 let registry = Arc::new(Registry::default());
9692 let forwarding = Arc::new(ForwardingTable::default());
9693 let supervisor = SupervisorHandle::new();
9694 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9695 let handler =
9696 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9697 .with_supervisor(supervisor);
9698 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
9699 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
9700 register_capability_manifest(
9701 &handler,
9702 &target_ctx,
9703 &mut target_rx,
9704 capability_manifest("target", &["credentials-provider/v1"], &[]),
9705 1,
9706 )
9707 .await;
9708 register_capability_manifest(
9709 &handler,
9710 &opener_ctx,
9711 &mut opener_rx,
9712 capability_manifest("opener", &[], &["credentials-provider/v1"]),
9713 2,
9714 )
9715 .await;
9716
9717 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
9718 let replies = handler
9719 .handle_control_frame(
9720 &client_ctx,
9721 route_open_frame_with_admission_facts(
9722 3,
9723 "target",
9724 unique_project_root("admission-facts"),
9725 Some(ConsumerIdentity {
9726 module_id: "opener".to_string(),
9727 launch_nonce: "opener-nonce".to_string(),
9728 }),
9729 None,
9730 ),
9731 )
9732 .await
9733 .expect("denied route.open returns a typed frame");
9734 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
9735 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
9736 assert!(
9737 target_rx.try_recv().is_err(),
9738 "forbidden route.open must not relay route.bind"
9739 );
9740 }
9741
9742 #[tokio::test]
9743 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
9744 let registry = Arc::new(Registry::default());
9745 let forwarding = Arc::new(ForwardingTable::default());
9746 let supervisor = SupervisorHandle::new();
9747 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9748 let handler =
9749 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9750 .with_supervisor(supervisor);
9751 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
9752 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
9753 register_capability_manifest(
9754 &handler,
9755 &target_ctx,
9756 &mut target_rx,
9757 capability_manifest("target", &["credentials-provider/v1"], &[]),
9758 1,
9759 )
9760 .await;
9761 register_capability_manifest(
9762 &handler,
9763 &old_opener_ctx,
9764 &mut old_opener_rx,
9765 capability_manifest("opener", &[], &[]),
9766 2,
9767 )
9768 .await;
9769 let (mut client_rx, _) = open_route_for_capability_test(
9770 &handler,
9771 &target_ctx,
9772 &mut target_rx,
9773 712,
9774 3,
9775 "target",
9776 Some(ConsumerIdentity {
9777 module_id: "opener".to_string(),
9778 launch_nonce: "opener-nonce".to_string(),
9779 }),
9780 )
9781 .await;
9782 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
9783
9784 handler
9785 .cleanup_connection(old_opener_ctx.connection_id)
9786 .expect("old opener registration cleans up");
9787 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
9788 register_capability_manifest(
9789 &handler,
9790 &new_opener_ctx,
9791 &mut new_opener_rx,
9792 capability_manifest("opener", &[], &["credentials-provider/v1"]),
9793 4,
9794 )
9795 .await;
9796
9797 assert_capability_denied_push(
9798 client_rx
9799 .try_recv()
9800 .expect("HELLO deny addition must emit route.closed")
9801 .frame,
9802 "target",
9803 );
9804 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
9805 assert!(matches!(
9806 target_rx.try_recv(),
9807 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
9808 ));
9809 }
9810
9811 #[tokio::test]
9812 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
9813 let registry = Arc::new(Registry::default());
9814 let forwarding = Arc::new(ForwardingTable::default());
9815 let supervisor = SupervisorHandle::new();
9816 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9817 let handler =
9818 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9819 .with_supervisor(supervisor);
9820 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
9821 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
9822 register_capability_manifest(
9823 &handler,
9824 &target_ctx,
9825 &mut target_rx,
9826 capability_manifest("target", &[], &[]),
9827 1,
9828 )
9829 .await;
9830 register_capability_manifest(
9831 &handler,
9832 &opener_ctx,
9833 &mut opener_rx,
9834 capability_manifest("opener", &[], &["credentials-provider/v1"]),
9835 2,
9836 )
9837 .await;
9838 let (mut client_rx, _) = open_route_for_capability_test(
9839 &handler,
9840 &target_ctx,
9841 &mut target_rx,
9842 722,
9843 3,
9844 "target",
9845 Some(ConsumerIdentity {
9846 module_id: "opener".to_string(),
9847 launch_nonce: "opener-nonce".to_string(),
9848 }),
9849 )
9850 .await;
9851 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
9852
9853 let replies = handler
9854 .handle_control_frame(
9855 &target_ctx,
9856 catalog_update_with_capabilities_frame(
9857 4,
9858 CapabilityDeclarations {
9859 provides: vec!["credentials-provider/v1".to_string()],
9860 requires: Vec::new(),
9861 must_never_reach: Vec::new(),
9862 },
9863 ),
9864 )
9865 .await
9866 .expect("claim catalog.update succeeds");
9867 assert!(matches!(
9868 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
9869 Ok(ModuleControlResponseToModule::CatalogUpdate {})
9870 ));
9871 assert_capability_denied_push(
9872 client_rx
9873 .try_recv()
9874 .expect("claim addition must emit route.closed")
9875 .frame,
9876 "target",
9877 );
9878 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
9879 assert!(matches!(
9880 target_rx.try_recv(),
9881 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
9882 ));
9883 }
9884
9885 #[tokio::test]
9886 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
9887 let registry = Arc::new(Registry::default());
9888 let forwarding = Arc::new(ForwardingTable::default());
9889 let supervisor = SupervisorHandle::new();
9890 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9891 let handler =
9892 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9893 .with_supervisor(supervisor);
9894 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
9895 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
9896 register_capability_manifest(
9897 &handler,
9898 &target_ctx,
9899 &mut target_rx,
9900 capability_manifest("target", &["credentials-provider/v1"], &[]),
9901 1,
9902 )
9903 .await;
9904 register_capability_manifest(
9905 &handler,
9906 &opener_ctx,
9907 &mut opener_rx,
9908 capability_manifest("opener", &[], &[]),
9909 2,
9910 )
9911 .await;
9912 let (mut client_rx, _) = open_route_for_capability_test(
9913 &handler,
9914 &target_ctx,
9915 &mut target_rx,
9916 732,
9917 3,
9918 "target",
9919 Some(ConsumerIdentity {
9920 module_id: "opener".to_string(),
9921 launch_nonce: "opener-nonce".to_string(),
9922 }),
9923 )
9924 .await;
9925 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
9926
9927 handler
9928 .handle_control_frame(
9929 &target_ctx,
9930 catalog_update_with_capabilities_frame(
9931 4,
9932 CapabilityDeclarations {
9933 provides: Vec::new(),
9934 requires: Vec::new(),
9935 must_never_reach: Vec::new(),
9936 },
9937 ),
9938 )
9939 .await
9940 .expect("claim removal catalog.update succeeds");
9941 assert_eq!(
9942 forwarding.active_binding_count().unwrap(),
9943 1,
9944 "removing an attested target claim must leave the route census unchanged"
9945 );
9946 assert!(
9947 client_rx.try_recv().is_err(),
9948 "claim removal must not emit route.closed capability_denied"
9949 );
9950 assert!(
9951 target_rx.try_recv().is_err(),
9952 "claim removal must not send the target a route GOODBYE"
9953 );
9954 }
9955
9956 #[tokio::test]
9959 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
9960 let registry = Arc::new(Registry::default());
9961 let forwarding = Arc::new(ForwardingTable::default());
9962 let supervisor = SupervisorHandle::new();
9963 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9964 let handler =
9965 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9966 .with_supervisor(supervisor);
9967 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
9968 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
9969 register_capability_manifest(
9970 &handler,
9971 &target_ctx,
9972 &mut target_rx,
9973 capability_manifest("target", &["credentials-provider/v1"], &[]),
9974 1,
9975 )
9976 .await;
9977 register_capability_manifest(
9978 &handler,
9979 &opener_ctx,
9980 &mut opener_rx,
9981 capability_manifest("opener", &[], &["credentials-provider/v1"]),
9982 2,
9983 )
9984 .await;
9985
9986 let (_client_rx, bind) = open_route_for_capability_test(
9987 &handler,
9988 &target_ctx,
9989 &mut target_rx,
9990 742,
9991 3,
9992 "target",
9993 None,
9994 )
9995 .await;
9996 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
9997 panic!("direct scope-honesty route must bind");
9998 };
9999 assert_eq!(principal, Some(Principal::Direct));
10000 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
10001 }
10002
10003 #[tokio::test]
10006 async fn must_never_reach_self_route_is_capability_forbidden() {
10007 let registry = Arc::new(Registry::default());
10008 let forwarding = Arc::new(ForwardingTable::default());
10009 let supervisor = SupervisorHandle::new();
10010 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
10011 let handler =
10012 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10013 .with_supervisor(supervisor);
10014 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
10015 register_capability_manifest(
10016 &handler,
10017 &self_ctx,
10018 &mut self_rx,
10019 capability_manifest(
10020 "self-provider",
10021 &["credentials-provider/v1"],
10022 &["credentials-provider/v1"],
10023 ),
10024 1,
10025 )
10026 .await;
10027
10028 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
10029 let replies = handler
10030 .handle_control_frame(
10031 &client_ctx,
10032 route_open_frame_with_admission_facts(
10033 2,
10034 "self-provider",
10035 unique_project_root("admission-facts"),
10036 Some(ConsumerIdentity {
10037 module_id: "self-provider".to_string(),
10038 launch_nonce: "self-nonce".to_string(),
10039 }),
10040 None,
10041 ),
10042 )
10043 .await
10044 .expect("self-route refusal returns a typed frame");
10045 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
10046 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
10047 assert!(
10048 self_rx.try_recv().is_err(),
10049 "self denial must not relay route.bind"
10050 );
10051 }
10052
10053 #[test]
10054 fn unsupported_channel_zero_frame_returns_error() {
10055 let handler = ControlHandler::default();
10056 let request = Frame::build(
10057 FrameType::Request,
10058 control_flags(),
10059 0,
10060 0,
10061 21,
10062 b"opaque".to_vec(),
10063 )
10064 .unwrap();
10065
10066 let response = handler
10067 .handle_control(ConnectionId::new(1), request)
10068 .unwrap();
10069
10070 assert_eq!(response[0].header.ty, FrameType::Error);
10071 assert_eq!(
10072 parse_error(&response[0])["code"],
10073 "unsupported_control_frame"
10074 );
10075 }
10076
10077 mod swap {
10082 use super::*;
10083
10084 const INCUMBENT: ConnectionId = ConnectionId::new(30);
10085 const CANDIDATE: ConnectionId = ConnectionId::new(40);
10086
10087 struct Swap {
10088 registry: Arc<Registry>,
10089 forwarding: Arc<ForwardingTable>,
10090 handler: ControlHandler,
10091 incumbent_ctx: RouteCtx,
10092 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
10093 candidate_ctx: RouteCtx,
10094 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
10095 }
10096
10097 async fn swap_with_incumbent() -> Swap {
10098 let registry = Arc::new(Registry::default());
10099 let forwarding = Arc::new(ForwardingTable::default());
10100 let handler =
10101 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10102 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
10103 hello_via_sink(
10104 &handler,
10105 &incumbent_ctx,
10106 &mut incumbent_rx,
10107 hello_frame("aft", PROTOCOL_VERSION, 7),
10108 )
10109 .await;
10110 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
10111 Swap {
10112 registry,
10113 forwarding,
10114 handler,
10115 incumbent_ctx,
10116 incumbent_rx,
10117 candidate_ctx,
10118 candidate_rx,
10119 }
10120 }
10121
10122 fn register_candidate(swap: &Swap, ready: Option<bool>) {
10123 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
10124 candidate_manifest.ready = ready;
10125 let registration = swap
10126 .registry
10127 .register_candidate_with_control_ops(
10128 candidate_manifest,
10129 PROTOCOL_VERSION,
10130 CANDIDATE,
10131 module_baseline_control_ops(),
10132 )
10133 .unwrap();
10134 swap.forwarding
10135 .register_candidate_module_connection(
10136 CANDIDATE,
10137 "aft".to_string(),
10138 PROTOCOL_VERSION,
10139 manifest_concurrency(®istration.manifest),
10140 swap.candidate_ctx.egress.clone(),
10141 )
10142 .unwrap();
10143 }
10144
10145 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
10146 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
10147 swap.registry.promote_candidate("aft").unwrap().unwrap();
10148 cutover.incumbent.unwrap()
10149 }
10150
10151 fn keyed_total(counters: &Value, key: &str) -> u64 {
10152 counters[key]
10153 .as_object()
10154 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
10155 .unwrap_or(0)
10156 }
10157
10158 #[tokio::test]
10164 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
10165 let mut swap = swap_with_incumbent().await;
10166 let handler = swap.handler.clone();
10167
10168 let cotenant = ConnectionId::new(31);
10170 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
10171 let (cotenant_task, cotenant_bind) = relay_route_open(
10172 &handler,
10173 cotenant,
10174 &cotenant_ctx.egress,
10175 &mut swap.incumbent_rx,
10176 100,
10177 "aft",
10178 "swap-cotenant",
10179 )
10180 .await;
10181 handler
10182 .handle_control_frame(
10183 &swap.incumbent_ctx,
10184 route_bind_ack(cotenant_bind.header.corr),
10185 )
10186 .await
10187 .unwrap();
10188 assert!(cotenant_task.await.unwrap().is_empty());
10189 let (cotenant_channel, cotenant_epoch) =
10190 published_route(&cotenant_rx.recv().await.unwrap());
10191
10192 let caller = ConnectionId::new(32);
10194 let (caller_ctx, mut caller_rx) = route_ctx(caller);
10195 let (caller_task, caller_bind) = relay_route_open(
10196 &handler,
10197 caller,
10198 &caller_ctx.egress,
10199 &mut swap.incumbent_rx,
10200 101,
10201 "aft",
10202 "swap-caller",
10203 )
10204 .await;
10205 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
10206
10207 register_candidate(&swap, None);
10208 cutover(&swap);
10209
10210 let ack = handler
10212 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
10213 .await;
10214 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
10215 if module_loop_error.is_some() {
10216 handler.cleanup_connection(INCUMBENT).unwrap();
10219 }
10220
10221 assert!(
10223 cotenant_rx.try_recv().is_err(),
10224 "the co-tenant route on the incumbent was torn down by one late ack: \
10225 {module_loop_error:?}"
10226 );
10227 assert!(matches!(
10228 swap.forwarding
10229 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
10230 .unwrap(),
10231 DataRoute::Client(DataRouteState::Bound(_))
10232 ));
10233 assert_eq!(module_loop_error, None);
10234 assert!(swap
10235 .registry
10236 .get_module_by_connection(INCUMBENT)
10237 .unwrap()
10238 .is_some());
10239
10240 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
10242 .await
10243 .expect("the incumbent is told to drop the abandoned binding")
10244 .unwrap()
10245 .frame;
10246 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10247 assert_eq!(goodbye.header.channel, abandoned_channel);
10248 assert_eq!(goodbye.header.epoch, abandoned_epoch);
10249 assert!(swap.incumbent_rx.try_recv().is_err());
10250
10251 let response = caller_task.await.unwrap();
10253 assert_eq!(response.len(), 1);
10254 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
10255 assert!(caller_rx.try_recv().is_err());
10256
10257 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
10261 let counters = handler.counters().snapshot();
10262 assert_eq!(
10263 keyed_total(&counters, "route_open_accepted_by_principal"),
10264 1
10265 );
10266 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
10267 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
10268 }
10269
10270 #[tokio::test]
10274 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
10275 let mut swap = swap_with_incumbent().await;
10276 register_candidate(&swap, None);
10277 let incumbent = cutover(&swap);
10278 swap.forwarding
10279 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
10280 .unwrap()
10281 .expect("the incumbent is still registered");
10282
10283 let client = ConnectionId::new(33);
10284 let (client_ctx, mut client_rx) = route_ctx(client);
10285 let route_handler = swap.handler.clone();
10286 let open_ctx = RouteCtx {
10287 connection_id: client,
10288 egress: client_ctx.egress.clone(),
10289 };
10290 let mut route_task = tokio::spawn(async move {
10291 route_handler
10292 .handle_control_frame(
10293 &open_ctx,
10294 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
10295 )
10296 .await
10297 .unwrap()
10298 });
10299 let bind = tokio::select! {
10300 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
10301 response = &mut route_task => {
10302 let response = response.unwrap();
10303 panic!(
10304 "post-cutover route.open was refused instead of relayed to the candidate: {}",
10305 parse_error(&response[0])["code"]
10306 );
10307 }
10308 };
10309 swap.handler
10310 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
10311 .await
10312 .unwrap();
10313 assert!(route_task.await.unwrap().is_empty());
10314 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
10315 match swap
10316 .forwarding
10317 .lookup_data_route(client, channel, epoch)
10318 .unwrap()
10319 {
10320 DataRoute::Client(DataRouteState::Bound(route)) => {
10321 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
10322 }
10323 other => panic!("expected a bound route on the candidate, got {other:?}"),
10324 }
10325 assert!(swap.incumbent_rx.try_recv().is_err());
10326 }
10327
10328 #[tokio::test]
10333 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
10334 let swap = swap_with_incumbent().await;
10335 register_candidate(&swap, Some(false));
10336 let update = Frame::build(
10337 FrameType::Request,
10338 control_flags(),
10339 0,
10340 0,
10341 55,
10342 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
10343 provides: manifest("aft", PROTOCOL_VERSION).provides,
10344 capabilities: None,
10345 ready: Some(true),
10346 })
10347 .unwrap(),
10348 )
10349 .unwrap();
10350
10351 let replies = swap
10352 .handler
10353 .handle_control_frame(&swap.candidate_ctx, update)
10354 .await
10355 .unwrap();
10356
10357 assert_eq!(replies.len(), 1);
10358 assert_eq!(
10359 replies[0].header.ty,
10360 FrameType::Response,
10361 "candidate catalog.update was refused: {:?}",
10362 serde_json::from_slice::<Value>(&replies[0].body).ok()
10363 );
10364 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
10365 assert_eq!(
10366 swap.registry
10367 .get_module("aft")
10368 .unwrap()
10369 .unwrap()
10370 .connection_id,
10371 INCUMBENT
10372 );
10373 }
10374 }
10375
10376 mod swap_admission {
10380 use super::*;
10381
10382 const INCUMBENT_NONCE: &str = "incumbent-nonce";
10383 const CANDIDATE_NONCE: &str = "candidate-nonce";
10384
10385 fn handler_with_incumbent(
10386 module_id: &str,
10387 reserved: bool,
10388 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
10389 let registry = Arc::new(Registry::default());
10390 let supervisor = SupervisorHandle::new();
10391 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
10392 if reserved {
10393 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
10394 }
10395 let handler =
10396 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
10397 let incumbent = handler
10398 .handle_control(
10399 ConnectionId::new(1),
10400 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
10401 )
10402 .unwrap();
10403 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
10404 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
10405 (registry, supervisor, handler)
10406 }
10407
10408 #[test]
10414 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
10415 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
10416
10417 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
10418 let replies = handler
10419 .handle_control(
10420 ConnectionId::new(connection),
10421 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
10422 )
10423 .unwrap();
10424 assert_eq!(replies[0].header.ty, FrameType::Error);
10425 assert_eq!(
10426 parse_error(&replies[0])["code"],
10427 "swap_token_invalid",
10428 "nonce {nonce:?}"
10429 );
10430 }
10431 assert!(registry.get_candidate("aft").unwrap().is_none());
10432 assert_eq!(
10433 registry.get_module("aft").unwrap().unwrap().connection_id,
10434 ConnectionId::new(1)
10435 );
10436
10437 let admitted = handler
10440 .handle_control(
10441 ConnectionId::new(4),
10442 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
10443 )
10444 .unwrap();
10445 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
10446 assert_eq!(
10447 registry
10448 .get_candidate("aft")
10449 .unwrap()
10450 .unwrap()
10451 .connection_id,
10452 ConnectionId::new(4)
10453 );
10454 assert_eq!(
10455 registry.get_module("aft").unwrap().unwrap().connection_id,
10456 ConnectionId::new(1),
10457 "the candidate must not take the active slot"
10458 );
10459 let replayed = handler
10460 .handle_control(
10461 ConnectionId::new(5),
10462 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
10463 )
10464 .unwrap();
10465 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
10466
10467 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
10471 let squatter = handler
10472 .handle_control(
10473 ConnectionId::new(6),
10474 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
10475 )
10476 .unwrap();
10477 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
10478 assert!(
10479 registry.get_module("aft").unwrap().is_none(),
10480 "a squatter took the active slot of an id being swapped"
10481 );
10482 }
10483
10484 #[test]
10489 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
10490 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
10491
10492 let replies = handler
10493 .handle_control(
10494 ConnectionId::new(2),
10495 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
10496 )
10497 .unwrap();
10498
10499 assert_eq!(
10500 replies[0].header.ty,
10501 FrameType::HelloAck,
10502 "reserved candidate refused: {:?}",
10503 serde_json::from_slice::<Value>(&replies[0].body).ok()
10504 );
10505 assert_eq!(
10506 registry
10507 .get_candidate("vault")
10508 .unwrap()
10509 .unwrap()
10510 .connection_id,
10511 ConnectionId::new(2)
10512 );
10513 }
10514
10515 #[test]
10518 fn without_an_open_swap_the_ordinary_gates_decide() {
10519 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
10520 supervisor.close_swap("vault");
10521
10522 let candidate = handler
10523 .handle_control(
10524 ConnectionId::new(2),
10525 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
10526 )
10527 .unwrap();
10528 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
10529 let duplicate = handler
10530 .handle_control(
10531 ConnectionId::new(3),
10532 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
10533 )
10534 .unwrap();
10535 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
10536 assert!(registry.get_candidate("vault").unwrap().is_none());
10537 }
10538 }
10539}
10540
10541#[cfg(test)]
10542mod concurrency_default_exposure_tests {
10543 use super::*;
10544
10545 fn hello_body(role_json: &str) -> Vec<u8> {
10546 format!(
10547 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
10548 )
10549 .into_bytes()
10550 }
10551
10552 fn manifest_from(body: &[u8]) -> ModuleManifest {
10553 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
10554 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
10555 .expect("manifest parses")
10556 }
10557
10558 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
10559
10560 #[test]
10561 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
10562 let body = hello_body(&format!(
10563 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
10564 ));
10565 let manifest = manifest_from(&body);
10566 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
10569 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
10570 }
10571
10572 #[test]
10573 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
10574 let body = hello_body(&format!(
10575 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
10576 ));
10577 let manifest = manifest_from(&body);
10578 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
10579 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
10580 }
10581
10582 #[test]
10583 fn non_management_roles_are_never_reported() {
10584 let body = hello_body(
10585 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
10586 );
10587 let manifest = manifest_from(&body);
10588 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
10589 }
10590}