1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 path::{Path, PathBuf},
5 sync::{Arc, Mutex},
6 time::{Duration, Instant as StdInstant},
7};
8
9use serde::{Deserialize, Serialize};
10use subc_control::{
11 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
12 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
13 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
14 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
15 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
16 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
17 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
18};
19use subc_protocol::{
20 error_codes,
21 manifest::{
22 validate_hello_capability_grammar, validate_hello_self_signal_declarations,
23 CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
24 ProviderRole,
25 },
26 session::{
27 HealthReport, ModuleControlPush, ModuleControlRequest, ModuleControlRequestFromModule,
28 ModuleControlResponse, ModuleControlResponseToModule, MODULE_CONTROL_OP_HEALTH_CHECK,
29 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
30 },
31 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
32 Priority, RouteTarget, PROTOCOL_VERSION,
33};
34use tokio::time::{timeout_at, Instant};
35use tracing::{debug, info, warn};
36
37use crate::{
38 capability_requirements::{
39 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
40 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
41 RuntimeModule,
42 },
43 daemon_config::RestartRequiredSection,
44 forwarding::{
45 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
46 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
47 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
48 },
49 observability::{
50 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
51 },
52 provenance::{
53 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
54 },
55 registry::{ChannelState, ConnectionId, Registry, RegistryError},
56 router::{RouteCtx, RouterError},
57 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
58 stderr_tail::{CaptureState, TailEntry},
59 supervise::{
60 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
61 SupervisorHandle, SwapHelloAdmission,
62 },
63 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
64};
65
66pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
72
73const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
74const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
75const CAP_PING_PONG: &str = "ping_pong_v1";
76const CAP_SESSION_ATTACH: &str = "session_attach_v1";
77const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
78
79const SUBC_CONTROL_OPS: &[&str] = &[
80 ops::SERVER_DESCRIBE,
81 ops::CATALOG_LIST,
82 ops::ROUTE_OPEN,
83 ops::ROUTE_POLL,
84 ops::ROUTE_CLOSING,
85 ops::ROUTE_CLOSED,
86 ops::SUPERVISOR_LIST,
87 ops::SUPERVISOR_RESTART,
88 ops::SUPERVISOR_SWAP,
89 ops::SUPERVISOR_RELOAD,
90 ops::SUPERVISOR_RESCAN,
91 ops::SUPERVISOR_RELEASE_RESERVED,
92 ops::SUPERVISOR_SET_ENABLED,
93 ops::SUPERVISOR_HEALTH_PROBE,
94 ops::SUPERVISOR_HEALTH,
95 ops::SUPERVISOR_STDERR_TAIL,
96 ops::SUPERVISOR_TERMINALS,
97 ops::SUPERVISOR_ROUTES,
98 ops::SUPERVISOR_PROVENANCE,
99 ops::SUPERVISOR_SPAWN_SNAPSHOT,
100 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
101];
102
103const MODULE_TO_SUBC_CONTROL_OPS: &[&str] =
104 &[MODULE_TO_SUBC_OP_CATALOG_UPDATE, "supervisor.live_roots"];
105
106const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
107
108pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
116
117pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
132
133pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
151
152const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
153const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
154
155fn reload_verdict(
156 configured: &Path,
157 spawned_from: Option<&Path>,
158 image: subc_control::RunningImageAgreement,
159) -> PendingReloadVerdict {
160 let path = match spawned_from {
161 Some(spawned_from) if configured == spawned_from => ReloadPathAgreement::Match,
162 Some(spawned_from) => ReloadPathAgreement::Mismatch {
163 configured: configured.to_path_buf(),
164 spawned_from: spawned_from.to_path_buf(),
165 },
166 None => ReloadPathAgreement::Unavailable {
167 reason: if matches!(
168 image,
169 subc_control::RunningImageAgreement::Unavailable {
170 reason: subc_control::RunningImageUnavailableReason::NotRunning
171 }
172 ) {
173 ReloadPathUnavailableReason::NotRunning
174 } else {
175 ReloadPathUnavailableReason::SpawnedPathUnavailable
176 },
177 },
178 };
179 PendingReloadVerdict { path, image }
180}
181
182#[derive(Clone)]
183struct DaemonProvenanceFacts {
184 build: DaemonBuildProvenance,
185 pid: Option<u32>,
186 started_at_ms: Option<u64>,
187 start_clock: Option<crate::clock::StartClock>,
188 executable_path: Option<PathBuf>,
189 executable_identity: Option<SpawnedFileIdentity>,
190 process_start_time: Option<u64>,
191 probe: ExecutableIdentityProbe,
192}
193
194impl Default for DaemonProvenanceFacts {
195 fn default() -> Self {
196 Self {
197 build: DaemonBuildProvenance {
198 build_git_sha: None,
199 build_lock_digest: None,
200 },
201 pid: None,
202 started_at_ms: None,
203 start_clock: None,
204 executable_path: None,
205 executable_identity: None,
206 process_start_time: None,
207 probe: ExecutableIdentityProbe::default(),
208 }
209 }
210}
211
212#[derive(Debug, Clone)]
213struct SupervisorRescanContext {
214 supervisor: Supervisor,
215 config_path: PathBuf,
216 configured_port: Option<u16>,
217 storage_config: Option<crate::daemon_config::StorageConfig>,
218 admission_facts_carrier_module_id: Option<String>,
219 admission_facts_targets: Option<Vec<String>>,
220}
221
222#[derive(Clone)]
224pub struct ControlHandler {
225 registry: Arc<Registry>,
226 forwarding: Arc<ForwardingTable>,
227 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
228 supervisor: SupervisorHandle,
229 subc_capabilities: Arc<[String]>,
230 route_bind_relay_timeout: Duration,
234 route_bind_relay_timeouts: BTreeMap<String, Duration>,
238 route_bind_breakers: RouteBindBreakers,
241 route_bind_concurrency: RouteBindConcurrency,
244 route_bind_breaker_threshold: u32,
246 route_bind_breaker_cooldown: Duration,
248 health_probe_timeout: Duration,
249 storage_config: Option<crate::daemon_config::StorageConfig>,
252 machine_id: Option<crate::machine_id::MachineId>,
256 admission_facts_carrier_module_id: Option<String>,
257 admission_facts_targets: Option<Vec<String>>,
258 rescan: Option<SupervisorRescanContext>,
259 connected_clients: ConnectedClients,
260 counters: DaemonCounters,
261 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
262 daemon_provenance: DaemonProvenanceFacts,
263 #[cfg(test)]
264 control_dispatch_delay: Option<Duration>,
265 #[cfg(test)]
266 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
267}
268
269impl fmt::Debug for ControlHandler {
270 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
271 f.debug_struct("ControlHandler")
272 .field("registry", &self.registry)
273 .field("forwarding", &self.forwarding)
274 .field("process_liveness", &self.process_liveness.is_some())
275 .field("supervisor", &self.supervisor)
276 .field("subc_capabilities", &self.subc_capabilities)
277 .finish()
278 }
279}
280
281struct RouteOpenRequest {
282 target: RouteTarget,
283 identity: BindIdentity,
284 consumer_identity: Option<ConsumerIdentity>,
285 consumer_capabilities: Option<Vec<String>>,
286 admission_facts: Option<serde_json::Value>,
287}
288
289struct RouteBindReservationGuard {
290 forwarding: Arc<ForwardingTable>,
291 endpoint: ModuleEndpointId,
292 relay_corr: u64,
293 armed: bool,
294}
295
296struct ModuleControlRpcGuard {
297 forwarding: Arc<ForwardingTable>,
298 endpoint: ModuleEndpointId,
299 corr: u64,
300 armed: bool,
301}
302
303impl ModuleControlRpcGuard {
304 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
305 Self {
306 forwarding,
307 endpoint,
308 corr,
309 armed: true,
310 }
311 }
312
313 fn disarm(&mut self) {
314 self.armed = false;
315 }
316}
317
318impl Drop for ModuleControlRpcGuard {
319 fn drop(&mut self) {
320 if self.armed {
321 let _ = self
322 .forwarding
323 .cancel_module_control_rpc(self.endpoint, self.corr);
324 }
325 }
326}
327
328impl RouteBindReservationGuard {
329 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
330 Self {
331 forwarding,
332 endpoint,
333 relay_corr,
334 armed: true,
335 }
336 }
337
338 fn release_and_disarm(&mut self) {
339 if !self.armed {
340 return;
341 }
342 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
343 self.endpoint,
344 self.relay_corr,
345 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
346 ) {
347 send_goodbye_target_best_effort(
348 &self.forwarding.counters(),
349 &target,
350 "canceled route.bind",
351 );
352 }
353 self.armed = false;
354 }
355
356 fn disarm(&mut self) {
357 self.armed = false;
358 }
359}
360
361impl Drop for RouteBindReservationGuard {
362 fn drop(&mut self) {
363 self.release_and_disarm();
364 }
365}
366
367#[derive(Debug, Clone, Default)]
397pub(crate) struct RouteBindBreakers {
398 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
399}
400
401#[derive(Debug, Clone, Default)]
402pub(crate) struct RouteBindConcurrency {
403 modules: Arc<Mutex<HashMap<String, usize>>>,
404}
405
406struct RouteBindConcurrencyGuard {
407 concurrency: RouteBindConcurrency,
408 module_id: String,
409}
410
411impl RouteBindConcurrency {
412 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
415 let mut modules = self
416 .modules
417 .lock()
418 .expect("route.bind concurrency mutex poisoned");
419 let in_flight = modules.entry(module_id.to_string()).or_default();
420 if *in_flight >= limit {
421 return Err(*in_flight);
422 }
423 *in_flight += 1;
424 Ok(RouteBindConcurrencyGuard {
425 concurrency: self.clone(),
426 module_id: module_id.to_string(),
427 })
428 }
429}
430
431impl Drop for RouteBindConcurrencyGuard {
432 fn drop(&mut self) {
433 let mut modules = self
434 .concurrency
435 .modules
436 .lock()
437 .expect("route.bind concurrency mutex poisoned");
438 let remove = {
439 let in_flight = modules
440 .get_mut(&self.module_id)
441 .expect("admitted route.bind has a concurrency entry");
442 *in_flight -= 1;
443 *in_flight == 0
444 };
445 if remove {
446 modules.remove(&self.module_id);
447 }
448 }
449}
450
451#[derive(Debug, Default)]
452struct ModuleBreakerState {
453 consecutive_timeouts: u32,
455 cooldown_until: Option<Instant>,
458 probe_in_flight: bool,
463}
464
465enum RouteBindAdmission<'a> {
467 Admitted {
468 guard: RouteBindBreakerGuard<'a>,
469 probe: bool,
472 },
473 Refused {
474 consecutive_timeouts: u32,
475 retry_in: Duration,
479 probe_in_flight: bool,
480 },
481}
482
483struct RouteBindBreakerGuard<'a> {
490 breakers: RouteBindBreakers,
491 module_id: &'a str,
492 settled: bool,
493}
494
495impl RouteBindBreakerGuard<'_> {
496 fn record_accepted(&mut self) -> bool {
500 self.settled = true;
501 self.breakers.record_accepted(self.module_id)
502 }
503
504 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
507 self.settled = true;
508 self.breakers
509 .record_timeout(self.module_id, threshold, cooldown)
510 }
511
512 fn record_inconclusive(&mut self) {
521 self.settled = true;
522 self.breakers.record_inconclusive(self.module_id);
523 }
524}
525
526impl Drop for RouteBindBreakerGuard<'_> {
527 fn drop(&mut self) {
528 if !self.settled {
529 self.breakers.record_inconclusive(self.module_id);
530 }
531 }
532}
533
534struct BreakerOpened {
538 consecutive_timeouts: u32,
539 reopened_after_probe: bool,
542}
543
544impl RouteBindBreakers {
545 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
546 self.modules
547 .lock()
548 .expect("route.bind breaker mutex poisoned")
549 }
550
551 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
554 let admitted = |probe| RouteBindAdmission::Admitted {
555 guard: RouteBindBreakerGuard {
556 breakers: self.clone(),
557 module_id,
558 settled: false,
559 },
560 probe,
561 };
562
563 let mut modules = self.lock();
564 let Some(state) = modules.get_mut(module_id) else {
565 return admitted(false);
566 };
567 let Some(cooldown_until) = state.cooldown_until else {
568 return admitted(false);
569 };
570 if state.probe_in_flight {
571 return RouteBindAdmission::Refused {
572 consecutive_timeouts: state.consecutive_timeouts,
573 retry_in: Duration::ZERO,
574 probe_in_flight: true,
575 };
576 }
577 let now = Instant::now();
578 if now < cooldown_until {
579 return RouteBindAdmission::Refused {
580 consecutive_timeouts: state.consecutive_timeouts,
581 retry_in: cooldown_until - now,
582 probe_in_flight: false,
583 };
584 }
585 state.probe_in_flight = true;
586 admitted(true)
587 }
588
589 fn record_accepted(&self, module_id: &str) -> bool {
590 self.lock()
591 .remove(module_id)
592 .is_some_and(|state| state.cooldown_until.is_some())
593 }
594
595 fn record_timeout(
596 &self,
597 module_id: &str,
598 threshold: u32,
599 cooldown: Duration,
600 ) -> Option<BreakerOpened> {
601 let mut modules = self.lock();
602 let state = modules.entry(module_id.to_string()).or_default();
603 let was_open = state.cooldown_until.is_some();
604 let was_probe = state.probe_in_flight;
605 state.probe_in_flight = false;
606 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
607 if state.consecutive_timeouts < threshold {
608 return None;
609 }
610 state.cooldown_until = Some(Instant::now() + cooldown);
611 Some(BreakerOpened {
612 consecutive_timeouts: state.consecutive_timeouts,
613 reopened_after_probe: was_open && was_probe,
614 })
615 }
616
617 fn record_inconclusive(&self, module_id: &str) {
618 if let Some(state) = self.lock().get_mut(module_id) {
619 state.probe_in_flight = false;
620 }
621 }
622
623 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
638 self.lock()
639 .remove(module_id)
640 .map(|state| state.consecutive_timeouts)
641 .filter(|discarded| *discarded > 0)
642 }
643
644 fn open_snapshot(&self) -> Option<serde_json::Value> {
651 let now = Instant::now();
652 let modules = self.lock();
653 let open = modules
654 .iter()
655 .filter_map(|(module_id, state)| {
656 let cooldown_until = state.cooldown_until?;
657 Some((
658 module_id.clone(),
659 serde_json::json!({
660 "consecutive_timeouts": state.consecutive_timeouts,
661 "cooldown_remaining_ms":
662 cooldown_until.saturating_duration_since(now).as_millis() as u64,
663 "probe_in_flight": state.probe_in_flight,
664 }),
665 ))
666 })
667 .collect::<serde_json::Map<String, serde_json::Value>>();
668 (!open.is_empty()).then_some(serde_json::Value::Object(open))
669 }
670}
671
672impl ControlHandler {
673 pub fn new(registry: Arc<Registry>) -> Self {
674 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
675 }
676
677 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
678 let counters = forwarding.counters();
679 let route_bind_breakers = forwarding.route_bind_breakers();
683 let route_bind_concurrency = forwarding.route_bind_concurrency();
684 Self {
685 registry,
686 forwarding,
687 process_liveness: None,
688 supervisor: SupervisorHandle::new(),
689 subc_capabilities: Arc::from([
690 CAP_MANIFEST_REGISTRATION.to_string(),
691 CAP_CHANNEL_LIFECYCLE.to_string(),
692 CAP_PING_PONG.to_string(),
693 CAP_SESSION_ATTACH.to_string(),
694 CAP_ADMISSION_FACTS_RELAY.to_string(),
695 ]),
696 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
697 route_bind_relay_timeouts: BTreeMap::new(),
698 route_bind_breakers,
699 route_bind_concurrency,
700 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
701 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
702 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
703 storage_config: None,
704 machine_id: None,
705 admission_facts_carrier_module_id: None,
706 admission_facts_targets: None,
707 rescan: None,
708 connected_clients: ConnectedClients::new(),
709 counters,
710 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
711 daemon_provenance: DaemonProvenanceFacts::default(),
712 #[cfg(test)]
713 control_dispatch_delay: None,
714 #[cfg(test)]
715 provenance_probe_override: None,
716 }
717 }
718
719 pub fn with_storage_config(
722 mut self,
723 storage_config: Option<crate::daemon_config::StorageConfig>,
724 ) -> Self {
725 self.storage_config = storage_config;
726 self
727 }
728
729 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
732 self.machine_id = machine_id;
733 self
734 }
735
736 pub fn with_admission_facts_config(
740 mut self,
741 carrier_module_id: Option<String>,
742 targets: Option<Vec<String>>,
743 ) -> Self {
744 self.admission_facts_carrier_module_id = carrier_module_id;
745 self.admission_facts_targets = targets;
746 self
747 }
748
749 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
752 self.route_bind_relay_timeout = timeout;
753 self
754 }
755
756 pub fn with_route_bind_relay_timeouts(
762 mut self,
763 timeouts: impl IntoIterator<Item = (String, Duration)>,
764 ) -> Self {
765 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
766 self
767 }
768
769 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
775 self.route_bind_relay_timeouts
776 .get(module_id)
777 .copied()
778 .unwrap_or(self.route_bind_relay_timeout)
779 }
780
781 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
789 self.route_bind_breaker_threshold = threshold.max(1);
790 self.route_bind_breaker_cooldown = cooldown;
791 self
792 }
793
794 #[cfg(test)]
795 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
796 self.health_probe_timeout = timeout;
797 self
798 }
799
800 #[cfg(test)]
801 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
802 self.control_dispatch_delay = Some(delay);
803 self
804 }
805
806 pub fn with_process_liveness(
807 mut self,
808 process_liveness: Arc<dyn ModuleProcessLiveness>,
809 ) -> Self {
810 self.process_liveness = Some(process_liveness);
811 self
812 }
813
814 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
815 self.supervisor = supervisor;
816 self
817 }
818
819 pub fn with_daemon_provenance(
820 mut self,
821 pid: u32,
822 started_at_ms: u64,
823 executable_path: Option<PathBuf>,
824 build_git_sha: Option<String>,
825 build_lock_digest: Option<String>,
826 ) -> Self {
827 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
828 let process_start_time = process_start_time(pid);
829 self.daemon_provenance = DaemonProvenanceFacts {
830 build: DaemonBuildProvenance {
831 build_git_sha,
832 build_lock_digest,
833 },
834 pid: Some(pid),
835 started_at_ms: Some(started_at_ms),
836 start_clock: None,
837 executable_path,
838 executable_identity,
839 process_start_time,
840 probe: ExecutableIdentityProbe::default(),
841 };
842 self
843 }
844
845 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
846 self.daemon_provenance.start_clock = Some(clock);
847 self
848 }
849
850 #[cfg(test)]
851 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
852 self.provenance_probe_override = Some(result);
853 self
854 }
855
856 pub fn with_capability_config(
860 self,
861 modules: impl IntoIterator<Item = (String, bool)>,
862 reserved_capabilities: BTreeMap<String, String>,
863 ) -> Self {
864 self.capability_evaluator
865 .configure(modules, reserved_capabilities);
866 self
867 }
868
869 pub fn with_supervisor_rescan(
870 mut self,
871 supervisor: Supervisor,
872 config_path: impl Into<PathBuf>,
873 configured_port: Option<u16>,
874 ) -> Self {
875 self.rescan = Some(SupervisorRescanContext {
876 supervisor,
877 config_path: config_path.into(),
878 configured_port,
879 storage_config: self.storage_config.clone(),
880 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
881 admission_facts_targets: self.admission_facts_targets.clone(),
882 });
883 self
884 }
885
886 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
887 self.connected_clients = connected_clients;
888 self
889 }
890
891 pub fn forwarding(&self) -> Arc<ForwardingTable> {
892 Arc::clone(&self.forwarding)
893 }
894
895 pub(crate) fn counters(&self) -> DaemonCounters {
896 self.counters.clone()
897 }
898
899 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
902 tokio::spawn(async move {
903 loop {
904 self.capability_evaluator
905 .wait_for_change_or_deadline()
906 .await;
907 self.refresh_capability_requirements();
908 }
909 });
910 }
911
912 fn runtime_capability_snapshot(
913 &self,
914 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
915 let runtime = self
916 .supervisor
917 .list()
918 .into_iter()
919 .map(|module| {
920 let status = module.status().map_err(|err| {
921 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
922 })?;
923 Ok(RuntimeModule {
924 module_id: status.module_id,
925 state: status.state,
926 enabled: status.enabled,
927 })
928 })
929 .collect::<Result<Vec<_>, RouterError>>()?;
930 let (_, registrations) = self.registry.list_modules().map_err(|err| {
931 RouterError::backend(
932 0,
933 0,
934 format!("failed to list capability registrations: {err}"),
935 )
936 })?;
937 let registrations = registrations
938 .into_iter()
939 .map(|registration| RegisteredModule {
940 module_id: registration.manifest.module_id,
941 module_version: registration.manifest.module_version,
942 capabilities: registration.manifest.capabilities,
943 })
944 .collect();
945 Ok((runtime, registrations))
946 }
947
948 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
955 let cached_registration = RegisteredModule {
956 module_id: registration.manifest.module_id.clone(),
957 module_version: registration.manifest.module_version.clone(),
958 capabilities: registration.manifest.capabilities.clone(),
959 };
960 if self.capability_evaluator.record_hello(&cached_registration) {
961 warn!(
962 module_id = %cached_registration.module_id,
963 "capability claims drifted from the cached manifest"
964 );
965 }
966 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
967 self.enforce_capability_denies();
968 }
969 self.refresh_capability_requirements();
970 }
971
972 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
976 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
977 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
978 self.supervisor.set_swap_promotion_observer(observer);
979 }
980
981 pub fn refresh_capability_requirements(&self) {
982 match self.runtime_capability_snapshot() {
983 Ok((runtime, registrations)) => {
984 log_requirement_events(
985 self.capability_evaluator
986 .evaluate_now(&runtime, ®istrations),
987 );
988 }
989 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
990 }
991 }
992
993 fn enforce_capability_denies(&self) {
997 let (_, registrations) = match self.registry.list_modules() {
998 Ok(snapshot) => snapshot,
999 Err(err) => {
1000 warn!(error = %err, "failed to read registrations for capability deny census");
1001 return;
1002 }
1003 };
1004 let manifests = registrations
1005 .into_iter()
1006 .map(|registration| {
1007 (
1008 registration.manifest.module_id.clone(),
1009 registration.manifest,
1010 )
1011 })
1012 .collect::<BTreeMap<_, _>>();
1013 let census = match self.forwarding.route_census(None) {
1014 Ok(census) => census,
1015 Err(err) => {
1016 warn!(error = %err, "failed to read route census for capability deny enforcement");
1017 return;
1018 }
1019 };
1020
1021 for (target_module_id, routes) in census {
1022 let Some(target_manifest) = manifests.get(&target_module_id) else {
1023 continue;
1024 };
1025 let mut closed_routes = Vec::new();
1026 let mut module_goodbyes = Vec::new();
1027 for route in routes {
1028 let Principal::Reserved {
1029 module_id: opening_module_id,
1030 } = &route.principal
1031 else {
1032 continue;
1033 };
1034 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1035 continue;
1036 };
1037 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1038 continue;
1039 };
1040
1041 match self.forwarding.release_client_route(
1042 route.goodbye_target.connection_id,
1043 route.goodbye_target.channel,
1044 route.goodbye_target.epoch,
1045 ) {
1046 Ok(RouteRelease::Removed(module_goodbye)) => {
1047 warn!(
1048 opening_module_id,
1049 target_module_id,
1050 capability,
1051 "force-closing route because an attested capability deny edge now matches"
1052 );
1053 closed_routes.push(route);
1054 module_goodbyes.push(module_goodbye);
1055 }
1056 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1057 Err(err) => warn!(
1058 opening_module_id,
1059 target_module_id,
1060 capability,
1061 error = %err,
1062 "failed to force-close capability-denied route"
1063 ),
1064 }
1065 }
1066
1067 if closed_routes.is_empty() {
1068 continue;
1069 }
1070 send_route_control_pushes(
1071 &self.forwarding,
1072 closed_routes,
1073 ClientControlPush::RouteClosed {
1074 module_id: target_module_id,
1075 reason: RouteCloseReason::CapabilityDenied,
1076 drained: false,
1077 abandoned: 0,
1078 excluded_subscriptions: 0,
1079 terminal: Some(false),
1080 },
1081 );
1082 self.emit_route_goodbyes(module_goodbyes);
1083 }
1084 }
1085
1086 fn not_ready_reason(
1092 &self,
1093 registration: &crate::registry::ModuleRegistration,
1094 ) -> Option<NotReadyReason> {
1095 if !registration.ready {
1096 return Some(NotReadyReason {
1097 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1098 capability: None,
1099 });
1100 }
1101 self.first_unprovided_required_capability(registration)
1102 .map(|capability| NotReadyReason {
1103 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1104 capability: Some(capability),
1105 })
1106 }
1107
1108 fn first_unprovided_required_capability(
1126 &self,
1127 registration: &crate::registry::ModuleRegistration,
1128 ) -> Option<String> {
1129 let required = registration
1130 .manifest
1131 .capabilities
1132 .iter()
1133 .flat_map(|declarations| declarations.requires.iter())
1134 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1135 .map(|requirement| requirement.capability.as_str())
1136 .collect::<BTreeSet<_>>();
1137 if required.is_empty() {
1138 return None;
1139 }
1140 let module_id = registration.manifest.module_id.as_str();
1141 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1142 if required
1143 .iter()
1144 .any(|capability| verdict(capability).is_none())
1145 {
1146 self.refresh_capability_requirements();
1147 }
1148 required
1149 .into_iter()
1150 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1151 .map(str::to_string)
1152 }
1153
1154 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1155 self.capability_evaluator
1156 .statuses()
1157 .into_iter()
1158 .map(capability_requirement_status)
1159 .collect()
1160 }
1161
1162 fn deregister_connection(
1169 &self,
1170 connection_id: ConnectionId,
1171 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1172 self.registry.deregister_connection(connection_id)
1173 }
1174
1175 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1176 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1177 return None;
1178 }
1179 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1180 parse_client_control_request(&frame.body)
1181 else {
1182 return None;
1183 };
1184 Some(target_module_id(&target).to_string())
1185 }
1186
1187 pub(crate) fn route_open_capacity_refusal(
1188 &self,
1189 ctx: &RouteCtx,
1190 frame: &Frame,
1191 target_module_id: &str,
1192 limit: usize,
1193 ) -> Result<Frame, RouterError> {
1194 self.route_open_admission_refusal_frame(
1195 ctx,
1196 frame,
1197 target_module_id,
1198 format!(
1199 "connection already has {limit} route.open binds in flight; retry after one settles"
1200 ),
1201 )
1202 }
1203
1204 fn route_open_admission_refusal_frame(
1212 &self,
1213 ctx: &RouteCtx,
1214 frame: &Frame,
1215 target_module_id: &str,
1216 message: impl Into<String>,
1217 ) -> Result<Frame, RouterError> {
1218 self.route_open_refusal_frame(
1219 ctx,
1220 frame,
1221 target_module_id,
1222 "open_admission_full",
1223 error_codes::TARGET_UNAVAILABLE,
1224 message,
1225 )
1226 }
1227
1228 #[cfg(test)]
1234 pub fn handle_control(
1235 &self,
1236 connection_id: ConnectionId,
1237 frame: Frame,
1238 ) -> Result<Vec<Frame>, RouterError> {
1239 match frame.header.ty {
1240 FrameType::Ping => Ok(vec![pong(&frame)?]),
1241 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1242 FrameType::Goodbye => self.handle_goodbye(connection_id),
1243 ty => Ok(vec![control_error_frame(
1244 &frame,
1245 "unsupported_control_frame",
1246 format!("unsupported channel-0 frame {ty:?}"),
1247 )?]),
1248 }
1249 }
1250
1251 pub async fn handle_control_frame(
1252 &self,
1253 ctx: &RouteCtx,
1254 frame: Frame,
1255 ) -> Result<Vec<Frame>, RouterError> {
1256 self.handle_control_frame_timed(ctx, frame, None).await
1257 }
1258
1259 pub(crate) async fn handle_control_frame_timed(
1260 &self,
1261 ctx: &RouteCtx,
1262 frame: Frame,
1263 dispatch_started_at: Option<StdInstant>,
1264 ) -> Result<Vec<Frame>, RouterError> {
1265 match frame.header.ty {
1266 FrameType::Ping => Ok(vec![pong(&frame)?]),
1267 FrameType::Hello => {
1268 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1269 }
1270 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1271 FrameType::Cancel => {
1272 if self
1273 .supervisor
1274 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1275 {
1276 Ok(Vec::new())
1277 } else {
1278 Ok(vec![control_error_frame(
1279 &frame,
1280 "unknown_subscription",
1281 "no supervisor spawn subscription has this correlation id",
1282 )?])
1283 }
1284 }
1285 FrameType::Request => {
1286 if self
1287 .forwarding
1288 .module_endpoint_for_connection(ctx.connection_id)
1289 .map_err(RouterError::Forwarding)?
1290 .is_some()
1291 {
1292 if !is_known_module_request_op(&frame.body) {
1293 return Ok(vec![control_error_frame(
1294 &frame,
1295 "unsupported_control_frame",
1296 "module-originated channel-0 REQUEST is not supported",
1297 )?]);
1298 }
1299 let request = match parse_module_control_request_from_module(&frame.body) {
1300 Ok(request) => request,
1301 Err((err, ControlRequestBodyError::UnknownOp)) => {
1302 return Ok(vec![control_error_frame(
1303 &frame,
1304 "unsupported_control_frame",
1305 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1306 )?])
1307 }
1308 Err((err, ControlRequestBodyError::InvalidBody)) => {
1309 return Ok(vec![control_error_frame(
1310 &frame,
1311 "invalid_control_body",
1312 format!("malformed module control body: {err}"),
1313 )?])
1314 }
1315 };
1316 let op = module_control_request_op(&request);
1317 let corr = frame.header.corr;
1318 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1319 let result =
1320 self.handle_module_control_request(ctx.connection_id, frame, request);
1321 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1322 return result;
1323 }
1324
1325 if is_known_module_request_op(&frame.body) {
1326 return Ok(vec![control_error_frame(
1327 &frame,
1328 "not_registered",
1329 "catalog.update requires an active module registration owned by this connection",
1330 )?]);
1331 }
1332
1333 let request = match parse_client_control_request(&frame.body) {
1334 Ok(request) => request,
1335 Err((err, ControlRequestBodyError::UnknownOp)) => {
1336 return Ok(vec![control_error_frame(
1337 &frame,
1338 "unknown_control_op",
1339 format!("unknown client control op: {err}"),
1340 )?])
1341 }
1342 Err((err, ControlRequestBodyError::InvalidBody)) => {
1343 return Ok(vec![control_error_frame(
1344 &frame,
1345 "invalid_control_body",
1346 format!("malformed client control body: {err}"),
1347 )?])
1348 }
1349 };
1350 let op = client_control_request_op(&request);
1351 let corr = frame.header.corr;
1352 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1353 #[cfg(test)]
1354 if let Some(delay) = self.control_dispatch_delay {
1355 tokio::time::sleep(delay).await;
1356 }
1357 let result = self
1358 .handle_client_control_request(ctx, frame, request)
1359 .await;
1360 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1361 result
1362 }
1363 FrameType::Push => {
1364 let Some(endpoint) = self
1365 .forwarding
1366 .module_endpoint_for_connection(ctx.connection_id)
1367 .map_err(RouterError::Forwarding)?
1368 else {
1369 return Ok(vec![control_error_frame(
1370 &frame,
1371 "unsupported_control_frame",
1372 "client-originated channel-0 PUSH is not supported",
1373 )?]);
1374 };
1375 self.handle_status_update(endpoint, frame)
1376 }
1377 FrameType::Response | FrameType::Error
1378 if self
1379 .forwarding
1380 .module_endpoint_for_connection(ctx.connection_id)
1381 .map_err(RouterError::Forwarding)?
1382 .is_some() =>
1383 {
1384 self.handle_module_relay_response(ctx.connection_id, frame)
1385 }
1386 ty => Ok(vec![control_error_frame(
1387 &frame,
1388 "unsupported_control_frame",
1389 format!("unsupported channel-0 frame {ty:?}"),
1390 )?]),
1391 }
1392 }
1393
1394 pub fn cleanup_connection(
1395 &self,
1396 connection_id: ConnectionId,
1397 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1398 let crash_closed = self
1399 .registry
1400 .get_module_by_connection(connection_id)?
1401 .and_then(|registration| {
1402 self.forwarding
1403 .module_endpoint_for_connection(connection_id)
1404 .ok()
1405 .flatten()
1406 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1407 .map(|routes| (registration.manifest.module_id, routes))
1408 });
1409 if let Some((module_id, routes)) = crash_closed {
1410 let terminal = match self.supervisor.get(&module_id) {
1411 None => false,
1412 Some(module) => match module.will_recover_after_connection_loss() {
1413 Ok(will_recover) => !will_recover,
1414 Err(err) => {
1415 warn!(
1416 %module_id,
1417 error = %err,
1418 "failed to read crash recovery verdict; reporting non-terminal conservatively"
1419 );
1420 false
1421 }
1422 },
1423 };
1424 send_route_control_pushes(
1425 &self.forwarding,
1426 routes,
1427 ClientControlPush::RouteClosed {
1428 module_id,
1429 reason: RouteCloseReason::Crash,
1430 drained: false,
1431 abandoned: 0,
1432 excluded_subscriptions: 0,
1433 terminal: Some(terminal),
1434 },
1435 );
1436 }
1437 let registrations = self.deregister_connection(connection_id);
1438 if let Ok(released_routes) = self.forwarding.cleanup_connection(connection_id) {
1439 self.emit_route_goodbyes(released_routes);
1440 }
1441 if matches!(®istrations, Ok(r) if !r.is_empty()) {
1445 crate::supervise::notify_registration_release();
1446 self.capability_evaluator.wake_deadline_loop();
1447 self.refresh_capability_requirements();
1448 }
1449 self.supervisor.remove_spawn_subscribers(connection_id);
1450 registrations
1451 }
1452
1453 pub(crate) fn handle_route_goodbye(
1454 &self,
1455 connection_id: ConnectionId,
1456 route_channel: u16,
1457 route_epoch: u32,
1458 ) -> Result<bool, RouterError> {
1459 debug!(
1460 connection_id = connection_id.get(),
1461 route_channel, route_epoch, "handling route GOODBYE"
1462 );
1463 let RouteRelease::Removed(released_route) = self
1464 .forwarding
1465 .release_client_route(connection_id, route_channel, route_epoch)
1466 .map_err(RouterError::Forwarding)?
1467 else {
1468 return Ok(false);
1469 };
1470 self.emit_route_goodbyes(vec![released_route]);
1471 Ok(true)
1472 }
1473
1474 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
1475 for released in released_routes {
1476 let frame = match Frame::build_with_version(
1477 released.negotiated_ver,
1478 FrameType::Goodbye,
1479 control_flags(),
1480 released.channel,
1481 released.epoch,
1482 0,
1483 Vec::new(),
1484 ) {
1485 Ok(frame) => frame,
1486 Err(err) => {
1487 warn!(
1488 route_channel = released.channel,
1489 error = %err,
1490 "failed to build route GOODBYE frame"
1491 );
1492 continue;
1493 }
1494 };
1495 if !released.close_on_delivery_failure() {
1496 crate::forwarding::send_module_route_goodbye(
1497 &self.counters,
1498 &released.sink,
1499 frame,
1500 released.module_id.as_deref(),
1501 "client route released",
1502 );
1503 continue;
1504 }
1505 if let Err(err) = released.sink.try_send(frame) {
1506 warn!(
1507 target_connection_id = released.connection_id.get(),
1508 route_channel = released.channel,
1509 error = %err,
1510 "route GOODBYE was not delivered to client; closing target connection"
1511 );
1512 if self
1513 .forwarding
1514 .escalate_client_delivery_failure(
1515 released.connection_id,
1516 released.channel,
1517 released.epoch,
1518 CloseReason::new(
1519 "route_goodbye_delivery_failed",
1520 format!(
1521 "failed to enqueue route GOODBYE for channel {}: {err}",
1522 released.channel
1523 ),
1524 ),
1525 crate::forwarding::UndeliveredFrame {
1526 module_id: released.module_id.as_deref(),
1527 sink: &released.sink,
1528 },
1529 )
1530 .unwrap_or(false)
1531 {
1532 self.counters.increment_goodbye_relay_client_failed();
1533 }
1534 }
1535 }
1536 }
1537
1538 fn send_abandoned_route_bind_goodbye(
1550 &self,
1551 module_sink: &crate::FrameSink,
1552 negotiated_ver: u8,
1553 module_channel: u16,
1554 module_epoch: u32,
1555 ) {
1556 let frame = match Frame::build_with_version(
1557 negotiated_ver,
1558 FrameType::Goodbye,
1559 control_flags(),
1560 module_channel,
1561 module_epoch,
1562 0,
1563 Vec::new(),
1564 ) {
1565 Ok(frame) => frame,
1566 Err(err) => {
1567 warn!(
1568 route_channel = module_channel,
1569 error = %err,
1570 "failed to build GOODBYE for abandoned route.bind"
1571 );
1572 return;
1573 }
1574 };
1575 crate::forwarding::send_module_route_goodbye(
1576 &self.counters,
1577 module_sink,
1578 frame,
1579 None,
1580 "abandoned route.bind",
1581 );
1582 }
1583
1584 fn handle_hello(
1585 &self,
1586 connection_id: ConnectionId,
1587 sink: Option<crate::FrameSink>,
1588 frame: Frame,
1589 ) -> Result<Vec<Frame>, RouterError> {
1590 debug!(
1591 connection_id = connection_id.get(),
1592 corr = frame.header.corr,
1593 "handling HELLO"
1594 );
1595 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
1596 Ok(value) => value,
1597 Err(err) => {
1598 return Ok(vec![control_error_frame(
1599 &frame,
1600 "invalid_hello",
1601 format!("malformed HELLO body: {err}"),
1602 )?])
1603 }
1604 };
1605 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
1606 return Ok(vec![control_error_frame(
1607 &frame,
1608 "invalid_capability_grammar",
1609 err.to_string(),
1610 )?]);
1611 }
1612 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
1613 return Ok(vec![control_error_frame(
1614 &frame,
1615 "invalid_manifest",
1616 err.to_string(),
1617 )?]);
1618 }
1619 if let Some(provenance) = hello_value
1620 .get("manifest")
1621 .and_then(|manifest| manifest.get("provenance"))
1622 {
1623 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
1624 return Ok(vec![control_error_frame(
1625 &frame,
1626 "invalid_manifest",
1627 format!("malformed manifest provenance: {err}"),
1628 )?]);
1629 }
1630 }
1631 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
1632 Ok(hello) => hello,
1633 Err(err) => {
1634 return Ok(vec![control_error_frame(
1635 &frame,
1636 "invalid_hello",
1637 format!("malformed HELLO body: {err}"),
1638 )?])
1639 }
1640 };
1641
1642 if hello.protocol_ver != hello.manifest.protocol_ver {
1643 return Ok(vec![control_error_frame(
1644 &frame,
1645 "invalid_manifest",
1646 format!(
1647 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
1648 hello.protocol_ver, hello.manifest.protocol_ver
1649 ),
1650 )?]);
1651 }
1652
1653 if hello.manifest.module_id.trim().is_empty() {
1654 return Ok(vec![control_error_frame(
1655 &frame,
1656 "invalid_manifest",
1657 "manifest module_id must not be empty",
1658 )?]);
1659 }
1660
1661 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
1662 Ok(negotiated_ver) => negotiated_ver,
1663 Err(message) => {
1664 return Ok(vec![control_error_frame(
1665 &frame,
1666 "version_unsupported",
1667 message,
1668 )?])
1669 }
1670 };
1671
1672 let swap_admission = self
1680 .supervisor
1681 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
1682 if swap_admission == SwapHelloAdmission::Refused {
1683 warn!(
1684 module_id = %hello.manifest.module_id,
1685 connection_id = connection_id.get(),
1686 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
1687 );
1688 return Ok(vec![control_error_frame(
1689 &frame,
1690 "swap_token_invalid",
1691 format!(
1692 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
1693 hello.manifest.module_id
1694 ),
1695 )?]);
1696 }
1697 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
1698
1699 if let Some(rejection) = (!swap_candidate)
1707 .then(|| {
1708 self.supervisor.reserved_hello_rejection(
1709 &hello.manifest.module_id,
1710 hello.launch_nonce.as_deref(),
1711 )
1712 })
1713 .flatten()
1714 {
1715 let message = match rejection {
1716 ReservedHelloRejection::Exact { module_id } => format!(
1717 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
1718 ),
1719 ReservedHelloRejection::Prefix {
1720 prefix,
1721 owner_module_id,
1722 } => format!(
1723 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
1724 hello.manifest.module_id
1725 ),
1726 };
1727 return Ok(vec![control_error_frame(
1728 &frame,
1729 "reserved_module",
1730 message,
1731 )?]);
1732 }
1733
1734 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
1735 &hello.manifest.module_id,
1736 hello.manifest.capabilities.as_ref(),
1737 );
1738 if let Some(refusal) = reserved_capability_refusals.first() {
1739 let capability = refusal.capability.clone();
1740 let bound_module = refusal.claimants[0].clone();
1741 log_duplicate_claim_events(reserved_capability_refusals);
1742 return Ok(vec![control_error_frame(
1743 &frame,
1744 "reserved_capability",
1745 format!(
1746 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
1747 capability, bound_module, hello.manifest.module_id
1748 ),
1749 )?]);
1750 }
1751
1752 if self
1755 .forwarding
1756 .connection_has_client_routes(connection_id)
1757 .map_err(RouterError::Forwarding)?
1758 {
1759 return Ok(vec![control_error_frame(
1760 &frame,
1761 "invalid_hello",
1762 "connection has open client routes and cannot also register as a module",
1763 )?]);
1764 }
1765
1766 let control_ops = effective_module_control_ops(hello.control_ops);
1767 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
1770 if swap_candidate {
1771 return self.register_swap_candidate(
1772 connection_id,
1773 sink,
1774 &frame,
1775 hello.manifest,
1776 negotiated_ver,
1777 control_ops,
1778 hello_ack,
1779 );
1780 }
1781 let registration = match self.registry.register_with_control_ops(
1782 hello.manifest,
1783 negotiated_ver,
1784 connection_id,
1785 control_ops,
1786 ) {
1787 Ok(registration) => registration,
1788 Err(RegistryError::DuplicateModuleId { module_id }) => {
1789 return Ok(vec![control_error_frame(
1790 &frame,
1791 "duplicate_module_id",
1792 format!(
1793 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
1794 ),
1795 )?])
1796 }
1797 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
1798 return Ok(vec![control_error_frame(
1799 &frame,
1800 "invalid_module_id",
1801 err.to_string(),
1802 )?])
1803 }
1804 Err(err) => {
1805 return Ok(vec![control_error_frame(
1806 &frame,
1807 "registry_error",
1808 err.to_string(),
1809 )?])
1810 }
1811 };
1812
1813 let reply = if let Some(sink) = sink {
1814 let concurrency = manifest_concurrency(®istration.manifest);
1830 if let Err(err) = self.forwarding.register_module_connection_acked(
1831 connection_id,
1832 registration.manifest.module_id.clone(),
1833 negotiated_ver,
1834 concurrency,
1835 sink,
1836 hello_ack,
1837 ) {
1838 if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
1842 crate::supervise::notify_registration_release();
1843 }
1844 return Ok(vec![control_error_frame(
1845 &frame,
1846 forwarding_error_code(&err),
1847 err.to_string(),
1848 )?]);
1849 }
1850 Vec::new()
1851 } else {
1852 vec![hello_ack]
1855 };
1856
1857 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
1866 info!(
1867 module_id = %registration.manifest.module_id,
1868 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
1869 );
1870 }
1871
1872 self.apply_registration_capabilities(®istration);
1873
1874 info!(
1875 module_id = %registration.manifest.module_id,
1876 module_version = %registration.manifest.module_version,
1877 negotiated_ver,
1878 routable_provider = manifest_provides_routable_role(®istration.manifest),
1879 connection_id = connection_id.get(),
1880 "module registered"
1881 );
1882
1883 Ok(reply)
1884 }
1885
1886 #[allow(clippy::too_many_arguments)]
1896 fn register_swap_candidate(
1897 &self,
1898 connection_id: ConnectionId,
1899 sink: Option<crate::FrameSink>,
1900 frame: &Frame,
1901 manifest: ModuleManifest,
1902 negotiated_ver: u8,
1903 control_ops: Vec<String>,
1904 hello_ack: Frame,
1905 ) -> Result<Vec<Frame>, RouterError> {
1906 let module_id = manifest.module_id.clone();
1907 let registration = match self.registry.register_candidate_with_control_ops(
1908 manifest,
1909 negotiated_ver,
1910 connection_id,
1911 control_ops,
1912 ) {
1913 Ok(registration) => registration,
1914 Err(RegistryError::DuplicateModuleId { module_id }) => {
1915 return Ok(vec![control_error_frame(
1916 frame,
1917 "duplicate_module_id",
1918 format!(
1919 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
1920 ),
1921 )?])
1922 }
1923 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
1924 return Ok(vec![control_error_frame(
1925 frame,
1926 "invalid_module_id",
1927 err.to_string(),
1928 )?])
1929 }
1930 Err(err) => {
1931 return Ok(vec![control_error_frame(
1932 frame,
1933 "registry_error",
1934 err.to_string(),
1935 )?])
1936 }
1937 };
1938 let reply = if let Some(sink) = sink {
1939 let concurrency = manifest_concurrency(®istration.manifest);
1943 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
1944 connection_id,
1945 module_id.clone(),
1946 negotiated_ver,
1947 concurrency,
1948 sink,
1949 hello_ack,
1950 ) {
1951 if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
1952 crate::supervise::notify_registration_release();
1953 }
1954 return Ok(vec![control_error_frame(
1955 frame,
1956 forwarding_error_code(&err),
1957 err.to_string(),
1958 )?]);
1959 }
1960 Vec::new()
1961 } else {
1962 vec![hello_ack]
1963 };
1964 self.supervisor.mark_swap_candidate_admitted(&module_id);
1965 info!(
1966 module_id = %module_id,
1967 module_version = %registration.manifest.module_version,
1968 negotiated_ver,
1969 ready = registration.ready,
1970 connection_id = connection_id.get(),
1971 "swap candidate registered; not routable until cutover"
1972 );
1973 Ok(reply)
1974 }
1975
1976 fn build_hello_ack(
1977 &self,
1978 frame: &Frame,
1979 negotiated_ver: u8,
1980 module_id: &str,
1981 ) -> Result<Frame, RouterError> {
1982 let ack = ModuleHelloAckBody {
1983 negotiated_ver,
1984 subc_ops: module_subc_ops(),
1985 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
1986 storage: self
1987 .storage_config
1988 .as_ref()
1989 .map(|cfg| cfg.descriptor_for(module_id)),
1990 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
1991 };
1992 let body = serde_json::to_vec(&ack).map_err(|err| {
1993 RouterError::backend(
1994 0,
1995 frame.header.corr,
1996 format!("failed to encode HELLO_ACK: {err}"),
1997 )
1998 })?;
1999
2000 Frame::build_with_version(
2001 negotiated_ver,
2002 FrameType::HelloAck,
2003 control_flags(),
2004 0,
2005 0,
2006 frame.header.corr,
2007 body,
2008 )
2009 .map_err(RouterError::FrameBuild)
2010 }
2011
2012 async fn handle_client_control_request(
2013 &self,
2014 ctx: &RouteCtx,
2015 frame: Frame,
2016 request: ClientControlRequest,
2017 ) -> Result<Vec<Frame>, RouterError> {
2018 match request {
2019 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2020 ClientControlRequest::CatalogList { module_id } => {
2021 self.handle_catalog_list(frame, module_id)
2022 }
2023 ClientControlRequest::RouteOpen {
2024 target,
2025 identity,
2026 consumer_identity,
2027 consumer_capabilities,
2028 admission_facts,
2029 } => {
2030 self.handle_route_open(
2031 ctx,
2032 frame,
2033 RouteOpenRequest {
2034 target,
2035 identity,
2036 consumer_identity,
2037 consumer_capabilities,
2038 admission_facts,
2039 },
2040 )
2041 .await
2042 }
2043 ClientControlRequest::RoutePoll {
2044 route_channel,
2045 route_epoch,
2046 kind,
2047 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2048 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2049 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2050 self.handle_supervisor_spawn_snapshot(frame)
2051 }
2052 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2053 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2054 }
2055 ClientControlRequest::SupervisorRestart {
2056 module_id,
2057 drain_timeout_ms,
2058 } => {
2059 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms)
2060 .await
2061 }
2062 ClientControlRequest::SupervisorSwap {
2063 module_id,
2064 ready_timeout_ms,
2065 } => {
2066 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms)
2067 .await
2068 }
2069 ClientControlRequest::SupervisorReload { module_id } => {
2070 self.handle_supervisor_reload(frame, module_id).await
2071 }
2072 ClientControlRequest::SupervisorRescan { preview } => {
2073 self.handle_supervisor_rescan(frame, preview).await
2074 }
2075 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2076 self.handle_supervisor_release_reserved(frame, module_id)
2077 .await
2078 }
2079 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2080 self.handle_supervisor_set_enabled(frame, module_id, enabled)
2081 .await
2082 }
2083 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2084 self.handle_supervisor_health_probe(frame, module_id).await
2085 }
2086 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2087 ClientControlRequest::SupervisorRoutes { module_id } => {
2088 self.handle_supervisor_routes(frame, module_id)
2089 }
2090 ClientControlRequest::SupervisorProvenance { module_id } => {
2091 self.handle_supervisor_provenance(frame, module_id).await
2092 }
2093 ClientControlRequest::SupervisorStderrTail {
2094 module_id,
2095 max_lines,
2096 max_bytes,
2097 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2098 ClientControlRequest::SupervisorTerminals { module_id } => {
2099 self.handle_supervisor_terminals(frame, module_id).await
2100 }
2101 }
2102 }
2103
2104 fn handle_module_control_request(
2105 &self,
2106 connection_id: ConnectionId,
2107 frame: Frame,
2108 request: ModuleControlRequestFromModule,
2109 ) -> Result<Vec<Frame>, RouterError> {
2110 match request {
2111 ModuleControlRequestFromModule::CatalogUpdate {
2112 provides,
2113 capabilities,
2114 ready,
2115 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2116 ModuleControlRequestFromModule::LiveRoots {} => {
2117 let registered = self
2118 .registry
2119 .get_module_by_connection(connection_id)
2120 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2121 let Some(registration) = registered else {
2122 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2123 };
2124 let response = self
2125 .forwarding
2126 .live_roots(®istration.manifest.module_id)
2127 .map_err(RouterError::Forwarding)?;
2128 Ok(vec![control_response_body_frame(
2129 &frame,
2130 &response,
2131 "ModuleControlResponseToModule::LiveRoots",
2132 )?])
2133 }
2134 }
2135 }
2136
2137 fn handle_catalog_update(
2138 &self,
2139 connection_id: ConnectionId,
2140 frame: Frame,
2141 provides: Vec<ProviderRole>,
2142 capabilities: Option<CapabilityDeclarations>,
2143 ready: Option<bool>,
2144 ) -> Result<Vec<Frame>, RouterError> {
2145 self.refresh_capability_requirements();
2146 let Some(registration) = self
2147 .registry
2148 .get_module_by_connection(connection_id)
2149 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2150 else {
2151 return Ok(vec![control_error_frame(
2152 &frame,
2153 "not_registered",
2154 "catalog.update requires an active module registration owned by this connection",
2155 )?]);
2156 };
2157
2158 if let Some(message) =
2159 catalog_update_frozen_field_message(®istration.manifest, &provides)
2160 {
2161 return Ok(vec![control_error_frame(
2162 &frame,
2163 "catalog_update_frozen_field",
2164 message,
2165 )?]);
2166 }
2167
2168 let mut candidate = registration.manifest.clone();
2169 candidate.provides = provides.clone();
2170 candidate.capabilities = capabilities
2171 .clone()
2172 .or_else(|| registration.manifest.capabilities.clone());
2173 if let Err(err) = candidate.validate_capability_grammar() {
2174 return Ok(vec![control_error_frame(
2175 &frame,
2176 "invalid_capability_grammar",
2177 err.to_string(),
2178 )?]);
2179 }
2180
2181 let updated = self
2182 .registry
2183 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
2184 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2185 if updated.is_none() {
2186 return Ok(vec![control_error_frame(
2187 &frame,
2188 "not_registered",
2189 "catalog.update requires an active module registration owned by this connection",
2190 )?]);
2191 }
2192 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
2193 log_duplicate_claim_events(
2194 self.capability_evaluator
2195 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
2196 );
2197 }
2198 if capability_census_trigger(
2199 registration.manifest.capabilities.as_ref(),
2200 updated
2201 .as_ref()
2202 .and_then(|entry| entry.manifest.capabilities.as_ref()),
2203 ) {
2204 self.enforce_capability_denies();
2205 }
2206 self.refresh_capability_requirements();
2207
2208 let response = ModuleControlResponseToModule::CatalogUpdate {};
2209 control_response_body_frame(
2210 &frame,
2211 &response,
2212 "ModuleControlResponseToModule::CatalogUpdate",
2213 )
2214 .map(|frame| vec![frame])
2215 }
2216
2217 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
2218 self.refresh_capability_requirements();
2219 let mut counters = self.counters.snapshot();
2225 if let (Ok((connections_with_routes, max)), Some(obj)) = (
2226 self.forwarding.client_route_concentration(),
2227 counters.as_object_mut(),
2228 ) {
2229 obj.insert(
2230 "client_connections_with_routes".into(),
2231 connections_with_routes.into(),
2232 );
2233 obj.insert("max_routes_on_one_connection".into(), max.into());
2234 }
2235 if let (Some(open_breakers), Some(obj)) = (
2241 self.route_bind_breakers.open_snapshot(),
2242 counters.as_object_mut(),
2243 ) {
2244 obj.insert("route_bind_breakers_open".into(), open_breakers);
2245 }
2246 let response = ClientControlResponse::ServerDescribe {
2247 protocol_ver: PROTOCOL_VERSION,
2248 subc_ops: subc_ops(),
2249 capabilities: self.subc_capabilities.as_ref().to_vec(),
2250 connected_clients: self.connected_clients.count(),
2251 counters: Some(counters),
2252 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
2253 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
2254 capability_requirements: self.capability_requirement_statuses(),
2255 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2256 };
2257 Ok(vec![control_response_body_frame(
2258 &frame,
2259 &response,
2260 "ClientControlResponse::ServerDescribe",
2261 )?])
2262 }
2263
2264 fn handle_catalog_list(
2265 &self,
2266 frame: Frame,
2267 module_id: Option<String>,
2268 ) -> Result<Vec<Frame>, RouterError> {
2269 let (generation, modules) = self.registry.list_modules().map_err(|err| {
2270 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
2271 })?;
2272 let entries = modules
2273 .into_iter()
2274 .filter(|registration| {
2275 module_id
2276 .as_deref()
2277 .map(|wanted| registration.manifest.module_id == wanted)
2278 .unwrap_or(true)
2279 })
2280 .map(|registration| {
2281 let not_ready = self.not_ready_reason(®istration);
2282 let roles = registration.manifest.provides;
2283 CatalogEntry {
2284 module_id: registration.manifest.module_id,
2285 ready: not_ready.is_none(),
2286 not_ready,
2287 module_version: Some(registration.manifest.module_version),
2288 roles,
2289 control_ops: registration.control_ops,
2290 capabilities: registration.manifest.capabilities,
2291 self_signals: registration.manifest.self_signals,
2292 }
2293 })
2294 .collect();
2295 let response = ClientControlResponse::CatalogList {
2296 generation,
2297 modules: entries,
2298 subc_ops: subc_ops(),
2299 };
2300 Ok(vec![control_response_body_frame(
2301 &frame,
2302 &response,
2303 "ClientControlResponse::CatalogList",
2304 )?])
2305 }
2306
2307 fn route_open_principal(
2308 &self,
2309 frame: &Frame,
2310 consumer_identity: Option<ConsumerIdentity>,
2311 ) -> Result<Result<Principal, Frame>, RouterError> {
2312 let Some(consumer_identity) = consumer_identity else {
2313 return Ok(Ok(Principal::Direct));
2314 };
2315
2316 if self.supervisor.spawned_consumer_authorized(
2317 &consumer_identity.module_id,
2318 &consumer_identity.launch_nonce,
2319 ) {
2320 return Ok(Ok(Principal::Reserved {
2321 module_id: consumer_identity.module_id,
2322 }));
2323 }
2324
2325 Ok(Err(control_error_frame(
2326 frame,
2327 "bad_consumer_identity",
2328 format!(
2329 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
2330 consumer_identity.module_id
2331 ),
2332 )?))
2333 }
2334
2335 fn route_open_refusal_frame(
2341 &self,
2342 ctx: &RouteCtx,
2343 frame: &Frame,
2344 module_id: &str,
2345 reason: &'static str,
2346 code: &'static str,
2347 message: impl Into<String>,
2348 ) -> Result<Frame, RouterError> {
2349 self.observe_route_open_refusal(ctx, module_id, reason, code);
2350 control_error_frame(frame, code, message.into())
2351 }
2352
2353 fn route_open_breaker_refusal_frame(
2371 &self,
2372 ctx: &RouteCtx,
2373 frame: &Frame,
2374 module_id: &str,
2375 consecutive_timeouts: u32,
2376 retry_in: Duration,
2377 probe_in_flight: bool,
2378 ) -> Result<Frame, RouterError> {
2379 self.counters
2380 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
2381 debug!(
2382 target: "control",
2383 code = "module_timeout",
2384 module_id = ?module_id,
2385 connection_id = ctx.connection_id.get(),
2386 consecutive_timeouts,
2387 retry_in_ms = retry_in.as_millis() as u64,
2388 probe_in_flight,
2389 "route.open refused by open bind-relay breaker"
2390 );
2391 let detail = if probe_in_flight {
2392 "one probe bind is already in flight; retry once it settles".to_string()
2393 } else {
2394 format!("not relaying for another {retry_in:?}")
2395 };
2396 control_error_frame(
2397 frame,
2398 "module_timeout",
2399 format!(
2400 "module_id '{module_id}' failed {consecutive_timeouts} consecutive route.bind \
2401 relays; {detail}"
2402 ),
2403 )
2404 }
2405
2406 fn observe_route_open_refusal(
2419 &self,
2420 ctx: &RouteCtx,
2421 module_id: &str,
2422 reason: &'static str,
2423 code: &'static str,
2424 ) {
2425 self.counters.increment_route_open_refused(code);
2426 info!(
2427 target: "control",
2428 code,
2429 reason,
2430 module_id = ?module_id,
2431 connection_id = ctx.connection_id.get(),
2432 "route.open refused"
2433 );
2434 }
2435
2436 fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
2481 self.counters.increment_route_open_accepted(principal);
2482 info!(
2483 target: "control",
2484 principal,
2485 module_id,
2486 connection_id = ctx.connection_id.get(),
2487 "route.open accepted"
2488 );
2489 }
2490
2491 fn supervised_absent_route_open_refusal_frame(
2492 &self,
2493 ctx: &RouteCtx,
2494 frame: &Frame,
2495 module_id: &str,
2496 code: &'static str,
2497 status: &crate::supervise::ModuleStatus,
2498 ) -> Result<Frame, RouterError> {
2499 self.counters.increment_route_open_refused(code);
2500 info!(
2501 target: "control",
2502 code,
2503 reason = "supervised_not_registered",
2504 module_id = ?module_id,
2505 connection_id = ctx.connection_id.get(),
2506 state = %status.state,
2507 enabled = status.enabled,
2508 live = status.live,
2509 "route.open refused"
2510 );
2511 control_error_frame(
2512 frame,
2513 code,
2514 format!(
2515 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
2516 status.state, status.enabled, status.live
2517 ),
2518 )
2519 }
2520
2521 async fn handle_route_open(
2522 &self,
2523 ctx: &RouteCtx,
2524 frame: Frame,
2525 request: RouteOpenRequest,
2526 ) -> Result<Vec<Frame>, RouterError> {
2527 let RouteOpenRequest {
2528 target,
2529 mut identity,
2530 consumer_identity,
2531 consumer_capabilities,
2532 admission_facts,
2533 } = request;
2534 let target_module_id = target_module_id(&target).to_string();
2535 debug!(
2536 connection_id = ctx.connection_id.get(),
2537 corr = frame.header.corr,
2538 module_id = %target_module_id,
2539 "handling route.open"
2540 );
2541
2542 let Some(registration) = self
2561 .registry
2562 .get_module(&target_module_id)
2563 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2564 else {
2565 if let Some((status, warming)) =
2566 self.supervisor_status(&target_module_id, frame.header.corr)?
2567 {
2568 if status.protocol == ModuleProtocol::None {
2575 return Ok(vec![self.route_open_refusal_frame(
2576 ctx,
2577 &frame,
2578 &target_module_id,
2579 "protocol_none",
2580 error_codes::MODULE_NO_PROTOCOL,
2581 format!(
2582 "module_id '{target_module_id}' is declared protocol: none; \
2583 it speaks no subc wire and serves no routes"
2584 ),
2585 )?]);
2586 }
2587 let code = if warming {
2588 "module_warming"
2589 } else {
2590 "target_unavailable"
2591 };
2592 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
2593 ctx,
2594 &frame,
2595 &target_module_id,
2596 code,
2597 &status,
2598 )?]);
2599 }
2600 if let Some(removed_ago_ms) =
2601 self.supervisor.removal_tombstone_age_ms(&target_module_id)
2602 {
2603 return Ok(vec![self.route_open_refusal_frame(
2604 ctx,
2605 &frame,
2606 &target_module_id,
2607 "removed",
2608 error_codes::MODULE_REMOVED,
2609 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
2610 )?]);
2611 }
2612 return Ok(vec![self.route_open_refusal_frame(
2613 ctx,
2614 &frame,
2615 &target_module_id,
2616 "not_registered",
2617 error_codes::UNKNOWN_MODULE,
2618 format!("module_id '{target_module_id}' is not registered"),
2619 )?]);
2620 };
2621
2622 if !registration.ready {
2626 self.counters
2627 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
2628 info!(
2629 target: "control",
2630 code = error_codes::MODULE_WARMING,
2631 module_id = ?target_module_id,
2632 connection_id = ctx.connection_id.get(),
2633 reason = "declared_not_ready",
2634 "route.open refused"
2635 );
2636 return Ok(vec![control_error_body_frame(
2637 &frame,
2638 ErrorBody {
2639 code: error_codes::MODULE_WARMING.to_string(),
2640 message: format!(
2641 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
2642 ),
2643 detail: Some(serde_json::json!({
2644 "reason": "declared_not_ready"
2645 })),
2646 },
2647 )?]);
2648 }
2649
2650 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
2669 self.counters
2670 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
2671 info!(
2672 target: "control",
2673 code = error_codes::MODULE_WARMING,
2674 module_id = ?target_module_id,
2675 connection_id = ctx.connection_id.get(),
2676 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
2677 capability = %capability,
2678 "route.open refused"
2679 );
2680 return Ok(vec![control_error_body_frame(
2681 &frame,
2682 ErrorBody {
2683 code: error_codes::MODULE_WARMING.to_string(),
2684 message: format!(
2685 "module_id '{target_module_id}' requires capability '{capability}', \
2686 which no registered module provides; retry"
2687 ),
2688 detail: Some(serde_json::json!({
2689 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
2690 "capability": capability,
2691 })),
2692 },
2693 )?]);
2694 }
2695
2696 if !target_has_required_role(&target, ®istration.manifest.provides) {
2697 return Ok(vec![self.route_open_refusal_frame(
2698 ctx,
2699 &frame,
2700 &target_module_id,
2701 "role_not_provided",
2702 "target_unavailable",
2703 format!("module_id '{target_module_id}' does not provide the requested target"),
2704 )?]);
2705 }
2706
2707 if registration.state != ChannelState::Active {
2708 return Ok(vec![self.route_open_refusal_frame(
2709 ctx,
2710 &frame,
2711 &target_module_id,
2712 "registration_not_active",
2713 "target_unavailable",
2714 format!("module_id '{target_module_id}' is not active"),
2715 )?]);
2716 }
2717
2718 if self
2719 .forwarding
2720 .module_is_draining(&target_module_id)
2721 .map_err(RouterError::Forwarding)?
2722 {
2723 return Ok(vec![self.route_open_refusal_frame(
2724 ctx,
2725 &frame,
2726 &target_module_id,
2727 "reloading",
2728 "module_reloading",
2729 format!("module_id '{target_module_id}' is reloading"),
2730 )?]);
2731 }
2732
2733 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
2734 process_liveness.process_live(&target_module_id) == Some(false)
2735 }) {
2736 if process_liveness.process_replacing(&target_module_id) {
2743 return Ok(vec![self.route_open_refusal_frame(
2744 ctx,
2745 &frame,
2746 &target_module_id,
2747 "reloading",
2748 "module_reloading",
2749 format!("module_id '{target_module_id}' is reloading"),
2750 )?]);
2751 }
2752 return Ok(vec![self.route_open_refusal_frame(
2753 ctx,
2754 &frame,
2755 &target_module_id,
2756 "supervisor_not_live",
2757 "target_unavailable",
2758 format!("module_id '{target_module_id}' is not live"),
2759 )?]);
2760 }
2761
2762 if !self
2763 .forwarding
2764 .has_live_module_connection(&target_module_id)
2765 .map_err(RouterError::Forwarding)?
2766 {
2767 return Ok(vec![self.route_open_refusal_frame(
2768 ctx,
2769 &frame,
2770 &target_module_id,
2771 "no_forwarding_connection",
2772 "target_unavailable",
2773 format!("module_id '{target_module_id}' has no live forwarding connection"),
2774 )?]);
2775 }
2776
2777 if let Some(error) =
2778 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
2779 {
2780 self.observe_route_open_refusal(
2781 ctx,
2782 &target_module_id,
2783 "op_not_allowed",
2784 "op_not_allowed",
2785 );
2786 return Ok(vec![error]);
2787 }
2788
2789 let principal = match self.route_open_principal(&frame, consumer_identity)? {
2790 Ok(principal) => principal,
2791 Err(error) => {
2792 self.observe_route_open_refusal(
2793 ctx,
2794 &target_module_id,
2795 "bad_consumer_identity",
2796 "bad_consumer_identity",
2797 );
2798 return Ok(vec![error]);
2799 }
2800 };
2801
2802 if let Principal::Reserved {
2806 module_id: opening_module_id,
2807 } = &principal
2808 {
2809 if let Some(opening_registration) = self
2810 .registry
2811 .get_module(opening_module_id)
2812 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2813 {
2814 if let Some(capability) =
2815 denied_capability(&opening_registration.manifest, ®istration.manifest)
2816 {
2817 warn!(
2818 opening_module_id,
2819 target_module_id,
2820 capability,
2821 "refusing route.open because an attested capability deny edge matches"
2822 );
2823 return Ok(vec![self.route_open_refusal_frame(
2824 ctx,
2825 &frame,
2826 &target_module_id,
2827 "capability_deny_edge",
2828 "capability_forbidden",
2829 format!(
2830 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
2831 ),
2832 )?]);
2833 }
2834 }
2835 }
2836
2837 if admission_facts.is_some() {
2838 let carrier_matches = matches!(
2839 &principal,
2840 Principal::Reserved { module_id }
2841 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
2842 );
2843 if !carrier_matches {
2844 return Ok(vec![self.route_open_refusal_frame(
2845 ctx,
2846 &frame,
2847 &target_module_id,
2848 "admission_facts_carrier_not_permitted",
2849 "admission_facts_not_permitted",
2850 "admission facts may only be carried by the configured reserved module",
2851 )?]);
2852 }
2853
2854 let target_allowed = self
2855 .admission_facts_targets
2856 .as_ref()
2857 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
2858 if !target_allowed {
2859 return Ok(vec![self.route_open_refusal_frame(
2860 ctx,
2861 &frame,
2862 &target_module_id,
2863 "admission_facts_target_not_listed",
2864 "admission_facts_target_not_allowed",
2865 format!(
2866 "admission facts are not permitted for target module_id '{target_module_id}'"
2867 ),
2868 )?]);
2869 }
2870
2871 }
2875
2876 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
2898 Ok(project_root) => project_root,
2899 Err(err) => {
2900 return Ok(vec![control_error_frame(
2901 &frame,
2902 "invalid_project_root",
2903 err.to_string(),
2904 )?])
2905 }
2906 };
2907 identity.project_root = project_root.as_path().to_path_buf();
2908
2909 let _concurrency_guard = match self
2920 .route_bind_concurrency
2921 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
2922 {
2923 Ok(guard) => guard,
2924 Err(in_flight) => {
2925 return Ok(vec![self.route_open_admission_refusal_frame(
2926 ctx,
2927 &frame,
2928 &target_module_id,
2929 format!(
2930 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
2931 ),
2932 )?]);
2933 }
2934 };
2935
2936 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
2939 RouteBindAdmission::Admitted { guard, probe } => {
2940 if probe {
2941 info!(
2942 module_id = %target_module_id,
2943 connection_id = ctx.connection_id.get(),
2944 "route.bind breaker half-open: admitting one probe"
2945 );
2946 }
2947 guard
2948 }
2949 RouteBindAdmission::Refused {
2950 consecutive_timeouts,
2951 retry_in,
2952 probe_in_flight,
2953 } => {
2954 return Ok(vec![self.route_open_breaker_refusal_frame(
2955 ctx,
2956 &frame,
2957 &target_module_id,
2958 consecutive_timeouts,
2959 retry_in,
2960 probe_in_flight,
2961 )?]);
2962 }
2963 };
2964
2965 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
2970 let relay_deadline = Instant::now() + route_bind_relay_timeout;
2971 let pending = match self
2972 .forwarding
2973 .begin_route_bind_relay_for(
2974 ctx.connection_id,
2975 ctx.egress.clone(),
2976 response_version(&frame),
2977 frame.header.corr,
2978 &target_module_id,
2979 principal.clone(),
2980 Some(project_root),
2981 relay_deadline,
2982 )
2983 .await
2984 {
2985 Ok(pending) => pending,
2986 Err(err) => {
2987 return Ok(vec![self.route_open_refusal_frame(
2988 ctx,
2989 &frame,
2990 &target_module_id,
2991 "relay_reservation_failed",
2992 forwarding_error_code(&err),
2993 err.to_string(),
2994 )?])
2995 }
2996 };
2997 let crate::forwarding::PendingRouteBindRelay {
2998 endpoint,
2999 module_sink,
3000 negotiated_ver,
3001 client_channel,
3002 client_epoch,
3003 module_channel,
3004 module_epoch,
3005 corr: relay_corr,
3006 receiver,
3007 } = pending;
3008 let mut reservation =
3009 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
3010
3011 debug!(
3012 connection_id = ctx.connection_id.get(),
3013 client_channel,
3014 client_epoch,
3015 module_channel,
3016 module_epoch,
3017 "reserved route handle pair"
3018 );
3019 let principal_label = match &principal {
3022 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
3023 Principal::Direct => "direct".to_string(),
3024 other => format!("{other:?}"),
3025 };
3026 let relay = ModuleControlRequest::RouteBind {
3027 route_channel: module_channel,
3028 epoch: module_epoch,
3029 target,
3030 identity,
3031 principal: Some(principal),
3032 consumer_capabilities,
3033 admission_facts,
3034 };
3035 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
3036 RouterError::backend(
3037 0,
3038 frame.header.corr,
3039 format!("failed to encode route.bind request: {err}"),
3040 )
3041 })?;
3042 let relay_frame = Frame::build_with_version(
3043 negotiated_ver,
3044 FrameType::Request,
3045 control_flags(),
3046 0,
3047 0,
3048 relay_corr,
3049 relay_body,
3050 )
3051 .map_err(RouterError::FrameBuild)?;
3052
3053 if let Err(err) = module_sink.send(relay_frame).await {
3054 reservation.release_and_disarm();
3055 return Ok(vec![self.route_open_refusal_frame(
3056 ctx,
3057 &frame,
3058 &target_module_id,
3059 "relay_send_failed",
3060 "target_unavailable",
3061 err.to_string(),
3062 )?]);
3063 }
3064
3065 if !self
3066 .forwarding
3067 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
3068 .map_err(RouterError::Forwarding)?
3069 {
3070 self.send_abandoned_route_bind_goodbye(
3071 &module_sink,
3072 negotiated_ver,
3073 module_channel,
3074 module_epoch,
3075 );
3076 }
3077
3078 match timeout_at(relay_deadline, receiver).await {
3079 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
3080 reservation.disarm();
3081 if breaker.record_accepted() {
3082 info!(
3083 module_id = %target_module_id,
3084 "route.bind breaker closed: the probe was accepted"
3085 );
3086 }
3087 self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
3088 Ok(Vec::new())
3089 }
3090 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
3091 reservation.release_and_disarm();
3092 breaker.record_inconclusive();
3096 self.counters
3097 .increment_route_open_refused("module_rejected");
3098 info!(
3099 target: "control",
3100 code = "module_rejected",
3101 module_code = ?body.code,
3102 module_id = ?target_module_id,
3103 connection_id = ctx.connection_id.get(),
3104 "route.open refused"
3105 );
3106 Ok(vec![control_error_body_frame(&frame, body)?])
3107 }
3108 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
3109 reservation.release_and_disarm();
3110 breaker.record_inconclusive();
3111 tracing::warn!(
3118 module_id = %target_module_id,
3119 "route.bind relay abandoned: {message}"
3120 );
3121 Ok(vec![self.route_open_refusal_frame(
3122 ctx,
3123 &frame,
3124 &target_module_id,
3125 "relay_abandoned",
3126 "target_unavailable",
3127 message,
3128 )?])
3129 }
3130 Ok(Err(_)) => {
3131 reservation.release_and_disarm();
3132 breaker.record_inconclusive();
3133 Ok(vec![self.route_open_refusal_frame(
3134 ctx,
3135 &frame,
3136 &target_module_id,
3137 "relay_waiter_canceled",
3138 "target_unavailable",
3139 "route.bind relay waiter was canceled before the module responded",
3140 )?])
3141 }
3142 Err(_) => {
3143 reservation.release_and_disarm();
3144 if let Some(opened) = breaker.record_timeout(
3148 self.route_bind_breaker_threshold,
3149 self.route_bind_breaker_cooldown,
3150 ) {
3151 warn!(
3152 module_id = %target_module_id,
3153 consecutive_timeouts = opened.consecutive_timeouts,
3154 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
3155 reopened_after_probe = opened.reopened_after_probe,
3156 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
3157 );
3158 }
3159 tracing::warn!(
3167 module_id = %target_module_id,
3168 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
3169 "route.bind relay timed out: module did not ack within budget"
3170 );
3171 Ok(vec![self.route_open_refusal_frame(
3172 ctx,
3173 &frame,
3174 &target_module_id,
3175 "relay_timed_out",
3176 "module_timeout",
3177 format!(
3178 "module_id '{target_module_id}' did not answer route.bind within {:?}",
3179 route_bind_relay_timeout
3180 ),
3181 )?])
3182 }
3183 }
3184 }
3185
3186 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3187 let response = ClientControlResponse::SupervisorSpawnSnapshot {
3188 snapshot: self.supervisor.spawn_snapshot(),
3189 };
3190 Ok(vec![control_response_body_frame(
3191 &frame,
3192 &response,
3193 "ClientControlResponse::SupervisorSpawnSnapshot",
3194 )?])
3195 }
3196
3197 fn handle_supervisor_spawn_subscribe(
3198 &self,
3199 ctx: &RouteCtx,
3200 frame: Frame,
3201 since: Option<SpawnCursor>,
3202 ) -> Result<Vec<Frame>, RouterError> {
3203 match self.supervisor.subscribe_spawns(
3204 ctx.connection_id,
3205 frame.header.corr,
3206 response_version(&frame),
3207 since,
3208 ctx.egress.clone(),
3209 ) {
3210 Ok(()) => Ok(Vec::new()),
3211 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
3212 Ok(vec![control_error_body_frame(
3213 &frame,
3214 ErrorBody {
3215 code: "spawn_cursor_incarnation_mismatch".to_string(),
3216 message: "spawn cursor belongs to a different daemon incarnation"
3217 .to_string(),
3218 detail: Some(serde_json::json!({
3219 "current_daemon_incarnation": current
3220 })),
3221 },
3222 )?])
3223 }
3224 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
3225 &frame,
3226 ErrorBody {
3227 code: "spawn_cursor_too_old".to_string(),
3228 message: "spawn cursor predates the retained event ring".to_string(),
3229 detail: Some(serde_json::json!({
3230 "oldest_retained_cursor": oldest
3231 })),
3232 },
3233 )?]),
3234 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
3235 0,
3236 frame.header.corr,
3237 format!("failed to open supervisor spawn subscription: {error}"),
3238 )),
3239 }
3240 }
3241
3242 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3243 let generation = self
3244 .registry
3245 .generation()
3246 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3247 let mut modules = Vec::new();
3248 for module in self.supervisor.list() {
3249 let status = module.status_for_control("list").map_err(|err| {
3250 RouterError::backend(
3251 0,
3252 frame.header.corr,
3253 format!("failed to read supervisor status: {err}"),
3254 )
3255 })?;
3256 let (configured, _) = module.configuration().map_err(|err| {
3257 RouterError::backend(
3258 0,
3259 frame.header.corr,
3260 format!("failed to read module configuration: {err}"),
3261 )
3262 })?;
3263 let image = module.running_image_agreement().await;
3265 let pending_reload = Some(reload_verdict(
3266 &configured.program,
3267 status.spawned_from.as_deref(),
3268 image,
3269 ));
3270 modules.push(SupervisorEntry {
3271 module_id: status.module_id,
3272 state: status.state.to_string(),
3273 enabled: status.enabled,
3274 live: status.live,
3275 protocol: status.protocol,
3276 health: status.health.status,
3277 pending_reload,
3278 last_probe_ms: status.health.last_probe_ms,
3279 last_exit_code: status.last_exit.as_ref().and_then(|e| e.code),
3280 last_exit_signal: status.last_exit.as_ref().and_then(|e| e.signal),
3281 last_exit_ms: status.last_exit.as_ref().map(|e| e.at_ms),
3282 last_exit_kind: status.last_exit.as_ref().map(|e| e.kind.into()),
3283 restart_count: Some(status.restart_count),
3284 max_restarts: Some(status.max_restarts),
3285 lifetime_restarts: Some(status.lifetime_restarts),
3286 spawn_generation: Some(status.spawn_generation),
3287 restart_window_secs: Some(status.restart_window.as_secs()),
3288 drain_timeout_ms: Some(status.drain_timeout.as_millis() as u64),
3289 restart_backoff_ms: Some(status.restart_backoff.as_millis() as u64),
3290 restart_max_backoff_ms: Some(status.restart_max_backoff.as_millis() as u64),
3291 });
3292 }
3293 let response = ClientControlResponse::SupervisorList {
3294 generation,
3295 modules,
3296 };
3297 Ok(vec![control_response_body_frame(
3298 &frame,
3299 &response,
3300 "ClientControlResponse::SupervisorList",
3301 )?])
3302 }
3303
3304 fn handle_supervisor_stderr_tail(
3305 &self,
3306 frame: Frame,
3307 module_id: String,
3308 max_lines: Option<u32>,
3309 max_bytes: Option<u32>,
3310 ) -> Result<Vec<Frame>, RouterError> {
3311 let Some(module) = self.supervisor.get(&module_id) else {
3312 return Ok(vec![control_error_frame(
3313 &frame,
3314 "unknown_module",
3315 format!("module_id '{module_id}' is not supervised"),
3316 )?]);
3317 };
3318
3319 let snapshot = module.stderr_tail(
3320 max_lines.map(|value| value as usize),
3321 max_bytes.map(|value| value as usize),
3322 );
3323
3324 let response = ClientControlResponse::SupervisorStderrTail {
3325 module_id,
3326 tail: StderrTail {
3327 capture: match snapshot.capture {
3328 CaptureState::Captured => StderrCaptureState::Captured,
3329 CaptureState::Incomplete { reason } => {
3330 StderrCaptureState::Incomplete { reason }
3331 }
3332 CaptureState::NotCaptured { reason } => {
3333 StderrCaptureState::NotCaptured { reason }
3334 }
3335 },
3336 entries: snapshot
3337 .entries
3338 .into_iter()
3339 .map(|entry| match entry {
3340 TailEntry::Line { text, truncated } => {
3341 StderrTailEntry::Line { text, truncated }
3342 }
3343 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
3344 })
3345 .collect(),
3346 dropped_lines: snapshot.dropped_lines,
3347 },
3348 };
3349 Ok(vec![control_response_body_frame(
3350 &frame,
3351 &response,
3352 "ClientControlResponse::SupervisorStderrTail",
3353 )?])
3354 }
3355
3356 async fn handle_supervisor_terminals(
3357 &self,
3358 frame: Frame,
3359 module_id: String,
3360 ) -> Result<Vec<Frame>, RouterError> {
3361 let Some(module) = self.supervisor.get(&module_id) else {
3362 return Ok(vec![control_error_frame(
3363 &frame,
3364 "unknown_module",
3365 format!("module_id '{module_id}' is not supervised"),
3366 )?]);
3367 };
3368
3369 let terminals = module
3372 .read_durable_terminal_history()
3373 .await
3374 .map_err(|error| {
3375 RouterError::backend(
3376 0,
3377 frame.header.corr,
3378 format!("failed to read terminal history: {error}"),
3379 )
3380 })?;
3381 let response = ClientControlResponse::SupervisorTerminals {
3382 module_id,
3383 terminals,
3384 };
3385 Ok(vec![control_response_body_frame(
3386 &frame,
3387 &response,
3388 "ClientControlResponse::SupervisorTerminals",
3389 )?])
3390 }
3391
3392 fn handle_supervisor_routes(
3393 &self,
3394 frame: Frame,
3395 module_id: Option<String>,
3396 ) -> Result<Vec<Frame>, RouterError> {
3397 let modules = self
3398 .forwarding
3399 .route_census(module_id.as_deref())
3400 .map_err(RouterError::Forwarding)?
3401 .into_iter()
3402 .map(|(module_id, routes)| SupervisorRouteModule {
3403 module_id,
3404 routes: routes
3405 .into_iter()
3406 .map(|route| SupervisorRoute {
3407 consumer: match route.principal {
3408 Principal::Reserved { module_id } => {
3409 SupervisorRouteConsumer::Reserved { module_id }
3410 }
3411 Principal::Direct | Principal::Unverified => {
3412 SupervisorRouteConsumer::Direct {
3413 connection_id: route.goodbye_target.connection_id.get(),
3414 }
3415 }
3416 },
3417 age_ms: Instant::now()
3418 .saturating_duration_since(route.bound_at)
3419 .as_millis()
3420 .try_into()
3421 .unwrap_or(u64::MAX),
3422 draining: route.draining,
3423 drain_reason: route.drain_reason,
3424 })
3425 .collect(),
3426 })
3427 .collect();
3428 let response = ClientControlResponse::SupervisorRoutes { modules };
3429 Ok(vec![control_response_body_frame(
3430 &frame,
3431 &response,
3432 "ClientControlResponse::SupervisorRoutes",
3433 )?])
3434 }
3435
3436 async fn handle_supervisor_provenance(
3437 &self,
3438 frame: Frame,
3439 module_id: Option<String>,
3440 ) -> Result<Vec<Frame>, RouterError> {
3441 let mut selected = if let Some(module_id) = module_id {
3442 let Some(module) = self.supervisor.get(&module_id) else {
3443 return Ok(vec![control_error_frame(
3444 &frame,
3445 "unknown_module",
3446 format!("module_id '{module_id}' is not supervised"),
3447 )?]);
3448 };
3449 vec![module]
3450 } else {
3451 self.supervisor.list()
3452 };
3453
3454 let mut modules = Vec::with_capacity(selected.len());
3455 for module in selected.drain(..) {
3456 let status = module.status().map_err(|err| {
3457 RouterError::backend(
3458 0,
3459 frame.header.corr,
3460 format!("failed to read supervisor status: {err}"),
3461 )
3462 })?;
3463 let module_declared = self
3464 .registry
3465 .get_module(&status.module_id)
3466 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3467 .and_then(|registration| registration.manifest.provenance)
3468 .map(|build| ModuleDeclaredProvenance::Reported { build })
3469 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
3470 #[cfg(test)]
3471 let running_image = match &self.provenance_probe_override {
3472 Some(result) => result.clone(),
3473 None => module.running_image_agreement().await,
3474 };
3475 #[cfg(not(test))]
3476 let running_image = module.running_image_agreement().await;
3477 modules.push(SupervisorModuleProvenance {
3478 module_id: status.module_id,
3479 module_declared,
3480 daemon_observed: SupervisorObservedProcess {
3481 pid: status.pid,
3482 spawned_at_ms: status.spawned_at_ms,
3483 spawned_from: status.spawned_from,
3484 running_image,
3485 },
3486 });
3487 }
3488 let daemon = SupervisorDaemonProvenance {
3489 daemon_build: self.daemon_provenance.build.clone(),
3490 daemon_observed: DaemonObservedProcess {
3491 pid: self.daemon_provenance.pid,
3492 started_at_ms: self
3493 .daemon_provenance
3494 .start_clock
3495 .map(|clock| clock.started_at_ms())
3496 .or(self.daemon_provenance.started_at_ms),
3497 running_image: self
3498 .daemon_provenance
3499 .probe
3500 .observe(
3501 self.daemon_provenance.pid,
3502 self.daemon_provenance.executable_path.as_deref(),
3503 self.daemon_provenance.executable_identity,
3504 self.daemon_provenance.process_start_time,
3505 )
3506 .await,
3507 },
3508 };
3509 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
3510 Ok(vec![control_response_body_frame(
3511 &frame,
3512 &response,
3513 "ClientControlResponse::SupervisorProvenance",
3514 )?])
3515 }
3516
3517 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3518 self.refresh_capability_requirements();
3519 let generation = self
3520 .registry
3521 .generation()
3522 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3523 let modules = self
3524 .supervisor
3525 .list()
3526 .into_iter()
3527 .map(|module| {
3528 let status = module.status_for_control("health").map_err(|err| {
3529 RouterError::backend(
3530 0,
3531 frame.header.corr,
3532 format!("failed to read supervisor health: {err}"),
3533 )
3534 })?;
3535 let module_id = status.module_id;
3536 let capability_detail = self
3537 .capability_evaluator
3538 .required_problem_detail(&module_id);
3539 Ok(SupervisorHealthEntry {
3540 module_id,
3541 status: status.health.status,
3542 detail: append_capability_problem_detail(
3543 status.health.detail,
3544 capability_detail,
3545 ),
3546 metrics: status.health.metrics,
3547 consecutive_failures: status.health.consecutive_failures,
3548 late_answer_count: status.health.late_answer_count,
3549 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
3550 last_action: status.health.last_action,
3551 last_action_ms: status.health.last_action_ms,
3552 last_probe_ms: status.health.last_probe_ms,
3553 })
3554 })
3555 .collect::<Result<Vec<_>, RouterError>>()?;
3556 let response = ClientControlResponse::SupervisorHealth {
3557 generation,
3558 modules,
3559 };
3560 Ok(vec![control_response_body_frame(
3561 &frame,
3562 &response,
3563 "ClientControlResponse::SupervisorHealth",
3564 )?])
3565 }
3566
3567 async fn handle_supervisor_restart(
3568 &self,
3569 frame: Frame,
3570 module_id: String,
3571 drain_timeout_ms: Option<u64>,
3572 ) -> Result<Vec<Frame>, RouterError> {
3573 let operation_lock = self.supervisor.operation_lock();
3574 let _operation_guard = operation_lock.lock().await;
3575 let Some(module) = self.supervisor.get(&module_id) else {
3576 return Ok(vec![control_error_frame(
3577 &frame,
3578 "unknown_module",
3579 format!("module_id '{module_id}' is not supervised"),
3580 )?]);
3581 };
3582
3583 if let Err(err) = module.restart(drain_timeout_ms).await {
3584 let (code, message) = match err {
3585 crate::supervise::SuperviseError::Disabled { .. } => {
3586 ("module_disabled", err.to_string())
3587 }
3588 crate::supervise::SuperviseError::SwapInProgress { .. } => {
3589 ("swap_in_progress", err.to_string())
3590 }
3591 _ => (
3592 "target_unavailable",
3593 format!("failed to restart module_id '{module_id}': {err}"),
3594 ),
3595 };
3596 return Ok(vec![control_error_frame(&frame, code, message)?]);
3597 }
3598
3599 let response = ClientControlResponse::SupervisorAck {
3600 module_id,
3601 applied: true,
3602 };
3603 Ok(vec![control_response_body_frame(
3604 &frame,
3605 &response,
3606 "ClientControlResponse::SupervisorAck",
3607 )?])
3608 }
3609
3610 async fn handle_supervisor_swap(
3614 &self,
3615 frame: Frame,
3616 module_id: String,
3617 ready_timeout_ms: Option<u64>,
3618 ) -> Result<Vec<Frame>, RouterError> {
3619 let module = {
3626 let operation_lock = self.supervisor.operation_lock();
3627 let _operation_guard = operation_lock.lock().await;
3628 self.supervisor.get(&module_id)
3629 };
3630 let Some(module) = module else {
3631 return Ok(vec![control_error_frame(
3632 &frame,
3633 "unknown_module",
3634 format!("module_id '{module_id}' is not supervised"),
3635 )?]);
3636 };
3637
3638 if let Err(err) = module
3639 .swap(ready_timeout_ms.map(Duration::from_millis))
3640 .await
3641 {
3642 use crate::supervise::SuperviseError;
3643 let message = err.to_string();
3644 let error = match err {
3645 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
3646 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
3647 code: "swap_refused".to_string(),
3648 message,
3649 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
3650 },
3651 SuperviseError::SwapFailed {
3652 arm,
3653 candidate_exit,
3654 ..
3655 } => ErrorBody {
3656 code: "swap_failed".to_string(),
3657 message,
3658 detail: Some(serde_json::json!({
3659 "arm": arm.as_str(),
3660 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
3661 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
3662 })),
3663 },
3664 _ => ErrorBody::new(
3665 "target_unavailable",
3666 format!("failed to swap module_id '{module_id}': {message}"),
3667 ),
3668 };
3669 return Ok(vec![control_error_body_frame(&frame, error)?]);
3670 }
3671
3672 let response = ClientControlResponse::SupervisorAck {
3673 module_id,
3674 applied: true,
3675 };
3676 Ok(vec![control_response_body_frame(
3677 &frame,
3678 &response,
3679 "ClientControlResponse::SupervisorAck",
3680 )?])
3681 }
3682
3683 async fn handle_supervisor_reload(
3684 &self,
3685 frame: Frame,
3686 module_id: String,
3687 ) -> Result<Vec<Frame>, RouterError> {
3688 let operation_lock = self.supervisor.operation_lock();
3689 let _operation_guard = operation_lock.lock().await;
3690 let Some(module) = self.supervisor.get(&module_id) else {
3691 return Ok(vec![control_error_frame(
3692 &frame,
3693 "unknown_module",
3694 format!("module_id '{module_id}' is not supervised"),
3695 )?]);
3696 };
3697
3698 if let Err(err) = module.reload().await {
3699 let (code, message) = match err {
3700 crate::supervise::SuperviseError::Disabled { .. } => {
3701 ("module_disabled", err.to_string())
3702 }
3703 crate::supervise::SuperviseError::SwapInProgress { .. } => {
3704 ("swap_in_progress", err.to_string())
3705 }
3706 _ => (
3707 "reload_failed",
3708 format!("failed to reload module_id '{module_id}': {err}"),
3709 ),
3710 };
3711 return Ok(vec![control_error_frame(&frame, code, message)?]);
3712 }
3713
3714 let response = ClientControlResponse::SupervisorAck {
3715 module_id,
3716 applied: true,
3717 };
3718 Ok(vec![control_response_body_frame(
3719 &frame,
3720 &response,
3721 "ClientControlResponse::SupervisorAck",
3722 )?])
3723 }
3724
3725 async fn handle_supervisor_rescan(
3726 &self,
3727 frame: Frame,
3728 preview: bool,
3729 ) -> Result<Vec<Frame>, RouterError> {
3730 let Some(context) = self.rescan.clone() else {
3731 return Ok(vec![control_error_frame(
3732 &frame,
3733 "rescan_unavailable",
3734 "the daemon was not started with a reloadable config path".to_string(),
3735 )?]);
3736 };
3737
3738 let operation_lock = self.supervisor.operation_lock();
3739 let _operation_guard = operation_lock.lock().await;
3740 let loaded = match crate::daemon_config::load(&context.config_path) {
3741 Ok(config) => config,
3742 Err(err) => {
3743 return Ok(vec![control_error_frame(
3744 &frame,
3745 "invalid_daemon_config",
3746 format!("supervisor rescan rejected daemon config: {err}"),
3747 )?])
3748 }
3749 };
3750 let Some(config) = loaded else {
3758 return Ok(vec![control_error_frame(
3759 &frame,
3760 "invalid_daemon_config",
3761 format!(
3762 "daemon config not found at {}; refusing to rescan (an absent config would \
3763 retire every supervised module)",
3764 context.config_path.display()
3765 ),
3766 )?]);
3767 };
3768 let (
3769 configured_port,
3770 storage_config,
3771 admission_facts_carrier_module_id,
3772 admission_facts_targets,
3773 modules,
3774 reserved_capabilities,
3775 ) = (
3776 config.port,
3777 config.storage,
3778 config.admission_facts_carrier_module_id,
3779 config.admission_facts_targets,
3780 config.modules,
3781 config.reserved_capabilities,
3782 );
3783
3784 let mut restart_required = Vec::new();
3793 for section in RestartRequiredSection::ALL {
3794 let changed = match section {
3795 RestartRequiredSection::Port => configured_port != context.configured_port,
3796 RestartRequiredSection::Storage => storage_config != context.storage_config,
3797 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
3798 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
3799 }
3800 RestartRequiredSection::AdmissionFactsTargets => {
3801 admission_facts_targets != context.admission_facts_targets
3802 }
3803 };
3804 if changed {
3805 restart_required.push(section.label().to_string());
3806 }
3807 }
3808 if !restart_required.is_empty() {
3809 warn!(
3810 config_path = %context.config_path.display(),
3811 sections = %restart_required.join(", "),
3812 "daemon config changed outside the modules section; restart the daemon to apply those changes"
3813 );
3814 }
3815
3816 for configured in &modules {
3817 if let Err(err) = validate_spec(&configured.module_spec()) {
3818 return Ok(vec![control_error_frame(
3819 &frame,
3820 "invalid_daemon_config",
3821 format!("supervisor rescan rejected daemon config: {err}"),
3822 )?]);
3823 }
3824 }
3825
3826 let configured_capabilities = modules
3827 .iter()
3828 .map(|module| (module.module_id.clone(), module.enabled))
3829 .collect::<Vec<_>>();
3830 let preview_capability_warnings = if preview {
3831 let (_, registrations) = self.runtime_capability_snapshot()?;
3832 let current_modules = self
3833 .supervisor
3834 .list()
3835 .into_iter()
3836 .map(|module| module.module_id().to_string())
3837 .collect::<BTreeSet<_>>();
3838 let resulting_modules = configured_capabilities.clone();
3839 let removed = current_modules
3840 .into_iter()
3841 .filter(|module_id| {
3842 !resulting_modules
3843 .iter()
3844 .any(|(configured_id, _)| configured_id == module_id)
3845 })
3846 .collect::<Vec<_>>();
3847 self.capability_evaluator.preview_removal_warnings(
3848 resulting_modules,
3849 &removed,
3850 ®istrations,
3851 )
3852 } else {
3853 Vec::new()
3854 };
3855 let result = match self
3856 .reconcile_supervised_modules(&context.supervisor, modules, preview)
3857 .await
3858 {
3859 Ok(result) => result,
3860 Err(message) => {
3861 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
3862 }
3863 };
3864 if !preview {
3865 self.capability_evaluator
3866 .configure(configured_capabilities, reserved_capabilities);
3867 self.capability_evaluator.wake_deadline_loop();
3868 self.refresh_capability_requirements();
3869 }
3870 let mut result = result;
3871 result.restart_required = restart_required;
3872 result.capability_warnings = preview_capability_warnings;
3873 let response = ClientControlResponse::SupervisorRescan { result };
3874 Ok(vec![control_response_body_frame(
3875 &frame,
3876 &response,
3877 "ClientControlResponse::SupervisorRescan",
3878 )?])
3879 }
3880
3881 async fn handle_supervisor_release_reserved(
3882 &self,
3883 frame: Frame,
3884 module_id: String,
3885 ) -> Result<Vec<Frame>, RouterError> {
3886 let Some(context) = self.rescan.clone() else {
3887 return Ok(vec![control_error_frame(
3888 &frame,
3889 "release_unavailable",
3890 "reserved-id release requires a daemon started with a reloadable config path",
3891 )?]);
3892 };
3893 let operation_lock = self.supervisor.operation_lock();
3894 let _operation_guard = operation_lock.lock().await;
3895 let loaded = match crate::daemon_config::load(&context.config_path) {
3896 Ok(Some(config)) => config,
3897 Ok(None) => {
3898 return Ok(vec![control_error_frame(
3899 &frame,
3900 "invalid_daemon_config",
3901 format!(
3902 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
3903 context.config_path.display()
3904 ),
3905 )?])
3906 }
3907 Err(err) => {
3908 return Ok(vec![control_error_frame(
3909 &frame,
3910 "invalid_daemon_config",
3911 format!("unable to verify reserved-id release against daemon config: {err}"),
3912 )?])
3913 }
3914 };
3915 if loaded
3916 .modules
3917 .iter()
3918 .any(|configured| configured.module_id == module_id)
3919 {
3920 return Ok(vec![control_error_frame(
3921 &frame,
3922 "reserved_module_configured",
3923 format!(
3924 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
3925 ),
3926 )?]);
3927 }
3928 if !self.supervisor.release_retained_reserved_gate(&module_id) {
3929 return Ok(vec![control_error_frame(
3930 &frame,
3931 "reserved_gate_not_retained",
3932 format!(
3933 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
3934 ),
3935 )?]);
3936 }
3937
3938 let response = ClientControlResponse::SupervisorAck {
3939 module_id,
3940 applied: true,
3941 };
3942 Ok(vec![control_response_body_frame(
3943 &frame,
3944 &response,
3945 "ClientControlResponse::SupervisorAck",
3946 )?])
3947 }
3948
3949 async fn reconcile_supervised_modules(
3958 &self,
3959 supervisor: &Supervisor,
3960 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
3961 preview: bool,
3962 ) -> Result<SupervisorRescanResult, String> {
3963 let mut current = BTreeMap::new();
3964 for module in self.supervisor.list() {
3965 let (spec, health) = module.configuration().map_err(|err| {
3966 format!(
3967 "failed to read configuration for module_id '{}': {err}",
3968 module.module_id()
3969 )
3970 })?;
3971 let enabled = module
3972 .status()
3973 .map_err(|err| {
3974 format!(
3975 "failed to read status for module_id '{}': {err}",
3976 module.module_id()
3977 )
3978 })?
3979 .enabled;
3980 current.insert(
3981 module.module_id().to_string(),
3982 (module, spec, health, enabled),
3983 );
3984 }
3985 let configured = configured_modules
3986 .into_iter()
3987 .map(|module| (module.module_id.clone(), module))
3988 .collect::<BTreeMap<_, _>>();
3989
3990 let added = configured
3991 .keys()
3992 .filter(|module_id| !current.contains_key(*module_id))
3993 .cloned()
3994 .collect::<Vec<_>>();
3995 let removed = current
3996 .keys()
3997 .filter(|module_id| !configured.contains_key(*module_id))
3998 .cloned()
3999 .collect::<Vec<_>>();
4000 let mut changed_pending_reload = Vec::new();
4001 let mut configuration_changes = BTreeSet::new();
4002 let mut enabled_changes = BTreeSet::new();
4003 let mut unchanged = 0_u32;
4004
4005 for (module_id, configured_module) in &configured {
4006 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
4007 else {
4008 continue;
4009 };
4010 let configuration_changed = *current_spec != configured_module.module_spec()
4011 || *current_health != configured_module.health;
4012 let enabled_changed = *current_enabled != configured_module.enabled;
4013 if configuration_changed {
4014 configuration_changes.insert(module_id.clone());
4015 changed_pending_reload.push(module_id.clone());
4016 }
4017 if enabled_changed {
4018 enabled_changes.insert(module_id.clone());
4019 }
4020 if !configuration_changed && !enabled_changed {
4021 unchanged = unchanged.saturating_add(1);
4022 }
4023 }
4024
4025 if preview {
4031 return Ok(SupervisorRescanResult {
4032 added,
4033 removed,
4034 changed_pending_reload,
4035 enabled_changes: enabled_changes.iter().cloned().collect(),
4036 unchanged,
4037 preview: true,
4038 restart_required: Vec::new(),
4042 capability_warnings: Vec::new(),
4043 });
4044 }
4045
4046 for module_id in &removed {
4047 let module = ¤t
4048 .get(module_id)
4049 .expect("removed module came from current supervisor state")
4050 .0;
4051 module.retire().await.map_err(|err| {
4052 format!("failed to retire module_id '{module_id}' during rescan: {err}")
4053 })?;
4054 self.supervisor.record_rescan_removal(module_id);
4088 self.supervisor.retire(module_id);
4089 }
4090
4091 for module_id in configured.keys() {
4092 let Some((module, _, _, _)) = current.get(module_id) else {
4093 continue;
4094 };
4095 let configured_module = configured
4096 .get(module_id)
4097 .expect("configured module id came from configured map");
4098 if configuration_changes.contains(module_id) {
4099 module
4100 .update_configuration(
4101 configured_module.module_spec(),
4102 configured_module.health,
4103 configured_module.drain_timeout_ms,
4104 )
4105 .await
4106 .map_err(|err| {
4107 format!(
4108 "failed to update module_id '{module_id}' configuration during rescan: {err}"
4109 )
4110 })?;
4111 }
4112 if enabled_changes.contains(module_id) {
4113 module
4114 .set_enabled(configured_module.enabled)
4115 .await
4116 .map_err(|err| {
4117 format!(
4118 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
4119 configured_module.enabled
4120 )
4121 })?;
4122 }
4123 }
4124
4125 for module_id in &added {
4126 let configured_module = configured
4127 .get(module_id)
4128 .expect("added module id came from configured map");
4129 supervisor
4130 .supervise_configured_with_health(
4131 configured_module.module_spec(),
4132 configured_module.enabled,
4133 configured_module.health,
4134 configured_module.drain_timeout_ms,
4135 configured_module.restart,
4136 )
4137 .map_err(|err| {
4138 format!("failed to add module_id '{module_id}' during rescan: {err}")
4139 })?;
4140 }
4141
4142 Ok(SupervisorRescanResult {
4143 added,
4144 removed,
4145 changed_pending_reload,
4146 enabled_changes: enabled_changes.iter().cloned().collect(),
4147 unchanged,
4148 preview: false,
4149 restart_required: Vec::new(),
4152 capability_warnings: Vec::new(),
4153 })
4154 }
4155
4156 async fn handle_supervisor_set_enabled(
4157 &self,
4158 frame: Frame,
4159 module_id: String,
4160 enabled: bool,
4161 ) -> Result<Vec<Frame>, RouterError> {
4162 let operation_lock = self.supervisor.operation_lock();
4163 let _operation_guard = operation_lock.lock().await;
4164 let Some(module) = self.supervisor.get(&module_id) else {
4165 return Ok(vec![control_error_frame(
4166 &frame,
4167 "unknown_module",
4168 format!("module_id '{module_id}' is not supervised"),
4169 )?]);
4170 };
4171
4172 let applied = match module.set_enabled(enabled).await {
4173 Ok(applied) => applied,
4174 Err(err) => {
4175 return Ok(vec![control_error_frame(
4176 &frame,
4177 "target_unavailable",
4178 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
4179 )?])
4180 }
4181 };
4182
4183 self.capability_evaluator.wake_deadline_loop();
4184 self.refresh_capability_requirements();
4185 let response = ClientControlResponse::SupervisorAck { module_id, applied };
4186 Ok(vec![control_response_body_frame(
4187 &frame,
4188 &response,
4189 "ClientControlResponse::SupervisorAck",
4190 )?])
4191 }
4192
4193 async fn handle_supervisor_health_probe(
4194 &self,
4195 frame: Frame,
4196 module_id: String,
4197 ) -> Result<Vec<Frame>, RouterError> {
4198 self.refresh_capability_requirements();
4199 let Some(registration) = self
4200 .registry
4201 .get_module(&module_id)
4202 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4203 else {
4204 return Ok(vec![control_error_frame(
4205 &frame,
4206 "unknown_module",
4207 format!("module_id '{module_id}' is not registered"),
4208 )?]);
4209 };
4210
4211 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
4224 {
4225 return Ok(vec![control_error_frame(
4226 &frame,
4227 "health_not_advertised",
4228 format!("module_id '{module_id}' did not advertise health.check"),
4229 )?]);
4230 }
4231
4232 let deadline = Instant::now() + self.health_probe_timeout;
4233 let pending = match self.forwarding.begin_module_control_rpc_for(
4234 &module_id,
4235 MODULE_CONTROL_OP_HEALTH_CHECK,
4236 deadline,
4237 ) {
4238 Ok(pending) => pending,
4239 Err(err) => {
4240 return Ok(vec![control_error_frame(
4241 &frame,
4242 forwarding_error_code(&err),
4243 err.to_string(),
4244 )?])
4245 }
4246 };
4247
4248 let PendingModuleControlRpc {
4249 endpoint,
4250 module_sink,
4251 negotiated_ver,
4252 corr: probe_corr,
4253 receiver,
4254 } = pending;
4255 let mut guard =
4256 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
4257 let probe_body =
4258 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
4259 RouterError::backend(
4260 0,
4261 frame.header.corr,
4262 format!("failed to encode health.check request: {err}"),
4263 )
4264 })?;
4265 let probe_frame = Frame::build_with_version(
4266 negotiated_ver,
4267 FrameType::Request,
4268 control_flags(),
4269 0,
4270 0,
4271 probe_corr,
4272 probe_body,
4273 )
4274 .map_err(RouterError::FrameBuild)?;
4275
4276 if let Err(err) = module_sink.send(probe_frame).await {
4277 return Ok(vec![control_error_frame(
4278 &frame,
4279 "target_unavailable",
4280 err.to_string(),
4281 )?]);
4282 }
4283
4284 match timeout_at(deadline, receiver).await {
4285 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
4286 guard.disarm();
4287 let Some(report) = response.health_report() else {
4288 return Ok(vec![control_error_frame(
4289 &frame,
4290 "invalid_control_body",
4291 "health.check RPC returned a non-health response",
4292 )?]);
4293 };
4294 let HealthReport {
4299 status,
4300 detail,
4301 metrics,
4302 } = report;
4303 let capability_detail = self
4304 .capability_evaluator
4305 .required_problem_detail(&module_id);
4306 let response = ClientControlResponse::SupervisorHealthProbe {
4307 module_id,
4308 status,
4309 detail: append_capability_problem_detail(detail, capability_detail),
4310 metrics,
4311 };
4312 Ok(vec![control_response_body_frame(
4313 &frame,
4314 &response,
4315 "ClientControlResponse::SupervisorHealthProbe",
4316 )?])
4317 }
4318 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
4319 guard.disarm();
4320 Ok(vec![control_error_body_frame(&frame, body)?])
4321 }
4322 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
4323 guard.disarm();
4324 Ok(vec![control_error_frame(
4325 &frame,
4326 "target_unavailable",
4327 message,
4328 )?])
4329 }
4330 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
4331 guard.disarm();
4332 Ok(vec![control_error_frame(
4333 &frame,
4334 "invalid_control_body",
4335 message,
4336 )?])
4337 }
4338 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
4339 guard.disarm();
4340 Ok(vec![control_error_frame(
4341 &frame,
4342 "invalid_control_body",
4343 format!("expected module-control op '{expected}', got '{actual}'"),
4344 )?])
4345 }
4346 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
4347 guard.disarm();
4348 Ok(vec![control_error_frame(
4349 &frame,
4350 "module_timeout",
4351 format!(
4352 "module_id '{module_id}' answered health.check after {:?}",
4353 self.health_probe_timeout
4354 ),
4355 )?])
4356 }
4357 Ok(Err(_)) => Ok(vec![control_error_frame(
4358 &frame,
4359 "target_unavailable",
4360 "health.check waiter was canceled before the module responded",
4361 )?]),
4362 Err(_) => Ok(vec![control_error_frame(
4363 &frame,
4364 "module_timeout",
4365 format!(
4366 "module_id '{module_id}' did not answer health.check within {:?}",
4367 self.health_probe_timeout
4368 ),
4369 )?]),
4370 }
4371 }
4372
4373 fn supervisor_status(
4374 &self,
4375 module_id: &str,
4376 corr: u64,
4377 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
4378 self.supervisor
4379 .get(module_id)
4380 .map(|module| {
4381 let warming = module.is_warming_for_control("status").map_err(|err| {
4382 RouterError::backend(
4383 0,
4384 corr,
4385 format!(
4386 "failed to read supervisor warming state for module_id '{module_id}': {err}"
4387 ),
4388 )
4389 })?;
4390 module.status_for_control("status").map_err(|err| {
4391 RouterError::backend(
4392 0,
4393 corr,
4394 format!(
4395 "failed to read supervisor status for module_id '{module_id}': {err}"
4396 ),
4397 )
4398 }).map(|status| (status, warming))
4399 })
4400 .transpose()
4401 }
4402
4403 fn guard_module_control_op(
4404 &self,
4405 frame: &Frame,
4406 module_id: &str,
4407 op: &str,
4408 ) -> Result<Option<Frame>, RouterError> {
4409 if self.module_grants_op(module_id, op, frame.header.corr)? {
4410 return Ok(None);
4411 }
4412
4413 Ok(Some(control_error_frame(
4414 frame,
4415 "op_not_allowed",
4416 format!("module_id '{module_id}' did not grant control op '{op}'"),
4417 )?))
4418 }
4419
4420 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
4421 let Some(registration) = self
4422 .registry
4423 .get_module(module_id)
4424 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
4425 else {
4426 return Ok(false);
4427 };
4428 Ok(module_registration_grants_op(®istration.control_ops, op))
4429 }
4430
4431 fn handle_status_update(
4432 &self,
4433 endpoint: ModuleEndpointId,
4434 frame: Frame,
4435 ) -> Result<Vec<Frame>, RouterError> {
4436 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
4437 Ok(update) => update,
4438 Err(err) => {
4439 if is_known_module_push_op(&frame.body) {
4444 return Ok(vec![control_error_frame(
4445 &frame,
4446 "invalid_control_body",
4447 format!("malformed module control push body: {err}"),
4448 )?]);
4449 }
4450 return Ok(Vec::new());
4451 }
4452 };
4453
4454 match update {
4455 ModuleControlPush::RouteStatus {
4456 route_channel,
4457 route_epoch,
4458 status,
4459 } => {
4460 self.forwarding
4461 .cache_status(endpoint, route_channel, route_epoch, status)
4462 .map_err(RouterError::Forwarding)?;
4463 }
4464 }
4465 Ok(Vec::new())
4466 }
4467
4468 fn handle_route_poll(
4469 &self,
4470 ctx: &RouteCtx,
4471 frame: Frame,
4472 route_channel: u16,
4473 route_epoch: u32,
4474 kind: PollKind,
4475 ) -> Result<Vec<Frame>, RouterError> {
4476 let snapshot = self
4477 .forwarding
4478 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
4479 .map_err(RouterError::Forwarding)?;
4480 let response = match (kind, snapshot) {
4481 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
4482 ClientControlResponse::RoutePoll {
4483 route_channel,
4484 route_epoch,
4485 status,
4486 live: None,
4487 }
4488 }
4489 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
4490 route_channel,
4491 route_epoch,
4492 status: None,
4493 live: None,
4494 },
4495 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
4496 let live = self
4522 .process_liveness
4523 .as_ref()
4524 .and_then(|source| source.process_live(&module_id))
4525 .unwrap_or(true);
4526 ClientControlResponse::RoutePoll {
4527 route_channel,
4528 route_epoch,
4529 status: None,
4530 live: Some(live),
4531 }
4532 }
4533 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
4534 route_channel,
4535 route_epoch,
4536 status: None,
4537 live: Some(false),
4538 },
4539 };
4540
4541 Ok(vec![control_response_body_frame(
4542 &frame,
4543 &response,
4544 "ClientControlResponse::RoutePoll",
4545 )?])
4546 }
4547
4548 pub(crate) fn observe_module_control_completion(
4549 &self,
4550 completion: ModuleControlRpcCompletion,
4551 ) -> bool {
4552 match completion {
4553 ModuleControlRpcCompletion::Unknown => false,
4554 ModuleControlRpcCompletion::Settled => true,
4555 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
4556 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
4557 info!(
4558 module_id = %module_id,
4559 latency_ms,
4560 "late health.check answer proves the module is alive"
4561 );
4562 match self
4563 .supervisor
4564 .record_late_health_answer(&module_id, latency_ms)
4565 {
4566 Ok(true) => {}
4567 Ok(false) => debug!(
4568 module_id = %module_id,
4569 latency_ms,
4570 "late health.check answer has no active supervisor snapshot"
4571 ),
4572 Err(err) => warn!(
4573 module_id = %module_id,
4574 latency_ms,
4575 error = %err,
4576 "failed to record late health.check answer"
4577 ),
4578 }
4579 true
4580 }
4581 }
4582 }
4583
4584 fn refuse_to_end_module_connection_for_a_client(
4605 &self,
4606 module_connection_id: ConnectionId,
4607 corr: u64,
4608 err: ForwardingError,
4609 ) -> Result<(), RouterError> {
4610 if let ForwardingError::ConnectionClosing { connection_id } = err {
4611 if connection_id != module_connection_id {
4612 warn!(
4613 module_connection_id = module_connection_id.get(),
4614 client_connection_id = connection_id.get(),
4615 corr,
4616 "dropping a route.bind response for a closing client; the module connection keeps serving"
4617 );
4618 return Ok(());
4619 }
4620 }
4621 Err(RouterError::Forwarding(err))
4622 }
4623
4624 fn handle_module_relay_response(
4625 &self,
4626 connection_id: ConnectionId,
4627 frame: Frame,
4628 ) -> Result<Vec<Frame>, RouterError> {
4629 let mut secondary_error = None;
4630 let outcome = match frame.header.ty {
4631 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
4632 Ok(probe) if probe.op == "route.bind" => {
4633 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
4634 Ok(ModuleControlResponse::RouteBindAck {}) => {
4635 RouteBindRelayOutcome::Accepted
4636 }
4637 Ok(other) => {
4638 let message =
4639 format!("route.bind response carried unexpected body: {other:?}");
4640 secondary_error = Some(control_error_frame(
4641 &frame,
4642 "invalid_control_body",
4643 message.clone(),
4644 )?);
4645 RouteBindRelayOutcome::ModuleGone(message)
4646 }
4647 Err(err) => {
4648 let message = format!("malformed route.bind response body: {err}");
4649 secondary_error = Some(control_error_frame(
4650 &frame,
4651 "invalid_control_body",
4652 message.clone(),
4653 )?);
4654 RouteBindRelayOutcome::ModuleGone(message)
4655 }
4656 }
4657 }
4658 Ok(probe) => {
4659 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
4660 {
4661 Ok(response) => ModuleControlRpcOutcome::Response(response),
4662 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
4663 "malformed {} response body: {err}",
4664 probe.op
4665 )),
4666 };
4667 let completion = self
4668 .forwarding
4669 .complete_module_control_rpc(
4670 connection_id,
4671 frame.header.corr,
4672 Some(&probe.op),
4673 outcome,
4674 )
4675 .map_err(RouterError::Forwarding)?;
4676 if !self.observe_module_control_completion(completion) {
4677 debug!(
4678 connection_id = connection_id.get(),
4679 corr = frame.header.corr,
4680 op = %probe.op,
4681 "dropping late or unknown module-control RPC response"
4682 );
4683 }
4684 return Ok(Vec::new());
4685 }
4686 Err(err) => {
4687 if let Some(expected_op) = self
4688 .forwarding
4689 .pending_module_control_op(connection_id, frame.header.corr)
4690 .map_err(RouterError::Forwarding)?
4691 {
4692 let completion = self
4693 .forwarding
4694 .complete_module_control_rpc(
4695 connection_id,
4696 frame.header.corr,
4697 None,
4698 ModuleControlRpcOutcome::MalformedResponse(format!(
4699 "malformed {expected_op} response body: {err}"
4700 )),
4701 )
4702 .map_err(RouterError::Forwarding)?;
4703 if !self.observe_module_control_completion(completion) {
4704 debug!(
4705 connection_id = connection_id.get(),
4706 corr = frame.header.corr,
4707 "dropping late malformed module-control RPC response"
4708 );
4709 }
4710 return Ok(Vec::new());
4711 }
4712 let message = format!("malformed route.bind response body: {err}");
4713 secondary_error = Some(control_error_frame(
4714 &frame,
4715 "invalid_control_body",
4716 message.clone(),
4717 )?);
4718 RouteBindRelayOutcome::ModuleGone(message)
4719 }
4720 },
4721 FrameType::Error => {
4722 if self
4723 .forwarding
4724 .pending_module_control_op(connection_id, frame.header.corr)
4725 .map_err(RouterError::Forwarding)?
4726 .is_some()
4727 {
4728 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
4729 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
4730 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
4731 "malformed module-control ERROR body: {err}"
4732 )),
4733 };
4734 let completion = self
4735 .forwarding
4736 .complete_module_control_rpc(
4737 connection_id,
4738 frame.header.corr,
4739 None,
4740 outcome,
4741 )
4742 .map_err(RouterError::Forwarding)?;
4743 if !self.observe_module_control_completion(completion) {
4744 debug!(
4745 connection_id = connection_id.get(),
4746 corr = frame.header.corr,
4747 "dropping late or unknown module-control RPC error"
4748 );
4749 }
4750 return Ok(Vec::new());
4751 }
4752 match serde_json::from_slice::<ErrorBody>(&frame.body) {
4753 Ok(body) => RouteBindRelayOutcome::Rejected(body),
4754 Err(err) => {
4755 let message = format!("malformed route.bind ERROR body: {err}");
4756 secondary_error = Some(control_error_frame(
4757 &frame,
4758 "invalid_control_body",
4759 message.clone(),
4760 )?);
4761 RouteBindRelayOutcome::ModuleGone(message)
4762 }
4763 }
4764 }
4765 ty => {
4766 return Ok(vec![control_error_frame(
4767 &frame,
4768 "unsupported_control_frame",
4769 format!("unsupported module channel-0 frame {ty:?}"),
4770 )?])
4771 }
4772 };
4773
4774 let settled =
4775 self.forwarding
4776 .complete_pending_relay(connection_id, frame.header.corr, outcome);
4777 let completion = match settled {
4778 Ok(completion) => completion,
4779 Err(err) => {
4780 self.refuse_to_end_module_connection_for_a_client(
4781 connection_id,
4782 frame.header.corr,
4783 err,
4784 )?;
4785 return Ok(secondary_error.into_iter().collect());
4786 }
4787 };
4788 if let Some(target) = completion.abandoned.as_ref() {
4789 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
4790 }
4791 if !completion.settled {
4792 debug!(
4793 connection_id = connection_id.get(),
4794 corr = frame.header.corr,
4795 frame_type = ?frame.header.ty,
4796 "dropping late or unknown route.bind relay response"
4797 );
4798 }
4799 Ok(secondary_error.into_iter().collect())
4800 }
4801
4802 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
4803 debug!(connection_id = connection_id.get(), "handling GOODBYE");
4804 let registrations = self
4805 .deregister_connection(connection_id)
4806 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
4807 let released_routes = self
4808 .forwarding
4809 .cleanup_connection(connection_id)
4810 .map_err(RouterError::Forwarding)?;
4811 self.emit_route_goodbyes(released_routes);
4812 if !registrations.is_empty() {
4814 crate::supervise::notify_registration_release();
4815 }
4816 Ok(Vec::new())
4817 }
4818}
4819
4820impl Default for ControlHandler {
4821 fn default() -> Self {
4822 Self::new(Arc::new(Registry::default()))
4823 }
4824}
4825
4826impl crate::supervise::SwapPromotionObserver for ControlHandler {
4827 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
4828 self.apply_registration_capabilities(registration);
4829 }
4830}
4831
4832fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
4833 CapabilityRequirementStatus {
4834 consumer: status.consumer,
4835 capability: status.capability,
4836 need: match status.need {
4837 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
4838 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
4839 },
4840 verdict: status.verdict.as_str().to_string(),
4841 episode_seq: status.episode_seq,
4842 config_satisfiable: status.config_satisfiable,
4843 runtime_available: status.runtime_available,
4844 detail: status.detail,
4845 }
4846}
4847
4848fn append_capability_problem_detail(
4849 detail: Option<String>,
4850 capability_detail: Option<String>,
4851) -> Option<String> {
4852 match (detail, capability_detail) {
4853 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
4854 (Some(detail), None) => Some(detail),
4855 (None, Some(capability_detail)) => Some(capability_detail),
4856 (None, None) => None,
4857 }
4858}
4859
4860fn subc_ops() -> Vec<String> {
4861 SUBC_CONTROL_OPS
4862 .iter()
4863 .map(|op| (*op).to_string())
4864 .collect()
4865}
4866
4867fn module_subc_ops() -> Vec<String> {
4868 SUBC_CONTROL_OPS
4869 .iter()
4870 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
4871 .map(|op| (*op).to_string())
4872 .collect()
4873}
4874
4875#[cfg(test)]
4876fn module_baseline_control_ops() -> Vec<String> {
4877 MODULE_BASELINE_CONTROL_OPS
4878 .iter()
4879 .map(|op| (*op).to_string())
4880 .collect()
4881}
4882
4883fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
4884 let mut seen = HashSet::new();
4885 let mut effective = Vec::new();
4886 for op in MODULE_BASELINE_CONTROL_OPS {
4887 if seen.insert((*op).to_string()) {
4888 effective.push((*op).to_string());
4889 }
4890 }
4891 for op in declared.unwrap_or_default() {
4892 if seen.insert(op.clone()) {
4893 effective.push(op);
4894 }
4895 }
4896 effective
4897}
4898
4899fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
4900 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
4901}
4902
4903fn target_module_id(target: &RouteTarget) -> &str {
4904 match target {
4905 RouteTarget::ToolProvider { module_id }
4906 | RouteTarget::ManagementSurface { module_id }
4907 | RouteTarget::InternalService { module_id, .. } => module_id,
4908 }
4909}
4910
4911fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
4912 roles.iter().any(|role| match (target, role) {
4913 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
4914 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
4915 (
4916 RouteTarget::InternalService { service_id, .. },
4917 ProviderRole::InternalService {
4918 service_id: provided,
4919 ..
4920 },
4921 ) => service_id == provided,
4922 _ => false,
4923 })
4924}
4925
4926fn is_routable_role(role: &ProviderRole) -> bool {
4927 matches!(
4928 role,
4929 ProviderRole::ToolProvider { .. }
4930 | ProviderRole::ManagementSurface { .. }
4931 | ProviderRole::InternalService { .. }
4932 )
4933}
4934
4935#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4936enum ControlRequestBodyError {
4937 UnknownOp,
4938 InvalidBody,
4939}
4940
4941#[derive(Debug, Deserialize)]
4942struct ControlOpProbe {
4943 op: String,
4944}
4945
4946const MODULE_PUSH_OPS: &[&str] = &["route.status"];
4949
4950fn is_known_module_push_op(body: &[u8]) -> bool {
4951 serde_json::from_slice::<ControlOpProbe>(body)
4952 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
4953 .unwrap_or(false)
4954}
4955
4956fn is_known_module_request_op(body: &[u8]) -> bool {
4957 serde_json::from_slice::<ControlOpProbe>(body)
4958 .map(|probe| MODULE_TO_SUBC_CONTROL_OPS.contains(&probe.op.as_str()))
4959 .unwrap_or(false)
4960}
4961
4962fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
4963 debug!(
4964 op = %op,
4965 connection_id = connection_id.get(),
4966 corr,
4967 "control dispatch"
4968 );
4969}
4970
4971fn log_slow_control_dispatch(
4972 dispatch_started_at: Option<StdInstant>,
4973 op: &'static str,
4974 connection_id: ConnectionId,
4975 corr: u64,
4976) {
4977 let Some(dispatch_started_at) = dispatch_started_at else {
4978 return;
4979 };
4980 let elapsed = dispatch_started_at.elapsed();
4981 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
4982 warn!(
4983 op = %op,
4984 connection_id = connection_id.get(),
4985 corr,
4986 elapsed_ms = elapsed.as_millis() as u64,
4987 "slow control dispatch"
4988 );
4989 }
4990}
4991
4992fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
4993 match request {
4994 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
4995 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
4996 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
4997 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
4998 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
4999 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
5000 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
5001 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
5002 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
5003 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
5004 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
5005 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
5006 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
5007 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
5008 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
5009 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
5010 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
5011 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
5012 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
5013 }
5014}
5015
5016fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
5017 match request {
5018 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
5019 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
5020 }
5021}
5022
5023fn parse_client_control_request(
5024 body: &[u8],
5025) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
5026 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
5027 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5028 Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
5029 ControlRequestBodyError::InvalidBody
5030 }
5031 Ok(_) => ControlRequestBodyError::UnknownOp,
5032 Err(_) => ControlRequestBodyError::InvalidBody,
5033 };
5034 (err, classification)
5035 })
5036}
5037
5038fn parse_module_control_request_from_module(
5039 body: &[u8],
5040) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
5041 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
5042 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5043 Ok(probe) if MODULE_TO_SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
5044 ControlRequestBodyError::InvalidBody
5045 }
5046 Ok(_) => ControlRequestBodyError::UnknownOp,
5047 Err(_) => ControlRequestBodyError::InvalidBody,
5048 };
5049 (err, classification)
5050 })
5051}
5052
5053#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
5054enum ProviderRoleKind {
5055 ToolProvider,
5056 PipelineStage,
5057 ManagementSurface,
5058 InternalService,
5059}
5060
5061fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
5062 match role {
5063 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
5064 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
5065 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
5066 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
5067 }
5068}
5069
5070fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
5071 roles.iter().map(provider_role_kind).collect()
5072}
5073
5074fn capability_census_trigger(
5078 old: Option<&CapabilityDeclarations>,
5079 new: Option<&CapabilityDeclarations>,
5080) -> bool {
5081 let old_provides = old
5082 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
5083 .unwrap_or_default();
5084 let old_denies = old
5085 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
5086 .unwrap_or_default();
5087 let new = new.cloned().unwrap_or(CapabilityDeclarations {
5088 provides: Vec::new(),
5089 requires: Vec::new(),
5090 must_never_reach: Vec::new(),
5091 });
5092
5093 new.provides
5094 .iter()
5095 .any(|capability| !old_provides.contains(capability))
5096 || new
5097 .must_never_reach
5098 .iter()
5099 .any(|capability| !old_denies.contains(capability))
5100}
5101
5102fn denied_capability<'a>(
5105 opening_manifest: &'a ModuleManifest,
5106 target_manifest: &ModuleManifest,
5107) -> Option<&'a str> {
5108 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
5109 let target_capabilities = target_manifest.capabilities.as_ref()?;
5110 opening_capabilities
5111 .must_never_reach
5112 .iter()
5113 .find(|denied| {
5114 target_capabilities
5115 .provides
5116 .iter()
5117 .any(|provided| provided == *denied)
5118 })
5119 .map(String::as_str)
5120}
5121
5122fn catalog_update_frozen_field_message(
5123 registered: &ModuleManifest,
5124 provides: &[ProviderRole],
5125) -> Option<String> {
5126 let old_has_provides = !registered.provides.is_empty();
5127 let new_has_provides = !provides.is_empty();
5128 if old_has_provides != new_has_provides {
5129 return Some(format!(
5130 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
5131 registered.module_id
5132 ));
5133 }
5134
5135 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
5136 return Some(format!(
5137 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
5138 registered.module_id
5139 ));
5140 }
5141
5142 let registered_concurrency = manifest_concurrency(registered);
5143 let mut candidate = registered.clone();
5144 candidate.provides = provides.to_vec();
5145 let candidate_concurrency = manifest_concurrency(&candidate);
5146 if candidate_concurrency != registered_concurrency {
5147 return Some(format!(
5148 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
5149 registered.module_id, registered_concurrency, candidate_concurrency
5150 ));
5151 }
5152
5153 None
5156}
5157
5158fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
5159 manifest.provides.iter().any(is_routable_role)
5160}
5161
5162fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
5168 manifest
5169 .provides
5170 .iter()
5171 .find_map(|provider| match provider {
5172 ProviderRole::ToolProvider { concurrency, .. }
5173 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
5174 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
5175 })
5176 .unwrap_or(Concurrency::ModuleManaged)
5177}
5178
5179fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
5186 let has_management_surface = manifest
5187 .provides
5188 .iter()
5189 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
5190 if !has_management_surface {
5191 return false;
5192 }
5193 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
5194 return false;
5195 };
5196 let Some(provides) = raw
5197 .get("manifest")
5198 .and_then(|manifest| manifest.get("provides"))
5199 .and_then(serde_json::Value::as_array)
5200 else {
5201 return false;
5202 };
5203 provides.iter().any(|role| {
5208 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
5209 && role.get("concurrency").is_none()
5210 })
5211}
5212
5213fn negotiate_version(peer_version: u8) -> Result<u8, String> {
5214 if peer_version != PROTOCOL_VERSION {
5215 return Err(format!(
5216 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
5217 ));
5218 }
5219 Ok(PROTOCOL_VERSION)
5220}
5221
5222fn pong(frame: &Frame) -> Result<Frame, RouterError> {
5223 Frame::build_with_version(
5224 response_version(frame),
5225 FrameType::Pong,
5226 frame.header.flags,
5227 0,
5228 0,
5229 frame.header.corr,
5230 Vec::new(),
5231 )
5232 .map_err(RouterError::FrameBuild)
5233}
5234
5235fn control_error_frame(
5236 frame: &Frame,
5237 code: &'static str,
5238 message: impl Into<String>,
5239) -> Result<Frame, RouterError> {
5240 control_error_body_frame(
5241 frame,
5242 ErrorBody {
5243 code: code.to_string(),
5244 message: message.into(),
5245 detail: None,
5246 },
5247 )
5248}
5249
5250fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
5251 let body = serde_json::to_vec(&error).map_err(|err| {
5252 RouterError::backend(
5253 0,
5254 frame.header.corr,
5255 format!("failed to encode control ERROR: {err}"),
5256 )
5257 })?;
5258
5259 Frame::build_with_version(
5260 response_version(frame),
5261 FrameType::Error,
5262 control_flags(),
5263 0,
5264 0,
5265 frame.header.corr,
5266 body,
5267 )
5268 .map_err(RouterError::FrameBuild)
5269}
5270
5271fn control_response_body_frame<T: Serialize>(
5272 frame: &Frame,
5273 reply: &T,
5274 label: &'static str,
5275) -> Result<Frame, RouterError> {
5276 let body = serde_json::to_vec(reply).map_err(|err| {
5277 RouterError::backend(
5278 0,
5279 frame.header.corr,
5280 format!("failed to encode {label}: {err}"),
5281 )
5282 })?;
5283
5284 Frame::build_with_version(
5285 response_version(frame),
5286 FrameType::Response,
5287 control_flags(),
5288 0,
5289 0,
5290 frame.header.corr,
5291 body,
5292 )
5293 .map_err(RouterError::FrameBuild)
5294}
5295
5296fn forwarding_error_code(err: &ForwardingError) -> &'static str {
5326 match err {
5327 ForwardingError::NoModuleConnection => "target_unavailable",
5328 ForwardingError::ModuleReloading { .. } => "module_reloading",
5329 ForwardingError::ClientRouteChannelExhausted { .. }
5330 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
5331 ForwardingError::StaleModuleEndpoint
5332 | ForwardingError::UnknownReservation { .. }
5333 | ForwardingError::ConnectionClosing { .. }
5334 | ForwardingError::ClientEgressClosed { .. }
5335 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
5336 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
5339 ForwardingError::RelayCorrelationExhausted
5340 | ForwardingError::RouteOpenBuild(_)
5341 | ForwardingError::Poisoned => "forwarding_error",
5342 }
5343}
5344
5345fn response_version(frame: &Frame) -> u8 {
5346 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
5347 frame.header.ver
5348 } else {
5349 PROTOCOL_VERSION
5350 }
5351}
5352
5353fn control_flags() -> Flags {
5354 Flags::new(false, Priority::Passive, false)
5355}
5356
5357fn send_goodbye_target_best_effort(
5362 counters: &DaemonCounters,
5363 target: &GoodbyeTarget,
5364 context: &'static str,
5365) {
5366 let Ok(frame) = Frame::build_with_version(
5367 target.negotiated_ver,
5368 FrameType::Goodbye,
5369 control_flags(),
5370 target.channel,
5371 target.epoch,
5372 0,
5373 Vec::new(),
5374 ) else {
5375 return;
5376 };
5377 crate::forwarding::send_module_route_goodbye(
5378 counters,
5379 &target.sink,
5380 frame,
5381 target.module_id.as_deref(),
5382 context,
5383 );
5384}
5385
5386pub(crate) fn send_route_control_pushes(
5387 forwarding: &ForwardingTable,
5388 routes: Vec<EndpointRoute>,
5389 push: ClientControlPush,
5390) {
5391 let body = match serde_json::to_vec(&push) {
5392 Ok(body) => body,
5393 Err(err) => {
5394 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
5395 return;
5396 }
5397 };
5398 let mut targets = Vec::new();
5399 for route in routes {
5400 let target = route.goodbye_target;
5401 if let Some(existing) = targets
5402 .iter()
5403 .find(|existing: &&GoodbyeTarget| existing.connection_id == target.connection_id)
5404 {
5405 debug_assert_eq!(
5406 existing.negotiated_ver, target.negotiated_ver,
5407 "one connection cannot negotiate multiple frame versions"
5408 );
5409 continue;
5410 }
5411 targets.push(target);
5412 }
5413 for target in targets {
5414 let frame = match Frame::build_with_version(
5415 target.negotiated_ver,
5416 FrameType::Push,
5417 control_flags(),
5418 0,
5419 0,
5420 0,
5421 body.clone(),
5422 ) {
5423 Ok(frame) => frame,
5424 Err(err) => {
5425 warn!(
5426 route_channel = target.channel,
5427 error = %err,
5428 "failed to build route lifecycle control PUSH frame"
5429 );
5430 continue;
5431 }
5432 };
5433 if let Err(err) = target.sink.try_send(frame) {
5434 if target.close_on_delivery_failure() {
5435 warn!(
5436 target_connection_id = target.connection_id.get(),
5437 route_channel = target.channel,
5438 error = %err,
5439 "route lifecycle control PUSH was not delivered to client; closing target connection"
5440 );
5441 let _ = forwarding.escalate_client_delivery_failure(
5442 target.connection_id,
5443 target.channel,
5444 target.epoch,
5445 CloseReason::new(
5446 "route_lifecycle_push_delivery_failed",
5447 format!(
5448 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
5449 target.channel
5450 ),
5451 ),
5452 crate::forwarding::UndeliveredFrame {
5453 module_id: target.module_id.as_deref(),
5454 sink: &target.sink,
5455 },
5456 );
5457 }
5458 }
5459 }
5460}
5461
5462#[cfg(test)]
5463mod tests {
5464 use std::{
5465 collections::BTreeMap,
5466 fmt,
5467 path::PathBuf,
5468 sync::{Arc, Mutex},
5469 time::Duration,
5470 };
5471
5472 use serde_json::{json, Value};
5473 use subc_protocol::{
5474 manifest::{
5475 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
5476 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
5477 },
5478 session::HealthStatus,
5479 FrameType,
5480 };
5481
5482 use super::*;
5483 use crate::{
5484 forwarding::{DataRoute, DataRouteState},
5485 registry::ChannelState,
5486 router::FrameSink,
5487 stderr_tail::DEFAULT_MAX_LINE_BYTES,
5488 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
5489 test_support::TestTempDir,
5490 RouteCtx, Router,
5491 };
5492 use tokio::{
5493 sync::mpsc,
5494 time::{sleep, Instant},
5495 };
5496 use tracing::{
5497 field::{Field, Visit},
5498 Event, Subscriber,
5499 };
5500 use tracing_subscriber::{layer::Context, prelude::*, Layer};
5501
5502 fn fake_aft_stub_path() -> PathBuf {
5521 let mut path = std::env::current_exe().expect("current_exe available in tests");
5522 path.pop(); path.pop(); path.push(if cfg!(windows) {
5525 "fake-aft-stub.exe"
5526 } else {
5527 "fake-aft-stub"
5528 });
5529 assert!(
5530 path.exists(),
5531 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
5532 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
5533 path.display()
5534 );
5535 path
5536 }
5537
5538 fn client_retries(code: &str) -> bool {
5543 subc_protocol::error_codes::is_retryable_route_open(code)
5544 }
5545
5546 #[test]
5557 fn retryability_of_forwarding_codes_matches_the_failure() {
5558 let transient = [
5561 ForwardingError::NoModuleConnection,
5562 ForwardingError::ModuleReloading {
5563 module_id: "m".into(),
5564 },
5565 ForwardingError::StaleModuleEndpoint,
5566 ForwardingError::UnknownReservation {
5567 client_channel: 1,
5568 module_channel: 1,
5569 },
5570 ForwardingError::ConnectionClosing {
5571 connection_id: ConnectionId::new(1),
5572 },
5573 ForwardingError::ClientEgressClosed {
5574 connection_id: ConnectionId::new(1),
5575 },
5576 ForwardingError::ModuleEgressUnavailable {
5577 connection_id: ConnectionId::new(1),
5578 },
5579 ];
5580 for err in transient {
5581 let code = forwarding_error_code(&err);
5582 assert!(
5583 client_retries(code),
5584 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
5585 );
5586 }
5587
5588 let permanent = [
5593 ForwardingError::ClientRouteChannelExhausted {
5594 connection_id: ConnectionId::new(1),
5595 },
5596 ForwardingError::ModuleRouteChannelExhausted {
5597 endpoint: ModuleEndpointId {
5598 connection_id: ConnectionId::new(1),
5599 generation: 1,
5600 },
5601 },
5602 ForwardingError::RelayCorrelationExhausted,
5603 ForwardingError::RouteOpenBuild("x".into()),
5604 ForwardingError::Poisoned,
5605 ];
5606 for err in permanent {
5607 let code = forwarding_error_code(&err);
5608 assert!(
5609 !client_retries(code),
5610 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
5611 );
5612 }
5613 }
5614
5615 #[tokio::test]
5643 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
5644 let handler = ControlHandler::default();
5645 let frame =
5646 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
5647
5648 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
5652 assert_eq!(
5653 stamped,
5654 Principal::Direct,
5655 "a caller that proved nothing must not be stamped as a supervised module"
5656 );
5657
5658 let forged = handler
5663 .route_open_principal(
5664 &frame,
5665 Some(ConsumerIdentity {
5666 module_id: "aft".to_string(),
5667 launch_nonce: "not-a-real-nonce".to_string(),
5668 }),
5669 )
5670 .unwrap();
5671 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
5672 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
5673 }
5674
5675 #[test]
5692 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
5693 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
5694 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
5695 let ClientControlRequest::RouteOpen {
5696 consumer_identity, ..
5697 } = parsed
5698 else {
5699 panic!("route.open body must parse as RouteOpen");
5700 };
5701 assert_eq!(
5702 consumer_identity,
5703 Some(ConsumerIdentity {
5704 module_id: "aft".to_string(),
5705 launch_nonce: "n".to_string(),
5706 }),
5707 "the wire field name must reach the value route_open_principal reads"
5708 );
5709 }
5710
5711 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
5712 ModuleManifest::builder(module_id, "0.1.0")
5713 .protocol_ver(protocol_ver)
5714 .provides(vec![ProviderRole::ToolProvider {
5715 tools: vec![Tool {
5716 name: "read".to_string(),
5717 description: None,
5718 execution_mode: ExecutionMode::Pure,
5719 schema: json!({"type": "object"}),
5720 }],
5721 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
5722 concurrency: Concurrency::ModuleManaged,
5723 emits_push: true,
5724 sub_supervises: true,
5725 }])
5726 .build()
5727 }
5728
5729 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
5730 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
5731 }
5732
5733 fn hello_frame_with_control_ops(
5734 module_id: &str,
5735 protocol_ver: u8,
5736 corr: u64,
5737 control_ops: Option<Vec<String>>,
5738 ) -> Frame {
5739 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
5740 }
5741
5742 fn hello_frame_with_nonce(
5743 module_id: &str,
5744 protocol_ver: u8,
5745 corr: u64,
5746 launch_nonce: Option<&str>,
5747 ) -> Frame {
5748 hello_frame_full(
5749 module_id,
5750 protocol_ver,
5751 corr,
5752 None,
5753 launch_nonce.map(ToOwned::to_owned),
5754 )
5755 }
5756
5757 fn hello_frame_full(
5758 module_id: &str,
5759 protocol_ver: u8,
5760 corr: u64,
5761 control_ops: Option<Vec<String>>,
5762 launch_nonce: Option<String>,
5763 ) -> Frame {
5764 let body = serde_json::to_vec(&ModuleHelloBody {
5765 manifest: manifest(module_id, protocol_ver),
5766 protocol_ver,
5767 control_ops,
5768 launch_nonce,
5769 })
5770 .unwrap();
5771 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
5772 }
5773
5774 fn non_routable_hello_frame_with_control_ops(
5775 module_id: &str,
5776 corr: u64,
5777 control_ops: Option<Vec<String>>,
5778 ) -> Frame {
5779 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
5780 manifest.provides.clear();
5781 let body = serde_json::to_vec(&ModuleHelloBody {
5782 manifest,
5783 protocol_ver: PROTOCOL_VERSION,
5784 control_ops,
5785 launch_nonce: None,
5786 })
5787 .unwrap();
5788 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
5789 }
5790
5791 fn capability_grammar_hello_frame(
5792 capabilities: Value,
5793 runtime_computed: Option<Value>,
5794 corr: u64,
5795 ) -> Frame {
5796 let mut body = serde_json::to_value(ModuleHelloBody {
5797 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
5798 protocol_ver: PROTOCOL_VERSION,
5799 control_ops: None,
5800 launch_nonce: None,
5801 })
5802 .expect("HELLO body serializes");
5803 body["manifest"]["capabilities"] = capabilities;
5804 if let Some(runtime_computed) = runtime_computed {
5805 body["runtime_computed"] = runtime_computed;
5806 }
5807 Frame::build(
5808 FrameType::Hello,
5809 control_flags(),
5810 0,
5811 0,
5812 corr,
5813 serde_json::to_vec(&body).expect("HELLO body reserializes"),
5814 )
5815 .expect("HELLO frame builds")
5816 }
5817
5818 fn channel_request(channel: u16, corr: u64) -> Frame {
5819 Frame::build(
5820 FrameType::Request,
5821 Flags::new(true, Priority::Interactive, false),
5822 channel,
5823 0,
5824 corr,
5825 b"opaque".to_vec(),
5826 )
5827 .unwrap()
5828 }
5829
5830 fn route_ctx(
5831 connection_id: ConnectionId,
5832 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
5833 let (tx, rx) = mpsc::channel(8);
5834 (
5835 RouteCtx {
5836 connection_id,
5837 egress: FrameSink::new(tx),
5838 },
5839 rx,
5840 )
5841 }
5842
5843 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
5844 serde_json::from_slice(&frame.body).unwrap()
5845 }
5846
5847 async fn hello_via_sink(
5852 handler: &ControlHandler,
5853 ctx: &RouteCtx,
5854 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
5855 hello: Frame,
5856 ) -> Frame {
5857 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
5858 assert!(
5859 replies.is_empty(),
5860 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
5861 );
5862 let ack = rx
5863 .try_recv()
5864 .expect("HELLO_ACK is queued on the module sink")
5865 .frame;
5866 assert_eq!(ack.header.ty, FrameType::HelloAck);
5867 ack
5868 }
5869
5870 fn parse_error(frame: &Frame) -> Value {
5871 serde_json::from_slice(&frame.body).unwrap()
5872 }
5873
5874 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
5875 serde_json::from_slice(&frame.body).unwrap()
5876 }
5877
5878 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
5879 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
5880 route_channel,
5881 route_epoch: 0,
5882 kind,
5883 })
5884 .unwrap();
5885 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
5886 }
5887
5888 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
5889 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
5890 module_id: module_id.to_string(),
5891 })
5892 .unwrap();
5893 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
5894 }
5895
5896 fn route_open_frame(corr: u64, module_id: &str, project_root: TestTempDir) -> Frame {
5897 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
5898 }
5899
5900 fn route_open_frame_with_consumer_capabilities(
5901 corr: u64,
5902 module_id: &str,
5903 project_root: TestTempDir,
5904 consumer_capabilities: Option<Vec<String>>,
5905 ) -> Frame {
5906 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
5907 target: RouteTarget::ToolProvider {
5908 module_id: module_id.to_string(),
5909 },
5910 identity: BindIdentity::new(
5911 project_root.path().to_path_buf(),
5912 "unit".to_string(),
5913 "session".to_string(),
5914 ),
5915 consumer_identity: None,
5916 consumer_capabilities,
5917 admission_facts: None,
5918 })
5919 .unwrap();
5920 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
5921 }
5922
5923 fn route_open_frame_with_admission_facts(
5924 corr: u64,
5925 module_id: &str,
5926 project_root: TestTempDir,
5927 consumer_identity: Option<subc_control::ConsumerIdentity>,
5928 facts: Option<Value>,
5929 ) -> Frame {
5930 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
5931 target: RouteTarget::ToolProvider {
5932 module_id: module_id.to_string(),
5933 },
5934 identity: BindIdentity::new(
5935 project_root.path().to_path_buf(),
5936 "unit".to_string(),
5937 format!("session-{corr}"),
5938 ),
5939 consumer_identity,
5940 consumer_capabilities: None,
5941 admission_facts: facts,
5942 })
5943 .unwrap();
5944 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
5945 }
5946
5947 #[derive(Clone, Default)]
5948 struct EventCapture {
5949 events: Arc<Mutex<Vec<CapturedEvent>>>,
5950 }
5951
5952 #[derive(Clone, Debug)]
5953 struct CapturedEvent {
5954 target: String,
5955 fields: BTreeMap<String, String>,
5956 }
5957
5958 impl EventCapture {
5959 fn events(&self) -> Vec<CapturedEvent> {
5960 self.events.lock().unwrap().clone()
5961 }
5962 }
5963
5964 impl<S> Layer<S> for EventCapture
5965 where
5966 S: Subscriber,
5967 {
5968 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
5969 let mut visitor = EventFieldVisitor::default();
5970 event.record(&mut visitor);
5971 self.events.lock().unwrap().push(CapturedEvent {
5972 target: event.metadata().target().to_string(),
5973 fields: visitor.fields,
5974 });
5975 }
5976 }
5977
5978 #[derive(Default)]
5979 struct EventFieldVisitor {
5980 fields: BTreeMap<String, String>,
5981 }
5982
5983 impl Visit for EventFieldVisitor {
5984 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
5985 self.fields
5986 .insert(field.name().to_string(), format!("{value:?}"));
5987 }
5988 }
5989
5990 fn health_response(corr: u64, status: HealthStatus) -> Frame {
5991 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
5992 status,
5993 detail: Some("warming".to_string()),
5994 metrics: Some(json!({"queue_depth": 3})),
5995 })
5996 .unwrap();
5997 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
5998 }
5999
6000 fn route_bind_ack(corr: u64) -> Frame {
6001 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
6002 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
6003 }
6004
6005 fn unique_project_root(label: &str) -> TestTempDir {
6006 TestTempDir::new(label)
6007 }
6008
6009 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
6010 match parse_route_poll(frame) {
6011 ClientControlResponse::RoutePoll {
6012 status: None,
6013 live: Some(live),
6014 ..
6015 } => assert_eq!(live, expected_live),
6016 other => panic!("unexpected route.poll response: {other:?}"),
6017 }
6018 }
6019
6020 fn bind_liveness_route(
6021 registry: &Registry,
6022 forwarding: &ForwardingTable,
6023 module_id: &str,
6024 ) -> (RouteCtx, u16, u32) {
6025 let module_connection = ConnectionId::new(101);
6026 let client_connection = ConnectionId::new(202);
6027 let registration = registry
6028 .register_with_control_ops(
6029 manifest(module_id, PROTOCOL_VERSION),
6030 PROTOCOL_VERSION,
6031 module_connection,
6032 module_baseline_control_ops(),
6033 )
6034 .unwrap();
6035 let (module_tx, _module_rx) = mpsc::channel(8);
6036 let endpoint = forwarding
6037 .register_module_connection(
6038 module_connection,
6039 module_id.to_string(),
6040 PROTOCOL_VERSION,
6041 manifest_concurrency(®istration.manifest),
6042 FrameSink::new(module_tx),
6043 )
6044 .unwrap();
6045 let (client_ctx, _client_rx) = route_ctx(client_connection);
6046 let pending = forwarding
6047 .begin_route_bind_relay_for_test(
6048 client_connection,
6049 client_ctx.egress.clone(),
6050 1,
6051 module_id,
6052 )
6053 .unwrap();
6054 assert_eq!(pending.endpoint, endpoint);
6055 let route_channel = pending.client_channel;
6056 let route_epoch = pending.client_epoch;
6057 forwarding
6058 .complete_pending_relay(
6059 module_connection,
6060 pending.corr,
6061 RouteBindRelayOutcome::Accepted,
6062 )
6063 .unwrap();
6064 (client_ctx, route_channel, route_epoch)
6065 }
6066
6067 struct FakeProcessLiveness {
6068 live: Option<bool>,
6069 }
6070
6071 impl ModuleProcessLiveness for FakeProcessLiveness {
6072 fn process_live(&self, _module_id: &str) -> Option<bool> {
6073 self.live
6074 }
6075 }
6076
6077 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6078 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
6079 {
6080 let registry = Arc::new(Registry::default());
6081 let supervisor_handle = SupervisorHandle::new();
6082 let supervisor = Supervisor::new(
6083 Arc::clone(®istry),
6084 RestartPolicy::new(1, Duration::from_millis(10)),
6085 )
6086 .with_handle(supervisor_handle.clone());
6087 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
6088 let module = supervisor
6089 .spawn(ModuleSpec {
6090 module_id: "stderr-tail-wire".to_string(),
6091 program: fake_aft_stub_path(),
6092 args: Vec::new(),
6093 env: vec![
6094 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
6095 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
6096 ],
6097 reserved: false,
6098 reserved_prefixes: Vec::new(),
6099 protocol: ModuleProtocol::Subc,
6100 overlap: Default::default(),
6101 })
6102 .unwrap();
6103
6104 let deadline = Instant::now() + Duration::from_secs(5);
6105 loop {
6106 let tail = module.stderr_tail(None, None);
6107 if tail
6108 .entries
6109 .iter()
6110 .any(|entry| matches!(entry, TailEntry::ProcessStart))
6111 && tail.entries.iter().any(|entry| {
6112 matches!(
6113 entry,
6114 TailEntry::Line {
6115 truncated: true,
6116 ..
6117 }
6118 )
6119 })
6120 {
6121 break;
6122 }
6123 assert!(
6124 Instant::now() < deadline,
6125 "module did not produce a truncated line and restart boundary: {tail:?}"
6126 );
6127 sleep(Duration::from_millis(10)).await;
6128 }
6129
6130 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
6131 let request = ClientControlRequest::SupervisorStderrTail {
6132 module_id: "stderr-tail-wire".to_string(),
6133 max_lines: None,
6134 max_bytes: None,
6135 };
6136 let frame = Frame::build(
6137 FrameType::Request,
6138 control_flags(),
6139 0,
6140 0,
6141 1,
6142 serde_json::to_vec(&request).unwrap(),
6143 )
6144 .unwrap();
6145 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6146 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
6147 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
6148 serde_json::from_slice(&responses[0].body).unwrap()
6149 else {
6150 panic!("expected supervisor.stderr_tail response");
6151 };
6152
6153 assert!(
6154 tail.entries
6155 .iter()
6156 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
6157 "the control response lost the restart boundary"
6158 );
6159 let Some(StderrTailEntry::Line { text, truncated }) = tail.entries.iter().find(|entry| {
6160 matches!(
6161 entry,
6162 StderrTailEntry::Line {
6163 truncated: true,
6164 ..
6165 }
6166 )
6167 }) else {
6168 panic!("the control response lost the truncated line");
6169 };
6170 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
6171 assert!(*truncated);
6172 }
6173
6174 #[tokio::test(flavor = "current_thread")]
6179 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
6180 let dir = TestTempDir::new("terminals-off-worker");
6181 let journal_path = dir.join("terminals.jsonl");
6182 let registry = Arc::new(Registry::default());
6183 let supervisor_handle = SupervisorHandle::new();
6184 let supervisor =
6185 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
6186 .with_handle(supervisor_handle.clone())
6187 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
6188 let module = supervisor
6189 .spawn(ModuleSpec {
6190 module_id: "terminal-off-worker".to_string(),
6191 program: fake_aft_stub_path(),
6192 args: Vec::new(),
6193 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
6194 reserved: false,
6195 reserved_prefixes: Vec::new(),
6196 protocol: ModuleProtocol::Subc,
6197 overlap: Default::default(),
6198 })
6199 .unwrap();
6200 let deadline = Instant::now() + Duration::from_secs(5);
6201 while module.terminal_history().entries.len() != 2 {
6202 assert!(Instant::now() < deadline, "module did not record two exits");
6203 sleep(Duration::from_millis(10)).await;
6204 }
6205
6206 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
6207 let handler =
6208 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
6209 let frame = Frame::build(
6210 FrameType::Request,
6211 control_flags(),
6212 0,
6213 0,
6214 1,
6215 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
6216 module_id: "terminal-off-worker".to_string(),
6217 })
6218 .unwrap(),
6219 )
6220 .unwrap();
6221 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6222 let spawned_at = std::time::Instant::now();
6223 let read = tokio::spawn({
6224 let handler = Arc::clone(&handler);
6225 async move { handler.handle_control_frame(&ctx, frame).await }
6226 });
6227 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
6230 .await
6231 .unwrap()
6232 .expect("the history read reached its pause");
6233 let elapsed = spawned_at.elapsed();
6234 assert!(
6235 elapsed < Duration::from_secs(2) && !read.is_finished(),
6236 "this task could not run while the history read was paused \
6237 (resumed after {elapsed:?}, read finished: {})",
6238 read.is_finished()
6239 );
6240
6241 drop(release);
6242 let responses = read.await.unwrap().unwrap();
6243 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
6244 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
6245 panic!("expected supervisor.terminals response");
6246 };
6247 assert_eq!(terminals.entries.len(), 2);
6248 assert_eq!(terminals.journal_skipped_lines, 0);
6249 assert_eq!(terminals.journal_read_errors, 0);
6250 }
6251
6252 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6253 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
6254 let registry = Arc::new(Registry::default());
6255 let supervisor_handle = SupervisorHandle::new();
6256 let supervisor =
6257 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
6258 .with_handle(supervisor_handle.clone());
6259 let module = supervisor
6260 .spawn(ModuleSpec {
6261 module_id: "terminal-golden".to_string(),
6262 program: fake_aft_stub_path(),
6263 args: Vec::new(),
6264 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
6265 reserved: false,
6266 reserved_prefixes: Vec::new(),
6267 protocol: ModuleProtocol::Subc,
6268 overlap: Default::default(),
6269 })
6270 .unwrap();
6271
6272 let deadline = Instant::now() + Duration::from_secs(5);
6273 while module.terminal_history().entries.len() != 2 {
6274 assert!(
6275 Instant::now() < deadline,
6276 "module did not retain two terminal exits: {:?}",
6277 module.terminal_history()
6278 );
6279 sleep(Duration::from_millis(10)).await;
6280 }
6281
6282 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
6283 let request = ClientControlRequest::SupervisorTerminals {
6284 module_id: "terminal-golden".to_string(),
6285 };
6286 let frame = Frame::build(
6287 FrameType::Request,
6288 control_flags(),
6289 0,
6290 0,
6291 1,
6292 serde_json::to_vec(&request).unwrap(),
6293 )
6294 .unwrap();
6295 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6296 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
6297 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
6298 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
6299 panic!("expected supervisor.terminals response");
6300 };
6301 assert_eq!(terminals.entries.len(), 2);
6302 assert_eq!(terminals.dropped, 0);
6303
6304 let mut rendered = serde_json::to_value(response).unwrap();
6305 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
6308 for (index, entry) in rendered["entries"]
6309 .as_array_mut()
6310 .expect("terminal response entries array")
6311 .iter_mut()
6312 .enumerate()
6313 {
6314 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
6315 }
6316
6317 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
6318 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
6319 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
6320 if std::env::var_os("UPDATE_GOLDEN").is_some() {
6321 std::fs::write(&golden_path, &serialized).unwrap();
6322 }
6323 let expected: Value =
6324 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
6325 assert_eq!(rendered, expected);
6326 }
6327
6328 #[test]
6329 fn hello_registers_manifest_and_returns_ack() {
6330 let registry = Arc::new(Registry::default());
6331 let handler = ControlHandler::new(Arc::clone(®istry));
6332 let conn = ConnectionId::new(1);
6333
6334 let responses = handler
6335 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
6336 .unwrap();
6337
6338 assert_eq!(responses.len(), 1);
6339 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
6340 assert_eq!(responses[0].header.channel, 0);
6341 assert_eq!(responses[0].header.corr, 7);
6342 let ack = parse_ack(&responses[0]);
6343 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
6344 assert!(ack
6345 .subc_capabilities
6346 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
6347 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
6348 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
6349 assert!(ack
6350 .subc_ops
6351 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
6352 assert!(ack
6353 .subc_ops
6354 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
6355
6356 let registration = registry.get_module("aft").unwrap().unwrap();
6357 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
6358 assert_eq!(registration.state, ChannelState::Active);
6359 assert_eq!(registration.connection_id, conn);
6360 assert_eq!(registration.control_ops, module_baseline_control_ops());
6361 }
6362
6363 #[test]
6364 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
6365 let invalid_identifiers = [
6366 ("case_change", "credentials-Provider/v1"),
6367 ("leading_zero", "credentials-provider/v01"),
6368 ("trailing_hyphen", "credentials-provider-/v1"),
6369 ("consecutive_hyphens", "credentials--provider/v1"),
6370 ("uppercase", "Credentials-provider/v1"),
6371 ("missing_v", "credentials-provider/1"),
6372 ("whitespace", "credentials provider/v1"),
6373 ("zero_version", "credentials-provider/v0"),
6374 ("out_of_range_version", "credentials-provider/v4294967296"),
6375 (
6376 "overlength_name",
6377 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
6378 ),
6379 ];
6380 let mut cases = invalid_identifiers
6381 .into_iter()
6382 .map(|(name, identifier)| {
6383 (
6384 format!("identifier_{name}"),
6385 "capabilities.provides[0]".to_string(),
6386 identifier.to_string(),
6387 json!({ "provides": [identifier] }),
6388 None,
6389 )
6390 })
6391 .collect::<Vec<_>>();
6392 cases.extend([
6393 (
6394 "unknown_need".to_string(),
6395 "capabilities.requires[0].need".to_string(),
6396 "deferred".to_string(),
6397 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
6398 None,
6399 ),
6400 (
6401 "duplicate_provides".to_string(),
6402 "capabilities.provides[1]".to_string(),
6403 "credentials-provider/v1".to_string(),
6404 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
6405 None,
6406 ),
6407 (
6408 "duplicate_must_never_reach".to_string(),
6409 "capabilities.must_never_reach[1]".to_string(),
6410 "credentials-provider/v1".to_string(),
6411 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
6412 None,
6413 ),
6414 (
6415 "duplicate_requires_same_need".to_string(),
6416 "capabilities.requires[1]".to_string(),
6417 "credentials-provider/v1".to_string(),
6418 json!({ "requires": [
6419 { "capability": "credentials-provider/v1", "need": "required" },
6420 { "capability": "credentials-provider/v1", "need": "required" }
6421 ] }),
6422 None,
6423 ),
6424 (
6425 "duplicate_requires_conflicting_need".to_string(),
6426 "capabilities.requires[1]".to_string(),
6427 "credentials-provider/v1".to_string(),
6428 json!({ "requires": [
6429 { "capability": "credentials-provider/v1", "need": "required" },
6430 { "capability": "credentials-provider/v1", "need": "optional" }
6431 ] }),
6432 None,
6433 ),
6434 (
6435 "capabilities_root_pointer".to_string(),
6436 "runtime_computed[0]".to_string(),
6437 "/capabilities".to_string(),
6438 json!({}),
6439 Some(json!(["/capabilities"])),
6440 ),
6441 (
6442 "capabilities_descendant_pointer".to_string(),
6443 "runtime_computed[0]".to_string(),
6444 "/capabilities/provides".to_string(),
6445 json!({}),
6446 Some(json!(["/capabilities/provides"])),
6447 ),
6448 (
6449 "malformed_pointer_without_leading_slash".to_string(),
6450 "runtime_computed[0]".to_string(),
6451 "capabilities".to_string(),
6452 json!({}),
6453 Some(json!(["capabilities"])),
6454 ),
6455 (
6456 "malformed_pointer_escape".to_string(),
6457 "runtime_computed[0]".to_string(),
6458 "/roles/~2/tools".to_string(),
6459 json!({}),
6460 Some(json!(["/roles/~2/tools"])),
6461 ),
6462 (
6463 "unknown_capabilities_field".to_string(),
6464 "capabilities.future".to_string(),
6465 "<array>".to_string(),
6466 json!({ "future": [] }),
6467 None,
6468 ),
6469 ]);
6470
6471 for (index, (name, field, value, capabilities, runtime_computed)) in
6472 cases.into_iter().enumerate()
6473 {
6474 let registry = Arc::new(Registry::default());
6475 let handler = ControlHandler::new(Arc::clone(®istry));
6476 let response = handler
6477 .handle_control(
6478 ConnectionId::new((index + 1) as u64),
6479 capability_grammar_hello_frame(
6480 capabilities,
6481 runtime_computed,
6482 index as u64 + 1,
6483 ),
6484 )
6485 .expect("invalid HELLO returns a refusal");
6486
6487 assert_eq!(response.len(), 1, "{name} must emit one refusal");
6488 let error = parse_error(&response[0]);
6489 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
6490 let message = error["message"]
6491 .as_str()
6492 .expect("error message is a string");
6493 assert!(
6494 message.contains(&field),
6495 "{name}: field missing from {message}"
6496 );
6497 assert!(
6498 message.contains(&value),
6499 "{name}: value missing from {message}"
6500 );
6501 assert_eq!(
6502 registry
6503 .active_registration_count()
6504 .expect("registry reads"),
6505 0,
6506 "{name}: refused HELLO must not create a catalog entry"
6507 );
6508 }
6509 }
6510
6511 #[test]
6512 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
6513 let registry = Arc::new(Registry::default());
6514 let handler = ControlHandler::new(Arc::clone(®istry));
6515 let capabilities = json!({
6516 "provides": ["credentials-provider/v1"],
6517 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
6518 "must_never_reach": ["federation-transport/v1"]
6519 });
6520 let response = handler
6521 .handle_control(
6522 ConnectionId::new(99),
6523 capability_grammar_hello_frame(
6524 capabilities.clone(),
6525 Some(json!(["/roles/0/tools"])),
6526 99,
6527 ),
6528 )
6529 .expect("valid HELLO registers");
6530 assert_eq!(response[0].header.ty, FrameType::HelloAck);
6531
6532 let request = Frame::build(
6533 FrameType::Request,
6534 control_flags(),
6535 0,
6536 0,
6537 100,
6538 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
6539 .expect("catalog request serializes"),
6540 )
6541 .expect("catalog request frame builds");
6542 let response = handler
6543 .handle_catalog_list(request, None)
6544 .expect("catalog list succeeds");
6545 let ClientControlResponse::CatalogList { modules, .. } =
6546 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
6547 else {
6548 panic!("catalog request must return catalog.list");
6549 };
6550 assert_eq!(modules.len(), 1);
6551 assert_eq!(
6552 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
6553 capabilities
6554 );
6555 }
6556
6557 #[test]
6558 fn catalog_list_mirrors_management_operation_description() {
6559 let registry = Arc::new(Registry::default());
6560 let handler = ControlHandler::new(Arc::clone(®istry));
6561 let description = "List managed records and return their identifiers and metadata.";
6562 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
6563 manifest.provides = vec![ProviderRole::ManagementSurface {
6564 operations: vec![ManagementOperation {
6565 name: "records.list".to_string(),
6566 kind: ManagementOperationKind::Query,
6567 description: Some(description.to_string()),
6568 }],
6569 config_schema: json!({"type": "object"}),
6570 observability: vec![ObservabilitySurface {
6571 name: "records.stats".to_string(),
6572 kind: ObservabilityKind::Snapshot,
6573 }],
6574 identity_scope: vec![IdentityScope::Project],
6575 concurrency: Concurrency::ModuleManaged,
6576 }];
6577 registry
6578 .register_with_control_ops(
6579 manifest,
6580 PROTOCOL_VERSION,
6581 ConnectionId::new(99),
6582 Vec::new(),
6583 )
6584 .expect("described management manifest registers");
6585
6586 let request = Frame::build(
6587 FrameType::Request,
6588 control_flags(),
6589 0,
6590 0,
6591 100,
6592 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
6593 .expect("catalog request serializes"),
6594 )
6595 .expect("catalog request frame builds");
6596 let response = handler
6597 .handle_catalog_list(request, None)
6598 .expect("catalog list succeeds");
6599 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
6600 assert_eq!(
6601 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
6602 "catalog.list must preserve the declared operation description verbatim"
6603 );
6604 }
6605
6606 #[test]
6607 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
6608 let registry = Arc::new(Registry::default());
6609 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
6610 [("vault".to_string(), true), ("squatter".to_string(), true)],
6611 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
6612 );
6613 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
6614 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
6615 provides: vec!["credentials-provider/v1".to_string()],
6616 requires: Vec::new(),
6617 must_never_reach: Vec::new(),
6618 });
6619 let frame = Frame::build(
6620 FrameType::Hello,
6621 control_flags(),
6622 0,
6623 0,
6624 77,
6625 serde_json::to_vec(&ModuleHelloBody {
6626 manifest: squatter,
6627 protocol_ver: PROTOCOL_VERSION,
6628 control_ops: None,
6629 launch_nonce: None,
6630 })
6631 .expect("HELLO serializes"),
6632 )
6633 .expect("HELLO frame builds");
6634 let response = handler
6635 .handle_control(ConnectionId::new(77), frame)
6636 .expect("reserved claim receives a typed refusal");
6637 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
6638 assert_eq!(
6639 registry
6640 .active_registration_count()
6641 .expect("registry reads"),
6642 0,
6643 "a reserved capability refusal must not leave a catalog entry"
6644 );
6645 }
6646
6647 #[test]
6648 fn server_describe_surfaces_required_capability_verdict_fields() {
6649 let registry = Arc::new(Registry::default());
6650 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
6651 [
6652 ("consumer".to_string(), true),
6653 ("provider".to_string(), false),
6654 ],
6655 BTreeMap::new(),
6656 );
6657 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
6658 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
6659 provides: Vec::new(),
6660 requires: vec![subc_protocol::manifest::CapabilityRequirement {
6661 capability: "credentials-provider/v1".to_string(),
6662 need: subc_protocol::manifest::CapabilityNeed::Required,
6663 }],
6664 must_never_reach: Vec::new(),
6665 });
6666 let hello = Frame::build(
6667 FrameType::Hello,
6668 control_flags(),
6669 0,
6670 0,
6671 78,
6672 serde_json::to_vec(&ModuleHelloBody {
6673 manifest: consumer,
6674 protocol_ver: PROTOCOL_VERSION,
6675 control_ops: None,
6676 launch_nonce: None,
6677 })
6678 .expect("HELLO serializes"),
6679 )
6680 .expect("HELLO frame builds");
6681 handler
6682 .handle_control(ConnectionId::new(78), hello)
6683 .expect("consumer registers");
6684 let describe = Frame::build(
6685 FrameType::Request,
6686 control_flags(),
6687 0,
6688 0,
6689 79,
6690 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
6691 .expect("request serializes"),
6692 )
6693 .expect("describe frame builds");
6694 let response = handler
6695 .handle_server_describe(describe)
6696 .expect("server.describe succeeds");
6697 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
6698 let requirement = &rendered["capability_requirements"][0];
6699 assert_eq!(requirement["consumer"], "consumer");
6700 assert_eq!(requirement["verdict"], "never_provided");
6701 assert_eq!(requirement["episode_seq"], 1);
6702 assert_eq!(requirement["config_satisfiable"], false);
6703 assert_eq!(requirement["runtime_available"], false);
6704 assert!(requirement["detail"]
6705 .as_str()
6706 .expect("detail string")
6707 .contains("credentials-provider/v1"));
6708 }
6709
6710 #[test]
6711 fn catalog_list_omits_capabilities_for_legacy_manifest() {
6712 let registry = Arc::new(Registry::default());
6713 let handler = ControlHandler::new(Arc::clone(®istry));
6714 let hello = handler
6715 .handle_control(
6716 ConnectionId::new(101),
6717 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
6718 )
6719 .expect("legacy HELLO registers");
6720 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
6721
6722 let request = Frame::build(
6723 FrameType::Request,
6724 control_flags(),
6725 0,
6726 0,
6727 102,
6728 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
6729 .expect("catalog request serializes"),
6730 )
6731 .expect("catalog request frame builds");
6732 let response = handler
6733 .handle_catalog_list(request, None)
6734 .expect("catalog list succeeds");
6735 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
6736 assert!(
6737 body["modules"][0].get("capabilities").is_none(),
6738 "legacy manifest must retain an absent capabilities field on catalog.list"
6739 );
6740 }
6741
6742 #[test]
6743 fn hello_ack_omits_storage_when_no_storage_config() {
6744 let registry = Arc::new(Registry::default());
6745 let handler = ControlHandler::new(Arc::clone(®istry));
6746 let responses = handler
6747 .handle_control(
6748 ConnectionId::new(1),
6749 hello_frame("aft", PROTOCOL_VERSION, 7),
6750 )
6751 .unwrap();
6752 let ack = parse_ack(&responses[0]);
6753 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
6754 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
6755 }
6756
6757 #[tokio::test]
6758 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
6759 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
6760 let registry = Arc::new(Registry::default());
6761 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
6762 let responses = handler
6763 .handle_control(
6764 ConnectionId::new(1),
6765 hello_frame("aft", PROTOCOL_VERSION, 7),
6766 )
6767 .unwrap();
6768 let ack = parse_ack(&responses[0]);
6769 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
6770
6771 let described = handler
6772 .handle_control_frame(
6773 &route_ctx(ConnectionId::new(2)).0,
6774 Frame::build(
6775 FrameType::Request,
6776 control_flags(),
6777 0,
6778 0,
6779 9,
6780 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
6781 )
6782 .unwrap(),
6783 )
6784 .await
6785 .unwrap();
6786 let ClientControlResponse::ServerDescribe { machine_id, .. } =
6787 serde_json::from_slice(&described[0].body).unwrap()
6788 else {
6789 panic!("server.describe answered with another shape");
6790 };
6791 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
6792 }
6793
6794 #[test]
6795 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
6796 let registry = Arc::new(Registry::default());
6799 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
6800 crate::daemon_config::StorageConfig::Sqlite {
6801 data_home: std::path::PathBuf::from("/data"),
6802 },
6803 ));
6804
6805 let responses = handler
6806 .handle_control(
6807 ConnectionId::new(1),
6808 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
6809 )
6810 .unwrap();
6811 let ack = parse_ack(&responses[0]);
6812 assert_eq!(
6813 ack.storage,
6814 Some(serde_json::json!({
6815 "module_id": "alfonso-routing",
6816 "storage_namespace": "default",
6817 "isolation": { "kind": "module" },
6818 "backend": {
6819 "backend": "sqlite",
6820 "path": "/data/cortexkit/alfonso-routing/store.db"
6821 }
6822 })),
6823 "the delivered descriptor is the module's own sqlite store path"
6824 );
6825 }
6826
6827 #[test]
6828 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
6829 let registry = Arc::new(Registry::default());
6830 let handler = ControlHandler::new(Arc::clone(®istry));
6831 let conn = ConnectionId::new(1);
6832 let responses = handler
6833 .handle_control(
6834 conn,
6835 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
6836 )
6837 .unwrap();
6838 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
6839 let registration = registry.get_module("aft").unwrap().unwrap();
6840 assert_eq!(registration.control_ops, module_baseline_control_ops());
6841
6842 let frame =
6843 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
6844 assert!(handler
6845 .guard_module_control_op(&frame, "aft", "route.bind")
6846 .unwrap()
6847 .is_none());
6848 let error = handler
6849 .guard_module_control_op(&frame, "aft", "test.synthetic")
6850 .unwrap()
6851 .expect("synthetic ungranted op should be rejected");
6852 assert_eq!(error.header.ty, FrameType::Error);
6853 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
6854 }
6855
6856 #[test]
6857 fn hello_control_ops_some_adds_optional_grants() {
6858 let registry = Arc::new(Registry::default());
6859 let handler = ControlHandler::new(Arc::clone(®istry));
6860 handler
6861 .handle_control(
6862 ConnectionId::new(1),
6863 hello_frame_with_control_ops(
6864 "aft",
6865 PROTOCOL_VERSION,
6866 7,
6867 Some(vec![
6868 "future.synthetic".to_string(),
6869 "route.bind".to_string(),
6870 ]),
6871 ),
6872 )
6873 .unwrap();
6874 let registration = registry.get_module("aft").unwrap().unwrap();
6875 assert_eq!(
6876 registration.control_ops,
6877 vec![
6878 "route.bind".to_string(),
6879 "route.status".to_string(),
6880 "future.synthetic".to_string(),
6881 ]
6882 );
6883 let frame =
6884 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
6885 assert!(handler
6886 .guard_module_control_op(&frame, "aft", "future.synthetic")
6887 .unwrap()
6888 .is_none());
6889 }
6890
6891 #[tokio::test]
6892 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
6893 let registry = Arc::new(Registry::default());
6894 let forwarding = Arc::new(ForwardingTable::default());
6895 let handler =
6896 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
6897 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
6898 hello_via_sink(
6899 &handler,
6900 &module_ctx,
6901 &mut module_rx,
6902 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
6903 )
6904 .await;
6905
6906 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
6907 let responses = handler
6908 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
6909 .await
6910 .unwrap();
6911 assert_eq!(responses.len(), 1);
6912 assert_eq!(responses[0].header.ty, FrameType::Error);
6913 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
6914 assert!(module_rx.try_recv().is_err());
6915 }
6916
6917 #[tokio::test]
6918 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
6919 let registry = Arc::new(Registry::default());
6920 let forwarding = Arc::new(ForwardingTable::default());
6921 let handler =
6922 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
6923 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
6924 hello_via_sink(
6925 &handler,
6926 &module_ctx,
6927 &mut module_rx,
6928 hello_frame_with_control_ops(
6929 "aft",
6930 PROTOCOL_VERSION,
6931 7,
6932 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
6933 ),
6934 )
6935 .await;
6936
6937 let project_root = unique_project_root("demux");
6938 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
6939 let route_handler = handler.clone();
6940 let route_task = tokio::spawn(async move {
6941 route_handler
6942 .handle_control_frame(
6943 &route_client_ctx,
6944 route_open_frame(100, "aft", project_root),
6945 )
6946 .await
6947 .unwrap()
6948 });
6949 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
6950 .await
6951 .unwrap()
6952 .unwrap();
6953 assert!(matches!(
6954 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
6955 ModuleControlRequest::RouteBind { .. }
6956 ));
6957
6958 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
6959 let health_handler = handler.clone();
6960 let health_task = tokio::spawn(async move {
6961 health_handler
6962 .handle_control_frame(
6963 &health_client_ctx,
6964 supervisor_health_probe_frame(101, "aft"),
6965 )
6966 .await
6967 .unwrap()
6968 });
6969 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
6970 .await
6971 .unwrap()
6972 .unwrap();
6973 assert_eq!(
6974 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
6975 ModuleControlRequest::HealthCheck {}
6976 );
6977
6978 handler
6979 .handle_control_frame(
6980 &module_ctx,
6981 health_response(health_frame.header.corr, HealthStatus::Degraded),
6982 )
6983 .await
6984 .unwrap();
6985 let health_response = health_task.await.unwrap();
6986 assert_eq!(health_response.len(), 1);
6987 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
6988 ClientControlResponse::SupervisorHealthProbe {
6989 module_id,
6990 status,
6991 detail,
6992 metrics,
6993 } => {
6994 assert_eq!(module_id, "aft");
6995 assert_eq!(status, HealthStatus::Degraded);
6996 assert_eq!(detail.as_deref(), Some("warming"));
6997 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
6998 }
6999 other => panic!("unexpected health response: {other:?}"),
7000 }
7001
7002 handler
7003 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
7004 .await
7005 .unwrap();
7006 let route_response = route_task.await.unwrap();
7007 assert!(route_response.is_empty());
7008 let published = route_client_rx.recv().await.unwrap();
7009 assert!(matches!(
7010 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
7011 ClientControlResponse::RouteOpen { .. }
7012 ));
7013 }
7014
7015 async fn relay_route_open(
7020 handler: &ControlHandler,
7021 client_connection: ConnectionId,
7022 client_egress: &FrameSink,
7023 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
7024 corr: u64,
7025 module_id: &str,
7026 project_root_label: &str,
7027 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
7028 let ctx = RouteCtx {
7029 connection_id: client_connection,
7030 egress: client_egress.clone(),
7031 };
7032 let handler = handler.clone();
7033 let project_root = unique_project_root(project_root_label);
7034 let module_id = module_id.to_string();
7035 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
7036 let task = tokio::spawn(async move {
7037 let _guard = tracing::dispatcher::set_default(&dispatch);
7038 handler
7039 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
7040 .await
7041 .unwrap()
7042 });
7043 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
7044 .await
7045 .expect("module receives the relayed route.bind")
7046 .expect("module egress is open");
7047 (task, bind.frame)
7048 }
7049
7050 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
7051 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
7052 ModuleControlRequest::RouteBind {
7053 route_channel,
7054 epoch,
7055 ..
7056 } => (route_channel, epoch),
7057 other => panic!("expected a route.bind request, got {other:?}"),
7058 }
7059 }
7060
7061 fn published_route(frame: &Frame) -> (u16, u32) {
7062 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
7063 ClientControlResponse::RouteOpen {
7064 route_channel,
7065 route_epoch,
7066 } => (route_channel, route_epoch),
7067 other => panic!("expected a route.open response, got {other:?}"),
7068 }
7069 }
7070
7071 #[tokio::test]
7091 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
7092 let registry = Arc::new(Registry::default());
7093 let forwarding = Arc::new(ForwardingTable::default());
7094 let handler =
7095 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
7096
7097 let module_connection = ConnectionId::new(30);
7098 let (module_ctx, mut module_rx) = route_ctx(module_connection);
7099 hello_via_sink(
7100 &handler,
7101 &module_ctx,
7102 &mut module_rx,
7103 hello_frame("aft", PROTOCOL_VERSION, 7),
7104 )
7105 .await;
7106
7107 let dying_client = ConnectionId::new(31);
7108 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
7109
7110 let (first_task, first_bind) = relay_route_open(
7113 &handler,
7114 dying_client,
7115 &dying_ctx.egress,
7116 &mut module_rx,
7117 100,
7118 "aft",
7119 "closing-first",
7120 )
7121 .await;
7122 handler
7123 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
7124 .await
7125 .unwrap();
7126 assert!(first_task.await.unwrap().is_empty());
7127 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
7128
7129 let (second_task, second_bind) = relay_route_open(
7131 &handler,
7132 dying_client,
7133 &dying_ctx.egress,
7134 &mut module_rx,
7135 101,
7136 "aft",
7137 "closing-second",
7138 )
7139 .await;
7140 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
7141
7142 assert!(forwarding
7145 .escalate_client_delivery_failure(
7146 dying_client,
7147 first_channel,
7148 first_epoch,
7149 CloseReason::new(
7150 "module_to_client_delivery_failed",
7151 "client egress refused a module frame",
7152 ),
7153 crate::forwarding::UndeliveredFrame {
7154 module_id: None,
7155 sink: &dying_ctx.egress,
7156 },
7157 )
7158 .unwrap());
7159 assert!(!dying_ctx.egress.is_closed());
7160
7161 let ack = handler
7163 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
7164 .await;
7165 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
7166 if module_loop_error.is_some() {
7167 handler.cleanup_connection(module_connection).unwrap();
7171 }
7172 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7176 .await
7177 .ok()
7178 .flatten();
7179
7180 assert!(
7182 registry
7183 .get_module_by_connection(module_connection)
7184 .unwrap()
7185 .is_some(),
7186 "one client's closing connection ended the shared module connection: \
7187 {module_loop_error:?}"
7188 );
7189 let cotenant = ConnectionId::new(32);
7191 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
7192 let (cotenant_task, cotenant_bind) = relay_route_open(
7193 &handler,
7194 cotenant,
7195 &cotenant_ctx.egress,
7196 &mut module_rx,
7197 102,
7198 "aft",
7199 "closing-cotenant",
7200 )
7201 .await;
7202 handler
7203 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
7204 .await
7205 .unwrap();
7206 assert!(cotenant_task.await.unwrap().is_empty());
7207 let (cotenant_channel, cotenant_epoch) =
7208 published_route(&cotenant_rx.recv().await.unwrap());
7209 assert!(matches!(
7210 forwarding
7211 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
7212 .unwrap(),
7213 DataRoute::Client(DataRouteState::Bound(_))
7214 ));
7215
7216 let goodbye = post_ack_module_frame
7219 .expect("module receives a GOODBYE for the abandoned route channel");
7220 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
7221 assert_eq!(goodbye.header.channel, abandoned_channel);
7222 assert_eq!(goodbye.header.epoch, abandoned_epoch);
7223
7224 assert!(dying_rx.try_recv().is_err());
7228 let second_response = second_task.await.unwrap();
7229 assert_eq!(second_response.len(), 1);
7230 assert_eq!(
7231 parse_error(&second_response[0])["code"],
7232 "target_unavailable"
7233 );
7234 }
7235
7236 #[test]
7243 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
7244 let handler = ControlHandler::default();
7245 let module_connection = ConnectionId::new(30);
7246 let client_connection = ConnectionId::new(31);
7247
7248 handler
7249 .refuse_to_end_module_connection_for_a_client(
7250 module_connection,
7251 77,
7252 ForwardingError::ConnectionClosing {
7253 connection_id: client_connection,
7254 },
7255 )
7256 .expect("a closing client must never end the module connection");
7257
7258 assert!(matches!(
7259 handler.refuse_to_end_module_connection_for_a_client(
7260 module_connection,
7261 78,
7262 ForwardingError::ConnectionClosing {
7263 connection_id: module_connection,
7264 },
7265 ),
7266 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
7267 connection_id
7268 })) if connection_id == module_connection
7269 ));
7270 assert!(matches!(
7271 handler.refuse_to_end_module_connection_for_a_client(
7272 module_connection,
7273 79,
7274 ForwardingError::Poisoned,
7275 ),
7276 Err(RouterError::Forwarding(ForwardingError::Poisoned))
7277 ));
7278 assert!(matches!(
7279 handler.refuse_to_end_module_connection_for_a_client(
7280 module_connection,
7281 80,
7282 ForwardingError::StaleModuleEndpoint,
7283 ),
7284 Err(RouterError::Forwarding(
7285 ForwardingError::StaleModuleEndpoint
7286 ))
7287 ));
7288 }
7289
7290 #[tokio::test]
7303 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
7304 let registry = Arc::new(Registry::default());
7305 let forwarding = Arc::new(ForwardingTable::default());
7306 let supervisor = SupervisorHandle::new();
7307 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7308 let handler =
7309 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
7310 .with_supervisor(supervisor);
7311
7312 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
7313 hello_via_sink(
7314 &handler,
7315 &target_ctx,
7316 &mut target_rx,
7317 hello_frame("target", PROTOCOL_VERSION, 1),
7318 )
7319 .await;
7320
7321 let wrong_nonce = handler
7325 .handle_control_frame(
7326 &route_ctx(ConnectionId::new(91)).0,
7327 route_open_frame_with_admission_facts(
7328 20,
7329 "target",
7330 unique_project_root("admission-facts"),
7331 Some(subc_control::ConsumerIdentity {
7332 module_id: "fed".to_string(),
7333 launch_nonce: "not-the-real-nonce".to_string(),
7334 }),
7335 None,
7336 ),
7337 )
7338 .await
7339 .unwrap();
7340 assert_eq!(
7341 parse_error(&wrong_nonce[0])["code"],
7342 "bad_consumer_identity",
7343 "a mismatched launch nonce must be refused, not stamped Reserved"
7344 );
7345
7346 let never_spawned = handler
7350 .handle_control_frame(
7351 &route_ctx(ConnectionId::new(92)).0,
7352 route_open_frame_with_admission_facts(
7353 21,
7354 "target",
7355 unique_project_root("admission-facts"),
7356 Some(subc_control::ConsumerIdentity {
7357 module_id: "never-spawned".to_string(),
7358 launch_nonce: "any-nonce".to_string(),
7359 }),
7360 None,
7361 ),
7362 )
7363 .await
7364 .unwrap();
7365 assert_eq!(
7366 parse_error(&never_spawned[0])["code"],
7367 "bad_consumer_identity",
7368 "an unspawned module_id must be refused rather than accepted for lack of a record"
7369 );
7370 }
7371
7372 #[tokio::test]
7395 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
7396 let registry = Arc::new(Registry::default());
7397 let forwarding = Arc::new(ForwardingTable::default());
7398 let supervisor = SupervisorHandle::new();
7399 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7400 let handler =
7401 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
7402 .with_supervisor(supervisor);
7403
7404 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
7405 hello_via_sink(
7406 &handler,
7407 &target_ctx,
7408 &mut target_rx,
7409 hello_frame("target", PROTOCOL_VERSION, 1),
7410 )
7411 .await;
7412
7413 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
7414 let route_handler = handler.clone();
7415 let route_task = tokio::spawn(async move {
7416 route_handler
7417 .handle_control_frame(
7418 &client_ctx,
7419 route_open_frame_with_admission_facts(
7420 30,
7421 "target",
7422 unique_project_root("admission-facts"),
7423 Some(subc_control::ConsumerIdentity {
7424 module_id: "fed".to_string(),
7425 launch_nonce: "fed-nonce".to_string(),
7426 }),
7427 None,
7428 ),
7429 )
7430 .await
7431 .unwrap()
7432 });
7433
7434 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7441 .await
7442 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
7443 .expect("module control channel closed before route.bind");
7444 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
7445 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
7446 panic!("expected route.bind")
7447 };
7448 assert_eq!(
7449 principal,
7450 Some(Principal::Reserved {
7451 module_id: "fed".to_string()
7452 }),
7453 "a correctly attested consumer must be stamped Reserved for its own id"
7454 );
7455
7456 handler
7457 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
7458 .await
7459 .unwrap();
7460 assert!(route_task.await.unwrap().is_empty());
7461 assert!(
7462 matches!(
7463 serde_json::from_slice::<ClientControlResponse>(
7464 &client_rx.recv().await.unwrap().body
7465 )
7466 .unwrap(),
7467 ClientControlResponse::RouteOpen { .. }
7468 ),
7469 "the route must actually open, not merely avoid an error"
7470 );
7471 }
7472
7473 #[tokio::test(start_paused = true)]
7474 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
7475 let registry = Arc::new(Registry::default());
7476 let forwarding = Arc::new(ForwardingTable::default());
7477 let supervisor = SupervisorHandle::new();
7478 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7479 let handler =
7480 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
7481 .with_supervisor(supervisor);
7482
7483 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
7484 hello_via_sink(
7485 &handler,
7486 &target_ctx,
7487 &mut target_rx,
7488 hello_frame("target", PROTOCOL_VERSION, 1),
7489 )
7490 .await;
7491
7492 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
7493 let direct_handler = handler.clone();
7494 let direct_open = tokio::spawn(async move {
7495 direct_handler
7496 .handle_control_frame(
7497 &direct_ctx,
7498 route_open_frame(2, "target", unique_project_root("route-census-direct")),
7499 )
7500 .await
7501 .unwrap()
7502 });
7503 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7504 .await
7505 .expect("no direct route.bind within 5s")
7506 .expect("target control channel closed before direct route.bind");
7507 handler
7508 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
7509 .await
7510 .unwrap();
7511 assert!(direct_open.await.unwrap().is_empty());
7512 let _ = direct_rx.recv().await.unwrap();
7513
7514 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
7515 let reserved_handler = handler.clone();
7516 let reserved_open = tokio::spawn(async move {
7517 reserved_handler
7518 .handle_control_frame(
7519 &reserved_ctx,
7520 route_open_frame_with_admission_facts(
7521 3,
7522 "target",
7523 unique_project_root("admission-facts"),
7524 Some(ConsumerIdentity {
7525 module_id: "fed".to_string(),
7526 launch_nonce: "fed-nonce".to_string(),
7527 }),
7528 None,
7529 ),
7530 )
7531 .await
7532 .unwrap()
7533 });
7534 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7535 .await
7536 .expect("no reserved route.bind within 5s")
7537 .expect("target control channel closed before reserved route.bind");
7538 handler
7539 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
7540 .await
7541 .unwrap();
7542 assert!(reserved_open.await.unwrap().is_empty());
7543 let _ = reserved_rx.recv().await.unwrap();
7544
7545 forwarding
7546 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
7547 .unwrap();
7548 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
7549 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
7550 module_id: Some("target".to_string()),
7551 })
7552 .unwrap();
7553 let census_frame =
7554 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
7555 let response = handler
7556 .handle_control_frame(&census_ctx, census_frame)
7557 .await
7558 .unwrap()
7559 .pop()
7560 .unwrap();
7561 let actual: Value = serde_json::from_slice(&response.body).unwrap();
7562 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
7563 assert!(matches!(
7564 decoded,
7565 ClientControlResponse::SupervisorRoutes { .. }
7566 ));
7567 let routes = actual["modules"][0]["routes"].as_array().unwrap();
7568 assert_eq!(routes.len(), 2);
7569 assert!(routes.iter().all(|route| route["draining"] == true));
7570 assert!(
7573 routes.iter().all(|route| route["drain_reason"] == "reload"),
7574 "draining routes must name the drain's reason: {routes:?}"
7575 );
7576 assert!(routes.iter().any(|route| {
7577 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
7578 }));
7579 assert!(routes.iter().any(|route| {
7580 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
7581 }));
7582
7583 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
7584 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
7585 if std::env::var_os("UPDATE_GOLDEN").is_some() {
7586 std::fs::write(
7587 &golden_path,
7588 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
7589 )
7590 .unwrap();
7591 }
7592 let expected: Value =
7593 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
7594 assert_eq!(actual, expected);
7595 }
7596
7597 async fn query_live_roots(
7598 handler: &ControlHandler,
7599 module_ctx: &RouteCtx,
7600 ) -> ModuleControlResponseToModule {
7601 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
7602 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
7603 let response = handler
7604 .handle_control_frame(module_ctx, frame)
7605 .await
7606 .unwrap()
7607 .pop()
7608 .unwrap();
7609 serde_json::from_slice(&response.body).unwrap()
7610 }
7611
7612 #[tokio::test(start_paused = true)]
7613 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
7614 let registry = Arc::new(Registry::default());
7615 let forwarding = Arc::new(ForwardingTable::default());
7616 let handler = ControlHandler::with_forwarding(registry, forwarding);
7617 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
7618 hello_via_sink(
7619 &handler,
7620 &target_ctx,
7621 &mut target_rx,
7622 hello_frame("target", PROTOCOL_VERSION, 1),
7623 )
7624 .await;
7625 let root = unique_project_root("live-roots-known");
7626 let path = ProjectRootId::from_path_allowing_missing(root.path())
7627 .unwrap()
7628 .as_path()
7629 .to_path_buf();
7630 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
7631 let open_handler = handler.clone();
7632 let opened = tokio::spawn(async move {
7633 open_handler
7634 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
7635 .await
7636 .unwrap()
7637 });
7638 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7639 .await
7640 .unwrap()
7641 .unwrap();
7642 handler
7643 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
7644 .await
7645 .unwrap();
7646 assert!(opened.await.unwrap().is_empty());
7647 let _ = client_rx.recv().await.unwrap();
7648
7649 let root = unique_project_root("live-roots-pending");
7650 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
7651 .unwrap()
7652 .as_path()
7653 .to_path_buf();
7654 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
7655 let open_handler = handler.clone();
7656 let pending = tokio::spawn(async move {
7657 open_handler
7658 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
7659 .await
7660 .unwrap()
7661 });
7662 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7663 .await
7664 .unwrap()
7665 .unwrap();
7666 let actual = query_live_roots(&handler, &target_ctx).await;
7667 let ModuleControlResponseToModule::LiveRoots {
7668 roots,
7669 unknown_root_bindings,
7670 total_bindings,
7671 } = actual
7672 else {
7673 panic!("expected live roots")
7674 };
7675 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
7676 assert_eq!(unknown_root_bindings, 0);
7677 assert_eq!(
7678 roots.len(),
7679 2,
7680 "root-known arm must retain each canonical root"
7681 );
7682 assert_eq!(
7683 total_bindings,
7684 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
7685 );
7686 let counts = roots
7687 .iter()
7688 .map(|root| (root.project_root.clone(), root.bound, root.pending))
7689 .collect::<Vec<_>>();
7690 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
7691 expected.sort_by(|a, b| a.0.cmp(&b.0));
7692 assert_eq!(
7693 counts, expected,
7694 "roots must sort by path and count pending separately"
7695 );
7696 handler
7697 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
7698 .await
7699 .unwrap();
7700 assert!(pending.await.unwrap().is_empty());
7701 }
7702
7703 #[tokio::test(start_paused = true)]
7704 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
7705 let forwarding = Arc::new(ForwardingTable::default());
7706 let handler =
7707 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
7708 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
7709 hello_via_sink(
7710 &handler,
7711 &target_ctx,
7712 &mut target_rx,
7713 hello_frame("target", PROTOCOL_VERSION, 1),
7714 )
7715 .await;
7716 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
7717 let pending = forwarding
7718 .begin_route_bind_relay_for_test(
7719 client_ctx.connection_id,
7720 client_ctx.egress.clone(),
7721 2,
7722 "target",
7723 )
7724 .unwrap();
7725 forwarding
7726 .complete_pending_relay(
7727 target_ctx.connection_id,
7728 pending.corr,
7729 RouteBindRelayOutcome::Accepted,
7730 )
7731 .unwrap();
7732 let actual = query_live_roots(&handler, &target_ctx).await;
7733 let ModuleControlResponseToModule::LiveRoots {
7734 roots,
7735 unknown_root_bindings,
7736 total_bindings,
7737 } = actual
7738 else {
7739 panic!("expected live roots")
7740 };
7741 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
7742 assert_eq!(
7743 unknown_root_bindings, 1,
7744 "unknown-root arm must not read as no bindings"
7745 );
7746 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
7747 assert_eq!(
7748 total_bindings,
7749 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
7750 );
7751 }
7752
7753 #[tokio::test(start_paused = true)]
7761 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
7762 let forwarding = Arc::new(ForwardingTable::default());
7763 let handler =
7764 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
7765 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
7766 let replies = handler
7767 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
7768 .await
7769 .unwrap();
7770 let queued_by_hello = module_rx.len();
7771
7772 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
7773 let open_handler = handler.clone();
7774 let open = tokio::spawn(async move {
7775 open_handler
7776 .handle_control_frame(
7777 &client_ctx,
7778 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
7779 )
7780 .await
7781 .unwrap()
7782 });
7783 let mut spins = 0;
7785 while module_rx.len() == queued_by_hello {
7786 spins += 1;
7787 assert!(spins < 10_000, "route.open never queued a route.bind");
7788 tokio::task::yield_now().await;
7789 }
7790
7791 for reply in replies {
7793 module_ctx.egress.send(reply).await.unwrap();
7794 }
7795
7796 let first = module_rx.recv().await.unwrap().frame;
7797 assert_eq!(
7798 first.header.ty,
7799 FrameType::HelloAck,
7800 "the first frame a registering module reads must be its HELLO_ACK"
7801 );
7802 assert_eq!(first.header.corr, 7);
7803 let second = module_rx.recv().await.unwrap().frame;
7804 assert_eq!(second.header.ty, FrameType::Request);
7805 assert!(
7806 matches!(
7807 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
7808 ModuleControlRequest::RouteBind { .. }
7809 ),
7810 "the route.bind follows the ack"
7811 );
7812 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
7813
7814 handler
7815 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
7816 .await
7817 .unwrap();
7818 assert!(open.await.unwrap().is_empty());
7819 let _ = client_rx.recv().await.unwrap();
7820 }
7821
7822 #[tokio::test(start_paused = true)]
7823 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
7824 let handler = ControlHandler::with_forwarding(
7825 Arc::new(Registry::default()),
7826 Arc::new(ForwardingTable::default()),
7827 );
7828 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
7829 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
7830 hello_via_sink(
7831 &handler,
7832 &first_ctx,
7833 &mut first_rx,
7834 hello_frame("first", PROTOCOL_VERSION, 1),
7835 )
7836 .await;
7837 hello_via_sink(
7838 &handler,
7839 &second_ctx,
7840 &mut second_rx,
7841 hello_frame("second", PROTOCOL_VERSION, 2),
7842 )
7843 .await;
7844 let root = unique_project_root("second-only");
7845 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
7846 let cloned = handler.clone();
7847 let open = tokio::spawn(async move {
7848 cloned
7849 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
7850 .await
7851 .unwrap()
7852 });
7853 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
7854 .await
7855 .unwrap()
7856 .unwrap();
7857 let first = query_live_roots(&handler, &first_ctx).await;
7858 let second = query_live_roots(&handler, &second_ctx).await;
7859 assert!(
7860 matches!(
7861 first,
7862 ModuleControlResponseToModule::LiveRoots {
7863 total_bindings: 0,
7864 ..
7865 }
7866 ),
7867 "cross-module scope must not expose another module's roots"
7868 );
7869 assert!(
7870 matches!(
7871 second,
7872 ModuleControlResponseToModule::LiveRoots {
7873 total_bindings: 1,
7874 ..
7875 }
7876 ),
7877 "second module must see its pending route"
7878 );
7879 handler
7880 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
7881 .await
7882 .unwrap();
7883 assert!(open.await.unwrap().is_empty());
7884 }
7885
7886 #[tokio::test(start_paused = true)]
7887 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
7888 let handler = ControlHandler::with_forwarding(
7889 Arc::new(Registry::default()),
7890 Arc::new(ForwardingTable::default()),
7891 );
7892 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
7893 hello_via_sink(
7894 &handler,
7895 &target_ctx,
7896 &mut target_rx,
7897 hello_frame("target", PROTOCOL_VERSION, 1),
7898 )
7899 .await;
7900 let actual = query_live_roots(&handler, &target_ctx).await;
7901 let ModuleControlResponseToModule::LiveRoots {
7902 roots,
7903 unknown_root_bindings,
7904 total_bindings,
7905 } = actual
7906 else {
7907 panic!("expected live roots")
7908 };
7909 assert!(roots.is_empty());
7910 assert_eq!(unknown_root_bindings, 0);
7911 assert_eq!(total_bindings, 0);
7912 assert_eq!(
7913 total_bindings,
7914 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
7915 );
7916 }
7917
7918 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
7925 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
7926 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
7927 let text = std::fs::read_to_string(&path)
7928 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
7929 let vectors: Vec<(String, Value)> = text
7930 .lines()
7931 .filter(|line| !line.trim().is_empty())
7932 .map(|line| {
7933 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
7934 let id = entry["corpus_id"]
7935 .as_str()
7936 .expect("every vector carries a corpus_id")
7937 .to_string();
7938 (id, entry["package"].clone())
7939 })
7940 .collect();
7941 assert_eq!(
7945 vectors.len(),
7946 3,
7947 "vendored fed corpus changed size; re-sync from subc-federation"
7948 );
7949
7950 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
7965 let richest = vectors
7966 .iter()
7967 .filter_map(|(_, package)| package.as_object())
7968 .map(|object| {
7969 object
7970 .keys()
7971 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
7972 .count()
7973 })
7974 .max()
7975 .unwrap_or(0);
7976 assert!(
7977 richest >= 2,
7978 "vendored corpus no longer carries a package with unmodelled fields, \
7979 so the relay test can no longer distinguish a verbatim relay from a lossy one"
7980 );
7981
7982 vectors
7983 }
7984
7985 #[tokio::test]
7993 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
7994 for (corpus_id, package) in fed_admission_facts_vectors() {
7995 let registry = Arc::new(Registry::default());
7996 let forwarding = Arc::new(ForwardingTable::default());
7997 let supervisor = SupervisorHandle::new();
7998 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7999 let handler =
8000 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8001 .with_supervisor(supervisor)
8002 .with_admission_facts_config(
8003 Some("fed".to_string()),
8004 Some(vec!["target".to_string()]),
8005 );
8006
8007 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
8008 hello_via_sink(
8009 &handler,
8010 &target_ctx,
8011 &mut target_rx,
8012 hello_frame("target", PROTOCOL_VERSION, 1),
8013 )
8014 .await;
8015
8016 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
8017 let route_handler = handler.clone();
8018 let expected = package.clone();
8019 let route_task = tokio::spawn(async move {
8020 route_handler
8021 .handle_control_frame(
8022 &client_ctx,
8023 route_open_frame_with_admission_facts(
8024 20,
8025 "target",
8026 unique_project_root("admission-facts"),
8027 Some(subc_control::ConsumerIdentity {
8028 module_id: "fed".to_string(),
8029 launch_nonce: "fed-nonce".to_string(),
8030 }),
8031 Some(package),
8032 ),
8033 )
8034 .await
8035 .unwrap()
8036 });
8037
8038 let bind_frame = target_rx.recv().await.unwrap();
8039 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8040 let ModuleControlRequest::RouteBind {
8041 admission_facts, ..
8042 } = bind
8043 else {
8044 panic!("{corpus_id}: expected route.bind")
8045 };
8046 assert_eq!(
8047 admission_facts,
8048 Some(expected),
8049 "{corpus_id}: relay must not add, drop or reshape any field"
8050 );
8051
8052 handler
8053 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8054 .await
8055 .unwrap();
8056 route_task.await.unwrap();
8057 }
8058 }
8059
8060 #[tokio::test]
8061 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
8062 let registry = Arc::new(Registry::default());
8063 let forwarding = Arc::new(ForwardingTable::default());
8064 let supervisor = SupervisorHandle::new();
8065 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8066 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
8067 let handler =
8068 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8069 .with_supervisor(supervisor)
8070 .with_admission_facts_config(
8071 Some("fed".to_string()),
8072 Some(vec!["target".to_string()]),
8073 );
8074
8075 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
8076 hello_via_sink(
8077 &handler,
8078 &target_ctx,
8079 &mut target_rx,
8080 hello_frame("target", PROTOCOL_VERSION, 1),
8081 )
8082 .await;
8083 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
8084 hello_via_sink(
8085 &handler,
8086 &other_ctx,
8087 &mut other_rx,
8088 hello_frame("other", PROTOCOL_VERSION, 2),
8089 )
8090 .await;
8091
8092 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
8093 let expected_facts = facts.clone();
8094 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
8095 let route_handler = handler.clone();
8096 let route_task = tokio::spawn(async move {
8097 route_handler
8098 .handle_control_frame(
8099 &client_ctx,
8100 route_open_frame_with_admission_facts(
8101 10,
8102 "target",
8103 unique_project_root("admission-facts"),
8104 Some(subc_control::ConsumerIdentity {
8105 module_id: "fed".to_string(),
8106 launch_nonce: "fed-nonce".to_string(),
8107 }),
8108 Some(facts.clone()),
8109 ),
8110 )
8111 .await
8112 .unwrap()
8113 });
8114 let bind_frame = target_rx.recv().await.unwrap();
8115 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8116 let ModuleControlRequest::RouteBind {
8117 admission_facts, ..
8118 } = bind
8119 else {
8120 panic!("expected route.bind")
8121 };
8122 assert_eq!(admission_facts, Some(expected_facts));
8123 handler
8124 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8125 .await
8126 .unwrap();
8127 assert!(route_task.await.unwrap().is_empty());
8128 assert!(matches!(
8129 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
8130 .unwrap(),
8131 ClientControlResponse::RouteOpen { .. }
8132 ));
8133
8134 let direct = handler
8135 .handle_control_frame(
8136 &route_ctx(ConnectionId::new(73)).0,
8137 route_open_frame_with_admission_facts(
8138 11,
8139 "target",
8140 unique_project_root("admission-facts"),
8141 None,
8142 Some(json!({"x": 1})),
8143 ),
8144 )
8145 .await
8146 .unwrap();
8147 assert_eq!(
8148 parse_error(&direct[0])["code"],
8149 "admission_facts_not_permitted"
8150 );
8151
8152 let different_reserved = handler
8153 .handle_control_frame(
8154 &route_ctx(ConnectionId::new(77)).0,
8155 route_open_frame_with_admission_facts(
8156 15,
8157 "target",
8158 unique_project_root("admission-facts"),
8159 Some(subc_control::ConsumerIdentity {
8160 module_id: "other".to_string(),
8161 launch_nonce: "other-nonce".to_string(),
8162 }),
8163 Some(json!({"x": 1})),
8164 ),
8165 )
8166 .await
8167 .unwrap();
8168 assert_eq!(
8169 parse_error(&different_reserved[0])["code"],
8170 "admission_facts_not_permitted"
8171 );
8172
8173 let other_target = handler
8174 .handle_control_frame(
8175 &route_ctx(ConnectionId::new(74)).0,
8176 route_open_frame_with_admission_facts(
8177 12,
8178 "other",
8179 unique_project_root("admission-facts"),
8180 Some(subc_control::ConsumerIdentity {
8181 module_id: "fed".to_string(),
8182 launch_nonce: "fed-nonce".to_string(),
8183 }),
8184 Some(json!({"x": 1})),
8185 ),
8186 )
8187 .await
8188 .unwrap();
8189 assert_eq!(
8190 parse_error(&other_target[0])["code"],
8191 "admission_facts_target_not_allowed"
8192 );
8193
8194 let nonexistent = handler
8195 .handle_control_frame(
8196 &route_ctx(ConnectionId::new(75)).0,
8197 route_open_frame_with_admission_facts(
8198 13,
8199 "missing",
8200 unique_project_root("admission-facts"),
8201 None,
8202 Some(json!({"x": 1})),
8203 ),
8204 )
8205 .await
8206 .unwrap();
8207 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
8208
8209 let described = handler
8210 .handle_control_frame(
8211 &route_ctx(ConnectionId::new(76)).0,
8212 Frame::build(
8213 FrameType::Request,
8214 control_flags(),
8215 0,
8216 0,
8217 14,
8218 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
8219 )
8220 .unwrap(),
8221 )
8222 .await
8223 .unwrap();
8224 let ClientControlResponse::ServerDescribe { capabilities, .. } =
8225 serde_json::from_slice(&described[0].body).unwrap()
8226 else {
8227 panic!("expected server.describe response")
8228 };
8229 assert!(capabilities
8230 .iter()
8231 .any(|cap| cap == "admission_facts_relay_v1"));
8232 }
8233
8234 #[tokio::test]
8235 async fn admission_facts_without_configured_carrier_are_rejected() {
8236 let registry = Arc::new(Registry::default());
8237 let forwarding = Arc::new(ForwardingTable::default());
8238 let handler = ControlHandler::with_forwarding(registry, forwarding);
8239 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
8240 hello_via_sink(
8241 &handler,
8242 &target_ctx,
8243 &mut target_rx,
8244 hello_frame("target", PROTOCOL_VERSION, 1),
8245 )
8246 .await;
8247
8248 let responses = handler
8249 .handle_control_frame(
8250 &route_ctx(ConnectionId::new(79)).0,
8251 route_open_frame_with_admission_facts(
8252 16,
8253 "target",
8254 unique_project_root("admission-facts"),
8255 None,
8256 Some(json!({"x": 1})),
8257 ),
8258 )
8259 .await
8260 .unwrap();
8261 assert_eq!(
8262 parse_error(&responses[0])["code"],
8263 "admission_facts_not_permitted"
8264 );
8265 }
8266
8267 #[tokio::test]
8268 async fn route_open_relays_consumer_capabilities_verbatim() {
8269 let registry = Arc::new(Registry::default());
8270 let forwarding = Arc::new(ForwardingTable::default());
8271 let handler =
8272 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8273 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
8274 hello_via_sink(
8275 &handler,
8276 &module_ctx,
8277 &mut module_rx,
8278 hello_frame("aft", PROTOCOL_VERSION, 7),
8279 )
8280 .await;
8281
8282 let expected = vec!["elicitation".to_string(), "roots".to_string()];
8283 let expected_for_request = expected.clone();
8284 let project_root = unique_project_root("consumer-capabilities-present");
8285 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
8286 let route_handler = handler.clone();
8287 let route_task = tokio::spawn(async move {
8288 route_handler
8289 .handle_control_frame(
8290 &client_ctx,
8291 route_open_frame_with_consumer_capabilities(
8292 401,
8293 "aft",
8294 project_root,
8295 Some(expected_for_request),
8296 ),
8297 )
8298 .await
8299 .unwrap()
8300 });
8301 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8302 .await
8303 .unwrap()
8304 .unwrap();
8305 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8306 let ModuleControlRequest::RouteBind {
8307 consumer_capabilities,
8308 ..
8309 } = bind
8310 else {
8311 panic!("expected route.bind request, got {bind:?}");
8312 };
8313 assert_eq!(consumer_capabilities, Some(expected.clone()));
8314
8315 handler
8316 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8317 .await
8318 .unwrap();
8319 let route_response = route_task.await.unwrap();
8320 assert!(route_response.is_empty());
8321 let published = client_rx.recv().await.unwrap();
8322 assert!(matches!(
8323 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8324 ClientControlResponse::RouteOpen { .. }
8325 ));
8326 }
8327
8328 #[tokio::test]
8329 async fn route_open_without_consumer_capabilities_relays_none() {
8330 let registry = Arc::new(Registry::default());
8331 let forwarding = Arc::new(ForwardingTable::default());
8332 let handler =
8333 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8334 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
8335 hello_via_sink(
8336 &handler,
8337 &module_ctx,
8338 &mut module_rx,
8339 hello_frame("aft", PROTOCOL_VERSION, 7),
8340 )
8341 .await;
8342
8343 let project_root = unique_project_root("consumer-capabilities-absent");
8344 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
8345 let route_handler = handler.clone();
8346 let route_task = tokio::spawn(async move {
8347 route_handler
8348 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
8349 .await
8350 .unwrap()
8351 });
8352 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8353 .await
8354 .unwrap()
8355 .unwrap();
8356 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8357 let ModuleControlRequest::RouteBind {
8358 consumer_capabilities,
8359 ..
8360 } = bind
8361 else {
8362 panic!("expected route.bind request, got {bind:?}");
8363 };
8364 assert_eq!(consumer_capabilities, None);
8365
8366 handler
8367 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8368 .await
8369 .unwrap();
8370 let route_response = route_task.await.unwrap();
8371 assert!(route_response.is_empty());
8372 let published = client_rx.recv().await.unwrap();
8373 assert!(matches!(
8374 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8375 ClientControlResponse::RouteOpen { .. }
8376 ));
8377 }
8378
8379 #[tokio::test]
8380 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
8381 let registry = Arc::new(Registry::default());
8382 let forwarding = Arc::new(ForwardingTable::default());
8383 let handler =
8384 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8385 .with_health_probe_timeout(Duration::from_secs(5));
8386 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
8387 hello_via_sink(
8388 &handler,
8389 &module_ctx,
8390 &mut module_rx,
8391 non_routable_hello_frame_with_control_ops(
8392 "mcp",
8393 300,
8394 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
8395 ),
8396 )
8397 .await;
8398 assert!(registry
8399 .get_module("mcp")
8400 .unwrap()
8401 .unwrap()
8402 .manifest
8403 .provides
8404 .is_empty());
8405
8406 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
8407 let route_response = handler
8408 .handle_control_frame(
8409 &route_client_ctx,
8410 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
8411 )
8412 .await
8413 .unwrap();
8414 assert_eq!(route_response[0].header.ty, FrameType::Error);
8415 assert_eq!(
8416 parse_error(&route_response[0])["code"],
8417 "target_unavailable"
8418 );
8419 assert!(parse_error(&route_response[0])["message"]
8420 .as_str()
8421 .unwrap()
8422 .contains("does not provide the requested target"));
8423 assert!(module_rx.try_recv().is_err());
8424
8425 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
8426 let health_handler = handler.clone();
8427 let health_task = tokio::spawn(async move {
8428 health_handler
8429 .handle_control_frame(
8430 &health_client_ctx,
8431 supervisor_health_probe_frame(302, "mcp"),
8432 )
8433 .await
8434 .unwrap()
8435 });
8436 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8437 .await
8438 .unwrap()
8439 .unwrap();
8440 assert_eq!(
8441 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
8442 ModuleControlRequest::HealthCheck {}
8443 );
8444 handler
8445 .handle_control_frame(
8446 &module_ctx,
8447 health_response(health_frame.header.corr, HealthStatus::Ok),
8448 )
8449 .await
8450 .unwrap();
8451 let health_response = health_task.await.unwrap();
8452 assert_eq!(health_response[0].header.ty, FrameType::Response);
8453 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
8454 ClientControlResponse::SupervisorHealthProbe {
8455 module_id, status, ..
8456 } => {
8457 assert_eq!(module_id, "mcp");
8458 assert_eq!(status, HealthStatus::Ok);
8459 }
8460 other => panic!("unexpected health response: {other:?}"),
8461 }
8462
8463 forwarding
8468 .cleanup_connection(module_ctx.connection_id)
8469 .unwrap();
8470 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
8471 let cleanup_response = tokio::time::timeout(
8472 Duration::from_millis(200),
8473 handler.handle_control_frame(
8474 &cleanup_probe_ctx,
8475 supervisor_health_probe_frame(303, "mcp"),
8476 ),
8477 )
8478 .await
8479 .expect("probe should fail immediately when the control lane is gone")
8480 .unwrap();
8481 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
8482 assert_eq!(
8483 parse_error(&cleanup_response[0])["code"],
8484 "target_unavailable"
8485 );
8486 assert!(parse_error(&cleanup_response[0])["message"]
8487 .as_str()
8488 .unwrap()
8489 .contains("no module connection"));
8490
8491 handler
8492 .cleanup_connection(module_ctx.connection_id)
8493 .unwrap();
8494 }
8495
8496 #[tokio::test]
8497 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
8498 let registry = Arc::new(Registry::default());
8499 let supervisor_handle = SupervisorHandle::new();
8500 let supervisor =
8501 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
8502 .with_handle(supervisor_handle.clone())
8503 .with_connection_file_path(
8504 std::env::temp_dir()
8505 .join(format!("subc-route-open-warming-{}", std::process::id())),
8506 );
8507 let module = supervisor
8508 .supervise_configured(
8509 ModuleSpec {
8510 module_id: "warming".to_string(),
8511 program: fake_aft_stub_path(),
8512 args: Vec::new(),
8513 env: Vec::new(),
8514 reserved: false,
8515 reserved_prefixes: Vec::new(),
8516 protocol: ModuleProtocol::Subc,
8517 overlap: Default::default(),
8518 },
8519 true,
8520 )
8521 .unwrap();
8522 assert_eq!(module.state().unwrap(), ModuleState::Running);
8523
8524 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8525 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
8526 let response = handler
8527 .handle_control_frame(
8528 &ctx,
8529 route_open_frame(304, "warming", unique_project_root("warming")),
8530 )
8531 .await
8532 .unwrap();
8533 module.stop().await.unwrap();
8534
8535 assert_eq!(response[0].header.ty, FrameType::Error);
8536 let error = parse_error(&response[0]);
8537 assert_eq!(error["code"], "module_warming");
8538 assert!(error["message"]
8539 .as_str()
8540 .unwrap()
8541 .contains("state=running, enabled=true, live=false"));
8542 }
8543
8544 #[tokio::test]
8549 async fn route_open_refusal_names_the_check_that_refused() {
8550 let handler = ControlHandler::new(Arc::new(Registry::default()));
8551 let capture = EventCapture::default();
8552 let _subscriber =
8553 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8554 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
8555 let response = handler
8556 .handle_control_frame(
8557 &ctx,
8558 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
8559 )
8560 .await
8561 .unwrap();
8562
8563 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
8564 let event = capture
8565 .events()
8566 .into_iter()
8567 .find(|event| {
8568 event.target == "control"
8569 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
8570 })
8571 .expect("route.open refusal event");
8572 assert_eq!(
8573 event.fields.get("reason"),
8574 Some(&"\"not_registered\"".to_string())
8575 );
8576 }
8577
8578 #[tokio::test]
8579 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
8580 let registry = Arc::new(Registry::default());
8581 let supervisor_handle = SupervisorHandle::new();
8582 let supervisor =
8583 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
8584 .with_handle(supervisor_handle.clone())
8585 .with_connection_file_path(std::env::temp_dir().join(format!(
8586 "subc-route-open-refusal-info-{}",
8587 std::process::id()
8588 )));
8589 let module = supervisor
8590 .supervise_configured(
8591 ModuleSpec {
8592 module_id: "warming".to_string(),
8593 program: fake_aft_stub_path(),
8594 args: Vec::new(),
8595 env: Vec::new(),
8596 reserved: false,
8597 reserved_prefixes: Vec::new(),
8598 protocol: ModuleProtocol::Subc,
8599 overlap: Default::default(),
8600 },
8601 true,
8602 )
8603 .unwrap();
8604 assert_eq!(module.state().unwrap(), ModuleState::Running);
8605
8606 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8607 assert!(handler
8608 .counters()
8609 .snapshot()
8610 .get("route_open_refused_by_code")
8611 .is_none());
8612 let capture = EventCapture::default();
8613 let _subscriber =
8614 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8615 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
8616 let response = handler
8617 .handle_control_frame(
8618 &ctx,
8619 route_open_frame(394, "warming", unique_project_root("refusal-info")),
8620 )
8621 .await
8622 .unwrap();
8623 module.stop().await.unwrap();
8624
8625 assert_eq!(parse_error(&response[0])["code"], "module_warming");
8626 let event = capture
8627 .events()
8628 .into_iter()
8629 .find(|event| {
8630 event.target == "control"
8631 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
8632 })
8633 .expect("route.open refusal event");
8634 assert_eq!(
8635 event.fields.get("module_id"),
8636 Some(&"\"warming\"".to_string())
8637 );
8638 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
8639 assert_eq!(
8640 event.fields.get("reason"),
8641 Some(&"\"supervised_not_registered\"".to_string())
8642 );
8643 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
8644 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
8645 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
8646 assert_eq!(
8647 handler.counters().snapshot()["route_open_refused_by_code"],
8648 json!({ "module_warming": 1 })
8649 );
8650 }
8651
8652 #[tokio::test(flavor = "current_thread")]
8653 async fn route_open_unknown_module_escapes_target_module_id() {
8654 let handler = ControlHandler::new(Arc::new(Registry::default()));
8655 let capture = EventCapture::default();
8656 let _subscriber =
8657 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8658 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
8659 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
8660 let response = handler
8661 .handle_control_frame(
8662 &ctx,
8663 route_open_frame(
8664 395,
8665 hostile_module_id,
8666 unique_project_root("hostile-target-module-id"),
8667 ),
8668 )
8669 .await
8670 .unwrap();
8671
8672 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
8673 let event = capture
8674 .events()
8675 .into_iter()
8676 .find(|event| {
8677 event.target == "control"
8678 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
8679 })
8680 .expect("route.open unknown-module refusal event");
8681 let logged = event.fields.get("module_id").expect("module_id field");
8682 assert!(!logged.bytes().any(|byte| byte < 0x20));
8683 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
8684 }
8685
8686 #[tokio::test(flavor = "current_thread")]
8687 async fn route_open_module_rejection_uses_daemon_counter_key() {
8688 let registry = Arc::new(Registry::default());
8689 let forwarding = Arc::new(ForwardingTable::default());
8690 let handler =
8691 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8692 let module_connection = ConnectionId::new(95);
8693 let (module_ctx, mut module_rx) = route_ctx(module_connection);
8694 hello_via_sink(
8695 &handler,
8696 &module_ctx,
8697 &mut module_rx,
8698 hello_frame("aft", PROTOCOL_VERSION, 395),
8699 )
8700 .await;
8701
8702 let client_connection = ConnectionId::new(96);
8703 let (client_ctx, _client_rx) = route_ctx(client_connection);
8704 let capture = EventCapture::default();
8705 let _subscriber =
8706 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8707 let (route_task, bind) = relay_route_open(
8708 &handler,
8709 client_connection,
8710 &client_ctx.egress,
8711 &mut module_rx,
8712 396,
8713 "aft",
8714 "hostile-module-code",
8715 )
8716 .await;
8717 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
8718 let rejection = Frame::build(
8719 FrameType::Error,
8720 control_flags(),
8721 0,
8722 0,
8723 bind.header.corr,
8724 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
8725 )
8726 .unwrap();
8727 handler
8728 .handle_control_frame(&module_ctx, rejection)
8729 .await
8730 .unwrap();
8731
8732 let response = route_task.await.unwrap();
8733 assert_eq!(parse_error(&response[0])["code"], hostile_code);
8734 let counters = handler.counters().snapshot();
8735 assert_eq!(
8736 counters["route_open_refused_by_code"],
8737 json!({ "module_rejected": 1 })
8738 );
8739 assert!(counters["route_open_refused_by_code"]
8740 .get(hostile_code)
8741 .is_none());
8742
8743 let event = capture
8744 .events()
8745 .into_iter()
8746 .find(|event| {
8747 event.target == "control"
8748 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
8749 })
8750 .expect("route.open module-rejection refusal event");
8751 let logged = event.fields.get("module_code").expect("module_code field");
8752 assert!(!logged.bytes().any(|byte| byte < 0x20));
8753 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
8754 }
8755
8756 #[tokio::test]
8757 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
8758 let registry = Arc::new(Registry::default());
8759 let supervisor_handle = SupervisorHandle::new();
8760 let missing_program = std::env::temp_dir().join(format!(
8761 "subc-route-open-missing-program-{}",
8762 std::process::id()
8763 ));
8764 let supervisor =
8765 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
8766 .with_handle(supervisor_handle.clone());
8767 let module = supervisor
8768 .supervise_configured(
8769 ModuleSpec {
8770 module_id: "failed".to_string(),
8771 program: missing_program,
8772 args: Vec::new(),
8773 env: Vec::new(),
8774 reserved: false,
8775 reserved_prefixes: Vec::new(),
8776 protocol: ModuleProtocol::Subc,
8777 overlap: Default::default(),
8778 },
8779 true,
8780 )
8781 .unwrap();
8782 assert_eq!(module.state().unwrap(), ModuleState::Failed);
8783
8784 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8785 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
8786 let response = handler
8787 .handle_control_frame(
8788 &ctx,
8789 route_open_frame(305, "failed", unique_project_root("failed")),
8790 )
8791 .await
8792 .unwrap();
8793
8794 assert_eq!(response[0].header.ty, FrameType::Error);
8795 let error = parse_error(&response[0]);
8796 assert_eq!(error["code"], "target_unavailable");
8797 assert!(error["message"]
8798 .as_str()
8799 .unwrap()
8800 .contains("state=failed, enabled=true, live=false"));
8801 }
8802
8803 #[tokio::test]
8804 async fn route_open_role_mismatch_remains_target_unavailable() {
8805 let registry = Arc::new(Registry::default());
8806 let handler = ControlHandler::new(Arc::clone(®istry));
8807 handler
8808 .handle_control(
8809 ConnectionId::new(41),
8810 non_routable_hello_frame_with_control_ops("health-only", 306, None),
8811 )
8812 .unwrap();
8813
8814 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
8815 let response = handler
8816 .handle_control_frame(
8817 &ctx,
8818 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
8819 )
8820 .await
8821 .unwrap();
8822
8823 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
8824 assert!(parse_error(&response[0])["message"]
8825 .as_str()
8826 .unwrap()
8827 .contains("does not provide the requested target"));
8828 }
8829
8830 #[tokio::test]
8831 async fn route_open_inactive_registration_remains_target_unavailable() {
8832 let registry = Arc::new(Registry::default());
8833 let handler = ControlHandler::new(Arc::clone(®istry));
8834 handler
8835 .handle_control(
8836 ConnectionId::new(43),
8837 hello_frame("inactive", PROTOCOL_VERSION, 308),
8838 )
8839 .unwrap();
8840 assert!(registry
8841 .set_module_state_for_test("inactive", ChannelState::Closed)
8842 .unwrap());
8843
8844 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
8845 let response = handler
8846 .handle_control_frame(
8847 &ctx,
8848 route_open_frame(309, "inactive", unique_project_root("inactive")),
8849 )
8850 .await
8851 .unwrap();
8852
8853 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
8854 assert!(parse_error(&response[0])["message"]
8855 .as_str()
8856 .unwrap()
8857 .contains("is not active"));
8858 }
8859
8860 #[tokio::test]
8861 async fn late_health_reply_is_recorded_through_the_module_response_path() {
8862 let registry = Arc::new(Registry::default());
8863 let forwarding = Arc::new(ForwardingTable::default());
8864 let supervisor_handle = SupervisorHandle::new();
8865 let supervisor = Supervisor::new(Arc::clone(®istry), crate::RestartPolicy::default())
8866 .with_forwarding(Arc::clone(&forwarding))
8867 .with_handle(supervisor_handle.clone());
8868 let module = supervisor
8869 .supervise_configured(
8870 crate::ModuleSpec {
8871 module_id: "late-health-response".to_string(),
8872 program: PathBuf::from("disabled-module"),
8873 args: Vec::new(),
8874 env: Vec::new(),
8875 reserved: false,
8876 reserved_prefixes: Vec::new(),
8877 protocol: ModuleProtocol::Subc,
8878 overlap: Default::default(),
8879 },
8880 false,
8881 )
8882 .unwrap();
8883 let handler =
8884 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8885 .with_supervisor(supervisor_handle);
8886 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
8887 handler
8888 .handle_control_frame(
8889 &module_ctx,
8890 hello_frame_with_control_ops(
8891 "late-health-response",
8892 PROTOCOL_VERSION,
8893 7,
8894 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
8895 ),
8896 )
8897 .await
8898 .unwrap();
8899 let probe_started_at = Instant::now() - Duration::from_millis(80);
8900 let pending = forwarding
8901 .begin_health_probe_rpc_for(
8902 "late-health-response",
8903 MODULE_CONTROL_OP_HEALTH_CHECK,
8904 probe_started_at,
8905 Instant::now() - Duration::from_millis(1),
8906 )
8907 .unwrap();
8908 assert!(forwarding
8909 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
8910 .unwrap());
8911
8912 let responses = handler
8913 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
8914 .await
8915 .unwrap();
8916
8917 assert!(responses.is_empty());
8918 let health = module.status().unwrap().health;
8919 assert_eq!(health.late_answer_count, 1);
8920 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
8921 }
8922
8923 #[tokio::test]
8924 async fn health_probe_timeout_and_module_death_are_typed() {
8925 let registry = Arc::new(Registry::default());
8926 let forwarding = Arc::new(ForwardingTable::default());
8927 let handler =
8928 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8929 .with_health_probe_timeout(Duration::from_millis(50));
8930 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
8931 hello_via_sink(
8932 &handler,
8933 &module_ctx,
8934 &mut module_rx,
8935 hello_frame_with_control_ops(
8936 "aft",
8937 PROTOCOL_VERSION,
8938 7,
8939 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
8940 ),
8941 )
8942 .await;
8943
8944 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
8945 let responses = handler
8946 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
8947 .await
8948 .unwrap();
8949 assert_eq!(responses[0].header.ty, FrameType::Error);
8950 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
8951 let _ = module_rx.try_recv();
8952
8953 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
8954 let health_handler = handler.clone();
8955 let death_task = tokio::spawn(async move {
8956 health_handler
8957 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
8958 .await
8959 .unwrap()
8960 });
8961 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8962 .await
8963 .unwrap()
8964 .unwrap();
8965 handler
8966 .cleanup_connection(module_ctx.connection_id)
8967 .unwrap();
8968 let responses = death_task.await.unwrap();
8969 assert_eq!(responses[0].header.ty, FrameType::Error);
8970 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
8971 }
8972
8973 #[test]
8974 fn hello_requires_exact_protocol_version() {
8975 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
8976 let registry = Arc::new(Registry::default());
8977 let handler = ControlHandler::new(Arc::clone(®istry));
8978 let responses = handler
8979 .handle_control(
8980 ConnectionId::new(connection),
8981 hello_frame("aft", offered, 9),
8982 )
8983 .unwrap();
8984
8985 assert_eq!(responses.len(), 1);
8986 assert_eq!(responses[0].header.ty, FrameType::Error);
8987 let error = parse_error(&responses[0]);
8988 assert_eq!(error["code"], "version_unsupported");
8989 assert!(registry.get_module("aft").unwrap().is_none());
8990 assert_eq!(registry.active_registration_count().unwrap(), 0);
8991 }
8992 }
8993
8994 #[test]
8995 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
8996 let registry = Arc::new(Registry::default());
8997 let forwarding = Arc::new(ForwardingTable::default());
8998 let handler =
8999 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9000 let module_connection = ConnectionId::new(301);
9001 let registration = registry
9002 .register_with_control_ops(
9003 manifest("aft-push", PROTOCOL_VERSION),
9004 PROTOCOL_VERSION,
9005 module_connection,
9006 module_baseline_control_ops(),
9007 )
9008 .unwrap();
9009 let (module_tx, _module_rx) = mpsc::channel(8);
9010 let endpoint = forwarding
9011 .register_module_connection(
9012 module_connection,
9013 "aft-push".to_string(),
9014 PROTOCOL_VERSION,
9015 manifest_concurrency(®istration.manifest),
9016 FrameSink::new(module_tx),
9017 )
9018 .unwrap();
9019
9020 let unknown = Frame::build(
9022 FrameType::Push,
9023 control_flags(),
9024 0,
9025 0,
9026 5,
9027 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
9028 )
9029 .unwrap();
9030 let out = handler.handle_status_update(endpoint, unknown).unwrap();
9031 assert!(
9032 out.is_empty(),
9033 "unknown push op must be ignored, got {out:?}"
9034 );
9035
9036 let malformed = Frame::build(
9038 FrameType::Push,
9039 control_flags(),
9040 0,
9041 0,
9042 6,
9043 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
9044 )
9045 .unwrap();
9046 let out = handler.handle_status_update(endpoint, malformed).unwrap();
9047 assert_eq!(out.len(), 1);
9048 assert_eq!(out[0].header.ty, FrameType::Error);
9049 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
9050 }
9051
9052 #[test]
9053 fn hello_rejected_when_connection_already_owns_client_routes() {
9054 let registry = Arc::new(Registry::default());
9055 let forwarding = Arc::new(ForwardingTable::default());
9056 let handler =
9057 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9058 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
9060 let client_connection = ConnectionId::new(202);
9061
9062 let responses = handler
9065 .handle_control(
9066 client_connection,
9067 hello_frame("aft-second", PROTOCOL_VERSION, 9),
9068 )
9069 .unwrap();
9070 assert_eq!(responses[0].header.ty, FrameType::Error);
9071 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
9072 assert!(registry.get_module("aft-second").unwrap().is_none());
9073 }
9074
9075 #[test]
9076 fn reserved_module_hello_requires_matching_launch_nonce() {
9077 let registry = Arc::new(Registry::default());
9078 let supervisor = SupervisorHandle::new();
9079 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
9082 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9083
9084 let no_nonce = handler
9086 .handle_control(
9087 ConnectionId::new(1),
9088 hello_frame("vault", PROTOCOL_VERSION, 1),
9089 )
9090 .unwrap();
9091 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
9092 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
9093 assert!(registry.get_module("vault").unwrap().is_none());
9094
9095 let wrong = handler
9097 .handle_control(
9098 ConnectionId::new(2),
9099 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
9100 )
9101 .unwrap();
9102 assert_eq!(wrong[0].header.ty, FrameType::Error);
9103 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
9104 assert!(registry.get_module("vault").unwrap().is_none());
9105
9106 let ok = handler
9108 .handle_control(
9109 ConnectionId::new(3),
9110 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
9111 )
9112 .unwrap();
9113 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
9114 assert!(registry.get_module("vault").unwrap().is_some());
9115 }
9116
9117 #[test]
9118 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
9119 let registry = Arc::new(Registry::default());
9120 let supervisor = SupervisorHandle::new();
9121 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
9122 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
9123 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9124
9125 let squat = handler
9126 .handle_control(
9127 ConnectionId::new(1),
9128 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
9129 )
9130 .unwrap();
9131 assert_eq!(squat[0].header.ty, FrameType::Error);
9132 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
9133 assert!(parse_error(&squat[0])["message"]
9134 .as_str()
9135 .unwrap()
9136 .contains("fed:"));
9137
9138 let accepted_peer = handler
9139 .handle_control(
9140 ConnectionId::new(2),
9141 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
9142 )
9143 .unwrap();
9144 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
9145
9146 let accepted_short = handler
9147 .handle_control(
9148 ConnectionId::new(3),
9149 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
9150 )
9151 .unwrap();
9152 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
9153
9154 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
9155 let response = handler
9156 .handle_control(
9157 ConnectionId::new(conn),
9158 hello_frame(module_id, PROTOCOL_VERSION, conn),
9159 )
9160 .unwrap();
9161 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
9162 }
9163 }
9164
9165 #[test]
9166 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
9167 let registry = Arc::new(Registry::default());
9168 let supervisor = SupervisorHandle::new();
9169 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
9170 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
9171 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
9172 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9173
9174 let owner_nonce = handler
9175 .handle_control(
9176 ConnectionId::new(1),
9177 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
9178 )
9179 .unwrap();
9180 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
9181 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
9182 assert!(registry.get_module("fed:special").unwrap().is_none());
9183
9184 let exact_nonce = handler
9185 .handle_control(
9186 ConnectionId::new(2),
9187 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
9188 )
9189 .unwrap();
9190 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
9191 assert!(registry.get_module("fed:special").unwrap().is_some());
9192 }
9193
9194 #[test]
9195 fn non_reserved_module_ignores_launch_nonce() {
9196 let registry = Arc::new(Registry::default());
9197 let handler = ControlHandler::new(Arc::clone(®istry));
9200 let no_nonce = handler
9201 .handle_control(
9202 ConnectionId::new(1),
9203 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
9204 )
9205 .unwrap();
9206 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
9207 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
9208
9209 let echoed_nonce = handler
9210 .handle_control(
9211 ConnectionId::new(2),
9212 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
9213 )
9214 .unwrap();
9215 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
9216 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
9217 }
9218
9219 #[test]
9220 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
9221 let handler = ControlHandler::default();
9222 let conn = ConnectionId::new(1);
9223 let malformed = Frame::build(
9224 FrameType::Hello,
9225 control_flags(),
9226 0,
9227 0,
9228 3,
9229 b"{not json".to_vec(),
9230 )
9231 .unwrap();
9232
9233 let error = handler.handle_control(conn, malformed).unwrap();
9234 assert_eq!(error[0].header.ty, FrameType::Error);
9235 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
9236
9237 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
9238 let pong = handler.handle_control(conn, ping).unwrap();
9239 assert_eq!(pong[0].header.ty, FrameType::Pong);
9240 assert_eq!(pong[0].header.corr, 4);
9241 }
9242
9243 #[test]
9244 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
9245 let registry = Arc::new(Registry::default());
9246 let handler = ControlHandler::new(Arc::clone(®istry));
9247
9248 handler
9249 .handle_control(
9250 ConnectionId::new(1),
9251 hello_frame("aft", PROTOCOL_VERSION, 1),
9252 )
9253 .unwrap();
9254 let duplicate = handler
9255 .handle_control(
9256 ConnectionId::new(2),
9257 hello_frame("aft", PROTOCOL_VERSION, 2),
9258 )
9259 .unwrap();
9260
9261 assert_eq!(duplicate[0].header.ty, FrameType::Error);
9262 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
9263 let registration = registry.get_module("aft").unwrap().unwrap();
9264 assert_eq!(registration.connection_id, ConnectionId::new(1));
9265 }
9266
9267 #[test]
9268 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
9269 let registry = Arc::new(Registry::default());
9270 let forwarding = Arc::new(ForwardingTable::default());
9271 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
9272 let handler =
9273 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9274 .with_process_liveness(process_liveness);
9275 let (ctx, route_channel, route_epoch) =
9276 bind_liveness_route(®istry, &forwarding, "aft-dead");
9277 let responses = handler
9278 .handle_route_poll(
9279 &ctx,
9280 route_poll_frame(41, PollKind::Liveness, route_channel),
9281 route_channel,
9282 route_epoch,
9283 PollKind::Liveness,
9284 )
9285 .unwrap();
9286
9287 assert_eq!(responses.len(), 1);
9288 assert_eq!(responses[0].header.ty, FrameType::Response);
9289 assert_route_poll_liveness(&responses[0], false);
9290 }
9291
9292 #[test]
9293 fn liveness_poll_without_process_source_uses_bound_route() {
9294 let registry = Arc::new(Registry::default());
9295 let forwarding = Arc::new(ForwardingTable::default());
9296 let handler =
9297 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9298 let (ctx, route_channel, route_epoch) =
9299 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
9300 let responses = handler
9301 .handle_route_poll(
9302 &ctx,
9303 route_poll_frame(42, PollKind::Liveness, route_channel),
9304 route_channel,
9305 route_epoch,
9306 PollKind::Liveness,
9307 )
9308 .unwrap();
9309
9310 assert_route_poll_liveness(&responses[0], true);
9311 }
9312
9313 #[test]
9314 fn liveness_poll_untracked_process_source_uses_bound_route() {
9315 let registry = Arc::new(Registry::default());
9316 let forwarding = Arc::new(ForwardingTable::default());
9317 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
9318 let handler =
9319 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9320 .with_process_liveness(process_liveness);
9321 let (ctx, route_channel, route_epoch) =
9322 bind_liveness_route(®istry, &forwarding, "aft-untracked");
9323 let responses = handler
9324 .handle_route_poll(
9325 &ctx,
9326 route_poll_frame(43, PollKind::Liveness, route_channel),
9327 route_channel,
9328 route_epoch,
9329 PollKind::Liveness,
9330 )
9331 .unwrap();
9332
9333 assert_route_poll_liveness(&responses[0], true);
9334 }
9335
9336 #[tokio::test]
9337 async fn unknown_op_returns_unknown_control_op() {
9338 let handler = ControlHandler::default();
9339 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
9340 let request = Frame::build(
9341 FrameType::Request,
9342 control_flags(),
9343 0,
9344 0,
9345 55,
9346 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
9347 )
9348 .unwrap();
9349
9350 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
9351
9352 assert_eq!(response.len(), 1);
9353 assert_eq!(response[0].header.ty, FrameType::Error);
9354 assert_eq!(response[0].header.corr, 55);
9355 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
9356 }
9357
9358 #[tokio::test]
9359 async fn supervisor_provenance_rejects_unknown_exact_module() {
9360 let handler = ControlHandler::default();
9361 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
9362 let request = Frame::build(
9363 FrameType::Request,
9364 control_flags(),
9365 0,
9366 0,
9367 57,
9368 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
9369 )
9370 .unwrap();
9371
9372 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
9373
9374 assert_eq!(response.len(), 1);
9375 assert_eq!(response[0].header.ty, FrameType::Error);
9376 assert_eq!(response[0].header.corr, 57);
9377 let error = parse_error(&response[0]);
9378 assert_eq!(error["code"], "unknown_module");
9379 assert_eq!(error["message"], "module_id 'missing' is not supervised");
9380 }
9381
9382 #[test]
9383 fn provenance_probe_override_keeps_handler_tests_deterministic() {
9384 let expected = subc_control::RunningImageAgreement::Unavailable {
9385 reason: subc_control::RunningImageUnavailableReason::HashFailed,
9386 };
9387 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
9388 assert_eq!(handler.provenance_probe_override, Some(expected));
9389 }
9390
9391 #[test]
9392 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
9393 let verdict = reload_verdict(
9394 std::path::Path::new("/bin/new"),
9395 Some(std::path::Path::new("/bin/old")),
9396 subc_control::RunningImageAgreement::Unavailable {
9397 reason: subc_control::RunningImageUnavailableReason::HashFailed,
9398 },
9399 );
9400 assert!(matches!(
9401 verdict.path,
9402 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
9403 if configured == std::path::Path::new("/bin/new")
9404 && spawned_from == std::path::Path::new("/bin/old")
9405 ));
9406 }
9407
9408 #[test]
9409 fn reload_verdict_detects_replaced_image_at_same_path() {
9410 let image = subc_control::RunningImageAgreement::Mismatch {
9411 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
9412 digest: "old".into(),
9413 },
9414 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
9415 digest: "new".into(),
9416 },
9417 };
9418 let verdict = reload_verdict(
9419 std::path::Path::new("/bin/same"),
9420 Some(std::path::Path::new("/bin/same")),
9421 image.clone(),
9422 );
9423 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
9424 assert_eq!(verdict.image, image);
9425 }
9426
9427 #[test]
9428 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
9429 let image = subc_control::RunningImageAgreement::Unavailable {
9430 reason: subc_control::RunningImageUnavailableReason::NotRunning,
9431 };
9432 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
9433 assert_eq!(
9434 verdict.path,
9435 subc_control::ReloadPathAgreement::Unavailable {
9436 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
9437 }
9438 );
9439 assert_eq!(verdict.image, image);
9440
9441 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
9442 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
9443 };
9444 let verdict = reload_verdict(
9445 std::path::Path::new("/bin/same"),
9446 Some(std::path::Path::new("/bin/same")),
9447 unconfirmed.clone(),
9448 );
9449 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
9450 assert_eq!(verdict.image, unconfirmed);
9451 }
9452
9453 #[test]
9454 fn reload_verdict_preserves_each_image_unavailability_reason() {
9455 use subc_control::RunningImageUnavailableReason as Reason;
9456
9457 for reason in [
9458 Reason::NotRunning,
9459 Reason::UnsupportedPlatform,
9460 Reason::RunningExecutableUnreadable,
9461 Reason::SpawnedPathUnreadable,
9462 Reason::HashFailed,
9463 Reason::ProcessIdentityUnconfirmed,
9464 Reason::Unknown("future_probe_reason".to_string()),
9465 ] {
9466 let image = subc_control::RunningImageAgreement::Unavailable {
9467 reason: reason.clone(),
9468 };
9469 let verdict = reload_verdict(
9470 std::path::Path::new("/bin/same"),
9471 Some(std::path::Path::new("/bin/same")),
9472 image.clone(),
9473 );
9474 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
9475 assert_eq!(verdict.image, image, "{reason:?}");
9476 }
9477 }
9478
9479 #[tokio::test]
9480 async fn malformed_control_bodies_return_invalid_control_body() {
9481 let handler = ControlHandler::default();
9482 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
9483
9484 for (corr, body) in [
9485 (56, br#"{"route_channel":1}"#.as_slice()),
9486 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
9487 (
9488 58,
9489 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
9490 ),
9491 ] {
9492 let request = Frame::build(
9493 FrameType::Request,
9494 control_flags(),
9495 0,
9496 0,
9497 corr,
9498 body.to_vec(),
9499 )
9500 .unwrap();
9501 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
9502
9503 assert_eq!(response.len(), 1);
9504 assert_eq!(response[0].header.ty, FrameType::Error);
9505 assert_eq!(response[0].header.corr, corr);
9506 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
9507 }
9508 }
9509
9510 #[tokio::test]
9511 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
9512 let registry = Arc::new(Registry::default());
9513 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
9514 let router = Router::with_control_handler(Arc::clone(&control));
9515 let connection = router.begin_connection();
9516 let (ctx, mut rx) = route_ctx(connection.id());
9517
9518 router
9519 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
9520 .await
9521 .unwrap();
9522 let response = rx.recv().await.unwrap();
9523 let ack = parse_ack(&response);
9524 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9525 let channel = 1;
9526
9527 let goodbye =
9528 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
9529 router.route_for_connection(&ctx, goodbye).await.unwrap();
9530 assert!(rx.try_recv().is_err());
9531 assert!(registry.get_module("aft").unwrap().is_none());
9532
9533 router
9534 .route_for_connection(&ctx, channel_request(channel, 13))
9535 .await
9536 .unwrap();
9537 let error_frame = rx.recv().await.unwrap();
9538 assert_eq!(error_frame.header.ty, FrameType::Error);
9539 assert_eq!(error_frame.header.channel, channel);
9540 }
9541
9542 #[tokio::test]
9543 async fn dropping_router_connection_releases_registration() {
9544 let registry = Arc::new(Registry::default());
9545 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
9546 let router = Router::with_control_handler(control);
9547 let connection = router.begin_connection();
9548 let (ctx, mut rx) = route_ctx(connection.id());
9549
9550 router
9551 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
9552 .await
9553 .unwrap();
9554 let response = rx.recv().await.unwrap();
9555 let ack = parse_ack(&response);
9556 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9557 assert!(registry.get_module("aft").unwrap().is_some());
9558
9559 drop(connection);
9560
9561 assert!(registry.get_module("aft").unwrap().is_none());
9562 assert_eq!(registry.active_registration_count().unwrap(), 0);
9563 }
9564
9565 fn capability_manifest(
9566 module_id: &str,
9567 provides: &[&str],
9568 must_never_reach: &[&str],
9569 ) -> ModuleManifest {
9570 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
9571 manifest.capabilities = Some(CapabilityDeclarations {
9572 provides: provides
9573 .iter()
9574 .map(|capability| (*capability).to_string())
9575 .collect(),
9576 requires: Vec::new(),
9577 must_never_reach: must_never_reach
9578 .iter()
9579 .map(|capability| (*capability).to_string())
9580 .collect(),
9581 });
9582 manifest
9583 }
9584
9585 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
9586 Frame::build(
9587 FrameType::Hello,
9588 control_flags(),
9589 0,
9590 0,
9591 corr,
9592 serde_json::to_vec(&ModuleHelloBody {
9593 protocol_ver: manifest.protocol_ver,
9594 manifest,
9595 control_ops: None,
9596 launch_nonce: None,
9597 })
9598 .expect("capability test HELLO serializes"),
9599 )
9600 .expect("capability test HELLO frame builds")
9601 }
9602
9603 fn catalog_update_with_capabilities_frame(
9604 corr: u64,
9605 capabilities: CapabilityDeclarations,
9606 ) -> Frame {
9607 Frame::build(
9608 FrameType::Request,
9609 control_flags(),
9610 0,
9611 0,
9612 corr,
9613 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
9614 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
9615 capabilities: Some(capabilities),
9616 ready: None,
9617 })
9618 .expect("capability catalog.update serializes"),
9619 )
9620 .expect("capability catalog.update frame builds")
9621 }
9622
9623 async fn register_capability_manifest(
9624 handler: &ControlHandler,
9625 ctx: &RouteCtx,
9626 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9627 manifest: ModuleManifest,
9628 corr: u64,
9629 ) {
9630 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
9631 }
9632
9633 async fn open_route_for_capability_test(
9634 handler: &ControlHandler,
9635 target_ctx: &RouteCtx,
9636 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9637 client_connection_id: u64,
9638 corr: u64,
9639 target_module_id: &str,
9640 consumer_identity: Option<ConsumerIdentity>,
9641 ) -> (
9642 mpsc::Receiver<crate::router::OutboundFrame>,
9643 ModuleControlRequest,
9644 ) {
9645 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
9646 let route_handler = handler.clone();
9647 let target_module_id = target_module_id.to_string();
9648 let route_task = tokio::spawn(async move {
9649 route_handler
9650 .handle_control_frame(
9651 &client_ctx,
9652 route_open_frame_with_admission_facts(
9653 corr,
9654 &target_module_id,
9655 unique_project_root("admission-facts"),
9656 consumer_identity,
9657 None,
9658 ),
9659 )
9660 .await
9661 .expect("capability test route.open succeeds")
9662 });
9663 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
9664 .await
9665 .expect("capability test route.open must reach route.bind")
9666 .expect("target control receiver stays open");
9667 let bind_request: ModuleControlRequest =
9668 serde_json::from_slice(&bind.body).expect("route.bind decodes");
9669 handler
9670 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
9671 .await
9672 .expect("capability test route.bind ACK succeeds");
9673 assert!(route_task.await.expect("route.open task joins").is_empty());
9674 let opened = client_rx
9675 .recv()
9676 .await
9677 .expect("successful route.open publishes a response");
9678 assert!(matches!(
9679 serde_json::from_slice::<ClientControlResponse>(&opened.body),
9680 Ok(ClientControlResponse::RouteOpen { .. })
9681 ));
9682 (client_rx, bind_request)
9683 }
9684
9685 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
9686 assert_eq!(frame.header.ty, FrameType::Push);
9687 assert_eq!(frame.header.channel, 0);
9688 assert_eq!(
9689 serde_json::from_slice::<ClientControlPush>(&frame.body)
9690 .expect("route.closed control push decodes"),
9691 ClientControlPush::RouteClosed {
9692 module_id: target_module_id.to_string(),
9693 reason: RouteCloseReason::CapabilityDenied,
9694 drained: false,
9695 abandoned: 0,
9696 excluded_subscriptions: 0,
9697 terminal: Some(false),
9698 }
9699 );
9700 }
9701
9702 #[tokio::test]
9703 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
9704 let registry = Arc::new(Registry::default());
9705 let forwarding = Arc::new(ForwardingTable::default());
9706 let supervisor = SupervisorHandle::new();
9707 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9708 let handler =
9709 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9710 .with_supervisor(supervisor);
9711 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
9712 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
9713 register_capability_manifest(
9714 &handler,
9715 &target_ctx,
9716 &mut target_rx,
9717 capability_manifest("target", &["credentials-provider/v1"], &[]),
9718 1,
9719 )
9720 .await;
9721 register_capability_manifest(
9722 &handler,
9723 &opener_ctx,
9724 &mut opener_rx,
9725 capability_manifest("opener", &[], &["credentials-provider/v1"]),
9726 2,
9727 )
9728 .await;
9729
9730 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
9731 let replies = handler
9732 .handle_control_frame(
9733 &client_ctx,
9734 route_open_frame_with_admission_facts(
9735 3,
9736 "target",
9737 unique_project_root("admission-facts"),
9738 Some(ConsumerIdentity {
9739 module_id: "opener".to_string(),
9740 launch_nonce: "opener-nonce".to_string(),
9741 }),
9742 None,
9743 ),
9744 )
9745 .await
9746 .expect("denied route.open returns a typed frame");
9747 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
9748 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
9749 assert!(
9750 target_rx.try_recv().is_err(),
9751 "forbidden route.open must not relay route.bind"
9752 );
9753 }
9754
9755 #[tokio::test]
9756 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
9757 let registry = Arc::new(Registry::default());
9758 let forwarding = Arc::new(ForwardingTable::default());
9759 let supervisor = SupervisorHandle::new();
9760 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9761 let handler =
9762 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9763 .with_supervisor(supervisor);
9764 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
9765 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
9766 register_capability_manifest(
9767 &handler,
9768 &target_ctx,
9769 &mut target_rx,
9770 capability_manifest("target", &["credentials-provider/v1"], &[]),
9771 1,
9772 )
9773 .await;
9774 register_capability_manifest(
9775 &handler,
9776 &old_opener_ctx,
9777 &mut old_opener_rx,
9778 capability_manifest("opener", &[], &[]),
9779 2,
9780 )
9781 .await;
9782 let (mut client_rx, _) = open_route_for_capability_test(
9783 &handler,
9784 &target_ctx,
9785 &mut target_rx,
9786 712,
9787 3,
9788 "target",
9789 Some(ConsumerIdentity {
9790 module_id: "opener".to_string(),
9791 launch_nonce: "opener-nonce".to_string(),
9792 }),
9793 )
9794 .await;
9795 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
9796
9797 handler
9798 .cleanup_connection(old_opener_ctx.connection_id)
9799 .expect("old opener registration cleans up");
9800 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
9801 register_capability_manifest(
9802 &handler,
9803 &new_opener_ctx,
9804 &mut new_opener_rx,
9805 capability_manifest("opener", &[], &["credentials-provider/v1"]),
9806 4,
9807 )
9808 .await;
9809
9810 assert_capability_denied_push(
9811 client_rx
9812 .try_recv()
9813 .expect("HELLO deny addition must emit route.closed")
9814 .frame,
9815 "target",
9816 );
9817 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
9818 assert!(matches!(
9819 target_rx.try_recv(),
9820 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
9821 ));
9822 }
9823
9824 #[tokio::test]
9825 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
9826 let registry = Arc::new(Registry::default());
9827 let forwarding = Arc::new(ForwardingTable::default());
9828 let supervisor = SupervisorHandle::new();
9829 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9830 let handler =
9831 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9832 .with_supervisor(supervisor);
9833 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
9834 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
9835 register_capability_manifest(
9836 &handler,
9837 &target_ctx,
9838 &mut target_rx,
9839 capability_manifest("target", &[], &[]),
9840 1,
9841 )
9842 .await;
9843 register_capability_manifest(
9844 &handler,
9845 &opener_ctx,
9846 &mut opener_rx,
9847 capability_manifest("opener", &[], &["credentials-provider/v1"]),
9848 2,
9849 )
9850 .await;
9851 let (mut client_rx, _) = open_route_for_capability_test(
9852 &handler,
9853 &target_ctx,
9854 &mut target_rx,
9855 722,
9856 3,
9857 "target",
9858 Some(ConsumerIdentity {
9859 module_id: "opener".to_string(),
9860 launch_nonce: "opener-nonce".to_string(),
9861 }),
9862 )
9863 .await;
9864 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
9865
9866 let replies = handler
9867 .handle_control_frame(
9868 &target_ctx,
9869 catalog_update_with_capabilities_frame(
9870 4,
9871 CapabilityDeclarations {
9872 provides: vec!["credentials-provider/v1".to_string()],
9873 requires: Vec::new(),
9874 must_never_reach: Vec::new(),
9875 },
9876 ),
9877 )
9878 .await
9879 .expect("claim catalog.update succeeds");
9880 assert!(matches!(
9881 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
9882 Ok(ModuleControlResponseToModule::CatalogUpdate {})
9883 ));
9884 assert_capability_denied_push(
9885 client_rx
9886 .try_recv()
9887 .expect("claim addition must emit route.closed")
9888 .frame,
9889 "target",
9890 );
9891 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
9892 assert!(matches!(
9893 target_rx.try_recv(),
9894 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
9895 ));
9896 }
9897
9898 #[tokio::test]
9899 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
9900 let registry = Arc::new(Registry::default());
9901 let forwarding = Arc::new(ForwardingTable::default());
9902 let supervisor = SupervisorHandle::new();
9903 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9904 let handler =
9905 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9906 .with_supervisor(supervisor);
9907 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
9908 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
9909 register_capability_manifest(
9910 &handler,
9911 &target_ctx,
9912 &mut target_rx,
9913 capability_manifest("target", &["credentials-provider/v1"], &[]),
9914 1,
9915 )
9916 .await;
9917 register_capability_manifest(
9918 &handler,
9919 &opener_ctx,
9920 &mut opener_rx,
9921 capability_manifest("opener", &[], &[]),
9922 2,
9923 )
9924 .await;
9925 let (mut client_rx, _) = open_route_for_capability_test(
9926 &handler,
9927 &target_ctx,
9928 &mut target_rx,
9929 732,
9930 3,
9931 "target",
9932 Some(ConsumerIdentity {
9933 module_id: "opener".to_string(),
9934 launch_nonce: "opener-nonce".to_string(),
9935 }),
9936 )
9937 .await;
9938 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
9939
9940 handler
9941 .handle_control_frame(
9942 &target_ctx,
9943 catalog_update_with_capabilities_frame(
9944 4,
9945 CapabilityDeclarations {
9946 provides: Vec::new(),
9947 requires: Vec::new(),
9948 must_never_reach: Vec::new(),
9949 },
9950 ),
9951 )
9952 .await
9953 .expect("claim removal catalog.update succeeds");
9954 assert_eq!(
9955 forwarding.active_binding_count().unwrap(),
9956 1,
9957 "removing an attested target claim must leave the route census unchanged"
9958 );
9959 assert!(
9960 client_rx.try_recv().is_err(),
9961 "claim removal must not emit route.closed capability_denied"
9962 );
9963 assert!(
9964 target_rx.try_recv().is_err(),
9965 "claim removal must not send the target a route GOODBYE"
9966 );
9967 }
9968
9969 #[tokio::test]
9972 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
9973 let registry = Arc::new(Registry::default());
9974 let forwarding = Arc::new(ForwardingTable::default());
9975 let supervisor = SupervisorHandle::new();
9976 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9977 let handler =
9978 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9979 .with_supervisor(supervisor);
9980 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
9981 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
9982 register_capability_manifest(
9983 &handler,
9984 &target_ctx,
9985 &mut target_rx,
9986 capability_manifest("target", &["credentials-provider/v1"], &[]),
9987 1,
9988 )
9989 .await;
9990 register_capability_manifest(
9991 &handler,
9992 &opener_ctx,
9993 &mut opener_rx,
9994 capability_manifest("opener", &[], &["credentials-provider/v1"]),
9995 2,
9996 )
9997 .await;
9998
9999 let (_client_rx, bind) = open_route_for_capability_test(
10000 &handler,
10001 &target_ctx,
10002 &mut target_rx,
10003 742,
10004 3,
10005 "target",
10006 None,
10007 )
10008 .await;
10009 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
10010 panic!("direct scope-honesty route must bind");
10011 };
10012 assert_eq!(principal, Some(Principal::Direct));
10013 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
10014 }
10015
10016 #[tokio::test]
10019 async fn must_never_reach_self_route_is_capability_forbidden() {
10020 let registry = Arc::new(Registry::default());
10021 let forwarding = Arc::new(ForwardingTable::default());
10022 let supervisor = SupervisorHandle::new();
10023 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
10024 let handler =
10025 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10026 .with_supervisor(supervisor);
10027 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
10028 register_capability_manifest(
10029 &handler,
10030 &self_ctx,
10031 &mut self_rx,
10032 capability_manifest(
10033 "self-provider",
10034 &["credentials-provider/v1"],
10035 &["credentials-provider/v1"],
10036 ),
10037 1,
10038 )
10039 .await;
10040
10041 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
10042 let replies = handler
10043 .handle_control_frame(
10044 &client_ctx,
10045 route_open_frame_with_admission_facts(
10046 2,
10047 "self-provider",
10048 unique_project_root("admission-facts"),
10049 Some(ConsumerIdentity {
10050 module_id: "self-provider".to_string(),
10051 launch_nonce: "self-nonce".to_string(),
10052 }),
10053 None,
10054 ),
10055 )
10056 .await
10057 .expect("self-route refusal returns a typed frame");
10058 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
10059 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
10060 assert!(
10061 self_rx.try_recv().is_err(),
10062 "self denial must not relay route.bind"
10063 );
10064 }
10065
10066 #[test]
10067 fn unsupported_channel_zero_frame_returns_error() {
10068 let handler = ControlHandler::default();
10069 let request = Frame::build(
10070 FrameType::Request,
10071 control_flags(),
10072 0,
10073 0,
10074 21,
10075 b"opaque".to_vec(),
10076 )
10077 .unwrap();
10078
10079 let response = handler
10080 .handle_control(ConnectionId::new(1), request)
10081 .unwrap();
10082
10083 assert_eq!(response[0].header.ty, FrameType::Error);
10084 assert_eq!(
10085 parse_error(&response[0])["code"],
10086 "unsupported_control_frame"
10087 );
10088 }
10089
10090 mod swap {
10095 use super::*;
10096
10097 const INCUMBENT: ConnectionId = ConnectionId::new(30);
10098 const CANDIDATE: ConnectionId = ConnectionId::new(40);
10099
10100 struct Swap {
10101 registry: Arc<Registry>,
10102 forwarding: Arc<ForwardingTable>,
10103 handler: ControlHandler,
10104 incumbent_ctx: RouteCtx,
10105 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
10106 candidate_ctx: RouteCtx,
10107 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
10108 }
10109
10110 async fn swap_with_incumbent() -> Swap {
10111 let registry = Arc::new(Registry::default());
10112 let forwarding = Arc::new(ForwardingTable::default());
10113 let handler =
10114 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10115 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
10116 hello_via_sink(
10117 &handler,
10118 &incumbent_ctx,
10119 &mut incumbent_rx,
10120 hello_frame("aft", PROTOCOL_VERSION, 7),
10121 )
10122 .await;
10123 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
10124 Swap {
10125 registry,
10126 forwarding,
10127 handler,
10128 incumbent_ctx,
10129 incumbent_rx,
10130 candidate_ctx,
10131 candidate_rx,
10132 }
10133 }
10134
10135 fn register_candidate(swap: &Swap, ready: Option<bool>) {
10136 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
10137 candidate_manifest.ready = ready;
10138 let registration = swap
10139 .registry
10140 .register_candidate_with_control_ops(
10141 candidate_manifest,
10142 PROTOCOL_VERSION,
10143 CANDIDATE,
10144 module_baseline_control_ops(),
10145 )
10146 .unwrap();
10147 swap.forwarding
10148 .register_candidate_module_connection(
10149 CANDIDATE,
10150 "aft".to_string(),
10151 PROTOCOL_VERSION,
10152 manifest_concurrency(®istration.manifest),
10153 swap.candidate_ctx.egress.clone(),
10154 )
10155 .unwrap();
10156 }
10157
10158 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
10159 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
10160 swap.registry.promote_candidate("aft").unwrap().unwrap();
10161 cutover.incumbent.unwrap()
10162 }
10163
10164 fn keyed_total(counters: &Value, key: &str) -> u64 {
10165 counters[key]
10166 .as_object()
10167 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
10168 .unwrap_or(0)
10169 }
10170
10171 #[tokio::test]
10177 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
10178 let mut swap = swap_with_incumbent().await;
10179 let handler = swap.handler.clone();
10180
10181 let cotenant = ConnectionId::new(31);
10183 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
10184 let (cotenant_task, cotenant_bind) = relay_route_open(
10185 &handler,
10186 cotenant,
10187 &cotenant_ctx.egress,
10188 &mut swap.incumbent_rx,
10189 100,
10190 "aft",
10191 "swap-cotenant",
10192 )
10193 .await;
10194 handler
10195 .handle_control_frame(
10196 &swap.incumbent_ctx,
10197 route_bind_ack(cotenant_bind.header.corr),
10198 )
10199 .await
10200 .unwrap();
10201 assert!(cotenant_task.await.unwrap().is_empty());
10202 let (cotenant_channel, cotenant_epoch) =
10203 published_route(&cotenant_rx.recv().await.unwrap());
10204
10205 let caller = ConnectionId::new(32);
10207 let (caller_ctx, mut caller_rx) = route_ctx(caller);
10208 let (caller_task, caller_bind) = relay_route_open(
10209 &handler,
10210 caller,
10211 &caller_ctx.egress,
10212 &mut swap.incumbent_rx,
10213 101,
10214 "aft",
10215 "swap-caller",
10216 )
10217 .await;
10218 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
10219
10220 register_candidate(&swap, None);
10221 cutover(&swap);
10222
10223 let ack = handler
10225 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
10226 .await;
10227 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
10228 if module_loop_error.is_some() {
10229 handler.cleanup_connection(INCUMBENT).unwrap();
10232 }
10233
10234 assert!(
10236 cotenant_rx.try_recv().is_err(),
10237 "the co-tenant route on the incumbent was torn down by one late ack: \
10238 {module_loop_error:?}"
10239 );
10240 assert!(matches!(
10241 swap.forwarding
10242 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
10243 .unwrap(),
10244 DataRoute::Client(DataRouteState::Bound(_))
10245 ));
10246 assert_eq!(module_loop_error, None);
10247 assert!(swap
10248 .registry
10249 .get_module_by_connection(INCUMBENT)
10250 .unwrap()
10251 .is_some());
10252
10253 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
10255 .await
10256 .expect("the incumbent is told to drop the abandoned binding")
10257 .unwrap()
10258 .frame;
10259 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10260 assert_eq!(goodbye.header.channel, abandoned_channel);
10261 assert_eq!(goodbye.header.epoch, abandoned_epoch);
10262 assert!(swap.incumbent_rx.try_recv().is_err());
10263
10264 let response = caller_task.await.unwrap();
10266 assert_eq!(response.len(), 1);
10267 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
10268 assert!(caller_rx.try_recv().is_err());
10269
10270 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
10274 let counters = handler.counters().snapshot();
10275 assert_eq!(
10276 keyed_total(&counters, "route_open_accepted_by_principal"),
10277 1
10278 );
10279 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
10280 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
10281 }
10282
10283 #[tokio::test]
10287 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
10288 let mut swap = swap_with_incumbent().await;
10289 register_candidate(&swap, None);
10290 let incumbent = cutover(&swap);
10291 swap.forwarding
10292 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
10293 .unwrap()
10294 .expect("the incumbent is still registered");
10295
10296 let client = ConnectionId::new(33);
10297 let (client_ctx, mut client_rx) = route_ctx(client);
10298 let route_handler = swap.handler.clone();
10299 let open_ctx = RouteCtx {
10300 connection_id: client,
10301 egress: client_ctx.egress.clone(),
10302 };
10303 let mut route_task = tokio::spawn(async move {
10304 route_handler
10305 .handle_control_frame(
10306 &open_ctx,
10307 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
10308 )
10309 .await
10310 .unwrap()
10311 });
10312 let bind = tokio::select! {
10313 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
10314 response = &mut route_task => {
10315 let response = response.unwrap();
10316 panic!(
10317 "post-cutover route.open was refused instead of relayed to the candidate: {}",
10318 parse_error(&response[0])["code"]
10319 );
10320 }
10321 };
10322 swap.handler
10323 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
10324 .await
10325 .unwrap();
10326 assert!(route_task.await.unwrap().is_empty());
10327 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
10328 match swap
10329 .forwarding
10330 .lookup_data_route(client, channel, epoch)
10331 .unwrap()
10332 {
10333 DataRoute::Client(DataRouteState::Bound(route)) => {
10334 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
10335 }
10336 other => panic!("expected a bound route on the candidate, got {other:?}"),
10337 }
10338 assert!(swap.incumbent_rx.try_recv().is_err());
10339 }
10340
10341 #[tokio::test]
10346 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
10347 let swap = swap_with_incumbent().await;
10348 register_candidate(&swap, Some(false));
10349 let update = Frame::build(
10350 FrameType::Request,
10351 control_flags(),
10352 0,
10353 0,
10354 55,
10355 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
10356 provides: manifest("aft", PROTOCOL_VERSION).provides,
10357 capabilities: None,
10358 ready: Some(true),
10359 })
10360 .unwrap(),
10361 )
10362 .unwrap();
10363
10364 let replies = swap
10365 .handler
10366 .handle_control_frame(&swap.candidate_ctx, update)
10367 .await
10368 .unwrap();
10369
10370 assert_eq!(replies.len(), 1);
10371 assert_eq!(
10372 replies[0].header.ty,
10373 FrameType::Response,
10374 "candidate catalog.update was refused: {:?}",
10375 serde_json::from_slice::<Value>(&replies[0].body).ok()
10376 );
10377 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
10378 assert_eq!(
10379 swap.registry
10380 .get_module("aft")
10381 .unwrap()
10382 .unwrap()
10383 .connection_id,
10384 INCUMBENT
10385 );
10386 }
10387 }
10388
10389 mod swap_admission {
10393 use super::*;
10394
10395 const INCUMBENT_NONCE: &str = "incumbent-nonce";
10396 const CANDIDATE_NONCE: &str = "candidate-nonce";
10397
10398 fn handler_with_incumbent(
10399 module_id: &str,
10400 reserved: bool,
10401 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
10402 let registry = Arc::new(Registry::default());
10403 let supervisor = SupervisorHandle::new();
10404 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
10405 if reserved {
10406 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
10407 }
10408 let handler =
10409 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
10410 let incumbent = handler
10411 .handle_control(
10412 ConnectionId::new(1),
10413 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
10414 )
10415 .unwrap();
10416 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
10417 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
10418 (registry, supervisor, handler)
10419 }
10420
10421 #[test]
10427 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
10428 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
10429
10430 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
10431 let replies = handler
10432 .handle_control(
10433 ConnectionId::new(connection),
10434 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
10435 )
10436 .unwrap();
10437 assert_eq!(replies[0].header.ty, FrameType::Error);
10438 assert_eq!(
10439 parse_error(&replies[0])["code"],
10440 "swap_token_invalid",
10441 "nonce {nonce:?}"
10442 );
10443 }
10444 assert!(registry.get_candidate("aft").unwrap().is_none());
10445 assert_eq!(
10446 registry.get_module("aft").unwrap().unwrap().connection_id,
10447 ConnectionId::new(1)
10448 );
10449
10450 let admitted = handler
10453 .handle_control(
10454 ConnectionId::new(4),
10455 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
10456 )
10457 .unwrap();
10458 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
10459 assert_eq!(
10460 registry
10461 .get_candidate("aft")
10462 .unwrap()
10463 .unwrap()
10464 .connection_id,
10465 ConnectionId::new(4)
10466 );
10467 assert_eq!(
10468 registry.get_module("aft").unwrap().unwrap().connection_id,
10469 ConnectionId::new(1),
10470 "the candidate must not take the active slot"
10471 );
10472 let replayed = handler
10473 .handle_control(
10474 ConnectionId::new(5),
10475 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
10476 )
10477 .unwrap();
10478 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
10479
10480 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
10484 let squatter = handler
10485 .handle_control(
10486 ConnectionId::new(6),
10487 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
10488 )
10489 .unwrap();
10490 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
10491 assert!(
10492 registry.get_module("aft").unwrap().is_none(),
10493 "a squatter took the active slot of an id being swapped"
10494 );
10495 }
10496
10497 #[test]
10502 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
10503 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
10504
10505 let replies = handler
10506 .handle_control(
10507 ConnectionId::new(2),
10508 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
10509 )
10510 .unwrap();
10511
10512 assert_eq!(
10513 replies[0].header.ty,
10514 FrameType::HelloAck,
10515 "reserved candidate refused: {:?}",
10516 serde_json::from_slice::<Value>(&replies[0].body).ok()
10517 );
10518 assert_eq!(
10519 registry
10520 .get_candidate("vault")
10521 .unwrap()
10522 .unwrap()
10523 .connection_id,
10524 ConnectionId::new(2)
10525 );
10526 }
10527
10528 #[test]
10531 fn without_an_open_swap_the_ordinary_gates_decide() {
10532 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
10533 supervisor.close_swap("vault");
10534
10535 let candidate = handler
10536 .handle_control(
10537 ConnectionId::new(2),
10538 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
10539 )
10540 .unwrap();
10541 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
10542 let duplicate = handler
10543 .handle_control(
10544 ConnectionId::new(3),
10545 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
10546 )
10547 .unwrap();
10548 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
10549 assert!(registry.get_candidate("vault").unwrap().is_none());
10550 }
10551 }
10552}
10553
10554#[cfg(test)]
10555mod concurrency_default_exposure_tests {
10556 use super::*;
10557
10558 fn hello_body(role_json: &str) -> Vec<u8> {
10559 format!(
10560 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
10561 )
10562 .into_bytes()
10563 }
10564
10565 fn manifest_from(body: &[u8]) -> ModuleManifest {
10566 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
10567 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
10568 .expect("manifest parses")
10569 }
10570
10571 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
10572
10573 #[test]
10574 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
10575 let body = hello_body(&format!(
10576 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
10577 ));
10578 let manifest = manifest_from(&body);
10579 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
10582 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
10583 }
10584
10585 #[test]
10586 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
10587 let body = hello_body(&format!(
10588 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
10589 ));
10590 let manifest = manifest_from(&body);
10591 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
10592 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
10593 }
10594
10595 #[test]
10596 fn non_management_roles_are_never_reported() {
10597 let body = hello_body(
10598 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
10599 );
10600 let manifest = manifest_from(&body);
10601 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
10602 }
10603}