1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 path::{Path, PathBuf},
5 sync::{Arc, Mutex},
6 time::{Duration, Instant as StdInstant},
7};
8
9use serde::{Deserialize, Serialize};
10use subc_control::{
11 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
12 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
13 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
14 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
15 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
16 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
17 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
18};
19use subc_protocol::{
20 error_codes,
21 manifest::{
22 validate_hello_capability_grammar, validate_hello_self_signal_declarations,
23 CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
24 ProviderRole,
25 },
26 session::{
27 HealthReport, ModuleControlPush, ModuleControlRequest, ModuleControlRequestFromModule,
28 ModuleControlResponse, ModuleControlResponseToModule, MODULE_CONTROL_OP_HEALTH_CHECK,
29 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
30 },
31 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
32 Priority, RouteTarget, PROTOCOL_VERSION,
33};
34use tokio::time::{timeout_at, Instant};
35use tracing::{debug, info, warn};
36
37use crate::{
38 capability_requirements::{
39 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
40 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
41 RuntimeModule,
42 },
43 daemon_config::RestartRequiredSection,
44 forwarding::{
45 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
46 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
47 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
48 },
49 observability::{
50 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
51 },
52 provenance::{
53 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
54 },
55 registry::{ChannelState, ConnectionId, Registry, RegistryError},
56 router::{RouteCtx, RouterError},
57 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
58 stderr_tail::{CaptureState, TailEntry},
59 supervise::{
60 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
61 SupervisorHandle, SwapHelloAdmission,
62 },
63 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
64};
65
66pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
72
73const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
74const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
75const CAP_PING_PONG: &str = "ping_pong_v1";
76const CAP_SESSION_ATTACH: &str = "session_attach_v1";
77const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
78
79const SUBC_CONTROL_OPS: &[&str] = &[
80 ops::SERVER_DESCRIBE,
81 ops::CATALOG_LIST,
82 ops::ROUTE_OPEN,
83 ops::ROUTE_POLL,
84 ops::ROUTE_CLOSING,
85 ops::ROUTE_CLOSED,
86 ops::SUPERVISOR_LIST,
87 ops::SUPERVISOR_RESTART,
88 ops::SUPERVISOR_SWAP,
89 ops::SUPERVISOR_RELOAD,
90 ops::SUPERVISOR_RESCAN,
91 ops::SUPERVISOR_RELEASE_RESERVED,
92 ops::SUPERVISOR_SET_ENABLED,
93 ops::SUPERVISOR_HEALTH_PROBE,
94 ops::SUPERVISOR_HEALTH,
95 ops::SUPERVISOR_STDERR_TAIL,
96 ops::SUPERVISOR_TERMINALS,
97 ops::SUPERVISOR_ROUTES,
98 ops::SUPERVISOR_PROVENANCE,
99 ops::SUPERVISOR_SPAWN_SNAPSHOT,
100 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
101];
102
103const MODULE_TO_SUBC_CONTROL_OPS: &[&str] =
104 &[MODULE_TO_SUBC_OP_CATALOG_UPDATE, "supervisor.live_roots"];
105
106const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
107
108pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
116
117pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
132
133pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
151
152const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
153const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
154
155fn reload_verdict(
156 configured: &Path,
157 spawned_from: Option<&Path>,
158 image: subc_control::RunningImageAgreement,
159) -> PendingReloadVerdict {
160 let path = match spawned_from {
161 Some(spawned_from) if configured == spawned_from => ReloadPathAgreement::Match,
162 Some(spawned_from) => ReloadPathAgreement::Mismatch {
163 configured: configured.to_path_buf(),
164 spawned_from: spawned_from.to_path_buf(),
165 },
166 None => ReloadPathAgreement::Unavailable {
167 reason: if matches!(
168 image,
169 subc_control::RunningImageAgreement::Unavailable {
170 reason: subc_control::RunningImageUnavailableReason::NotRunning
171 }
172 ) {
173 ReloadPathUnavailableReason::NotRunning
174 } else {
175 ReloadPathUnavailableReason::SpawnedPathUnavailable
176 },
177 },
178 };
179 PendingReloadVerdict { path, image }
180}
181
182#[derive(Clone)]
183struct DaemonProvenanceFacts {
184 build: DaemonBuildProvenance,
185 pid: Option<u32>,
186 started_at_ms: Option<u64>,
187 start_clock: Option<crate::clock::StartClock>,
188 executable_path: Option<PathBuf>,
189 executable_identity: Option<SpawnedFileIdentity>,
190 process_start_time: Option<u64>,
191 probe: ExecutableIdentityProbe,
192}
193
194impl Default for DaemonProvenanceFacts {
195 fn default() -> Self {
196 Self {
197 build: DaemonBuildProvenance {
198 build_git_sha: None,
199 build_lock_digest: None,
200 },
201 pid: None,
202 started_at_ms: None,
203 start_clock: None,
204 executable_path: None,
205 executable_identity: None,
206 process_start_time: None,
207 probe: ExecutableIdentityProbe::default(),
208 }
209 }
210}
211
212#[derive(Debug, Clone)]
213struct SupervisorRescanContext {
214 supervisor: Supervisor,
215 config_path: PathBuf,
216 configured_port: Option<u16>,
217 storage_config: Option<crate::daemon_config::StorageConfig>,
218 admission_facts_carrier_module_id: Option<String>,
219 admission_facts_targets: Option<Vec<String>>,
220}
221
222#[derive(Clone)]
224pub struct ControlHandler {
225 registry: Arc<Registry>,
226 forwarding: Arc<ForwardingTable>,
227 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
228 supervisor: SupervisorHandle,
229 subc_capabilities: Arc<[String]>,
230 route_bind_relay_timeout: Duration,
234 route_bind_relay_timeouts: BTreeMap<String, Duration>,
238 route_bind_breakers: RouteBindBreakers,
241 route_bind_concurrency: RouteBindConcurrency,
244 route_bind_breaker_threshold: u32,
246 route_bind_breaker_cooldown: Duration,
248 health_probe_timeout: Duration,
249 storage_config: Option<crate::daemon_config::StorageConfig>,
252 machine_id: Option<crate::machine_id::MachineId>,
256 admission_facts_carrier_module_id: Option<String>,
257 admission_facts_targets: Option<Vec<String>>,
258 rescan: Option<SupervisorRescanContext>,
259 connected_clients: ConnectedClients,
260 counters: DaemonCounters,
261 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
262 daemon_provenance: DaemonProvenanceFacts,
263 #[cfg(test)]
264 control_dispatch_delay: Option<Duration>,
265 #[cfg(test)]
266 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
267}
268
269impl fmt::Debug for ControlHandler {
270 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
271 f.debug_struct("ControlHandler")
272 .field("registry", &self.registry)
273 .field("forwarding", &self.forwarding)
274 .field("process_liveness", &self.process_liveness.is_some())
275 .field("supervisor", &self.supervisor)
276 .field("subc_capabilities", &self.subc_capabilities)
277 .finish()
278 }
279}
280
281struct RouteOpenRequest {
282 target: RouteTarget,
283 identity: BindIdentity,
284 consumer_identity: Option<ConsumerIdentity>,
285 consumer_capabilities: Option<Vec<String>>,
286 admission_facts: Option<serde_json::Value>,
287}
288
289struct RouteBindReservationGuard {
290 forwarding: Arc<ForwardingTable>,
291 endpoint: ModuleEndpointId,
292 relay_corr: u64,
293 armed: bool,
294}
295
296struct ModuleControlRpcGuard {
297 forwarding: Arc<ForwardingTable>,
298 endpoint: ModuleEndpointId,
299 corr: u64,
300 armed: bool,
301}
302
303impl ModuleControlRpcGuard {
304 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
305 Self {
306 forwarding,
307 endpoint,
308 corr,
309 armed: true,
310 }
311 }
312
313 fn disarm(&mut self) {
314 self.armed = false;
315 }
316}
317
318impl Drop for ModuleControlRpcGuard {
319 fn drop(&mut self) {
320 if self.armed {
321 let _ = self
322 .forwarding
323 .cancel_module_control_rpc(self.endpoint, self.corr);
324 }
325 }
326}
327
328impl RouteBindReservationGuard {
329 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
330 Self {
331 forwarding,
332 endpoint,
333 relay_corr,
334 armed: true,
335 }
336 }
337
338 fn release_and_disarm(&mut self) {
339 if !self.armed {
340 return;
341 }
342 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
343 self.endpoint,
344 self.relay_corr,
345 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
346 ) {
347 send_goodbye_target_best_effort(
348 &self.forwarding.counters(),
349 &target,
350 "canceled route.bind",
351 );
352 }
353 self.armed = false;
354 }
355
356 fn disarm(&mut self) {
357 self.armed = false;
358 }
359}
360
361impl Drop for RouteBindReservationGuard {
362 fn drop(&mut self) {
363 self.release_and_disarm();
364 }
365}
366
367#[derive(Debug, Clone, Default)]
397pub(crate) struct RouteBindBreakers {
398 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
399}
400
401#[derive(Debug, Clone, Default)]
402pub(crate) struct RouteBindConcurrency {
403 modules: Arc<Mutex<HashMap<String, usize>>>,
404}
405
406struct RouteBindConcurrencyGuard {
407 concurrency: RouteBindConcurrency,
408 module_id: String,
409}
410
411impl RouteBindConcurrency {
412 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
415 let mut modules = self
416 .modules
417 .lock()
418 .expect("route.bind concurrency mutex poisoned");
419 let in_flight = modules.entry(module_id.to_string()).or_default();
420 if *in_flight >= limit {
421 return Err(*in_flight);
422 }
423 *in_flight += 1;
424 Ok(RouteBindConcurrencyGuard {
425 concurrency: self.clone(),
426 module_id: module_id.to_string(),
427 })
428 }
429}
430
431impl Drop for RouteBindConcurrencyGuard {
432 fn drop(&mut self) {
433 let mut modules = self
434 .concurrency
435 .modules
436 .lock()
437 .expect("route.bind concurrency mutex poisoned");
438 let remove = {
439 let in_flight = modules
440 .get_mut(&self.module_id)
441 .expect("admitted route.bind has a concurrency entry");
442 *in_flight -= 1;
443 *in_flight == 0
444 };
445 if remove {
446 modules.remove(&self.module_id);
447 }
448 }
449}
450
451#[derive(Debug, Default)]
452struct ModuleBreakerState {
453 consecutive_timeouts: u32,
455 cooldown_until: Option<Instant>,
458 probe_in_flight: bool,
463}
464
465enum RouteBindAdmission<'a> {
467 Admitted {
468 guard: RouteBindBreakerGuard<'a>,
469 probe: bool,
472 },
473 Refused {
474 consecutive_timeouts: u32,
475 retry_in: Duration,
479 probe_in_flight: bool,
480 },
481}
482
483struct RouteBindBreakerGuard<'a> {
490 breakers: RouteBindBreakers,
491 module_id: &'a str,
492 settled: bool,
493}
494
495impl RouteBindBreakerGuard<'_> {
496 fn record_accepted(&mut self) -> bool {
500 self.settled = true;
501 self.breakers.record_accepted(self.module_id)
502 }
503
504 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
507 self.settled = true;
508 self.breakers
509 .record_timeout(self.module_id, threshold, cooldown)
510 }
511
512 fn record_inconclusive(&mut self) {
521 self.settled = true;
522 self.breakers.record_inconclusive(self.module_id);
523 }
524}
525
526impl Drop for RouteBindBreakerGuard<'_> {
527 fn drop(&mut self) {
528 if !self.settled {
529 self.breakers.record_inconclusive(self.module_id);
530 }
531 }
532}
533
534struct BreakerOpened {
538 consecutive_timeouts: u32,
539 reopened_after_probe: bool,
542}
543
544impl RouteBindBreakers {
545 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
546 self.modules
547 .lock()
548 .expect("route.bind breaker mutex poisoned")
549 }
550
551 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
554 let admitted = |probe| RouteBindAdmission::Admitted {
555 guard: RouteBindBreakerGuard {
556 breakers: self.clone(),
557 module_id,
558 settled: false,
559 },
560 probe,
561 };
562
563 let mut modules = self.lock();
564 let Some(state) = modules.get_mut(module_id) else {
565 return admitted(false);
566 };
567 let Some(cooldown_until) = state.cooldown_until else {
568 return admitted(false);
569 };
570 if state.probe_in_flight {
571 return RouteBindAdmission::Refused {
572 consecutive_timeouts: state.consecutive_timeouts,
573 retry_in: Duration::ZERO,
574 probe_in_flight: true,
575 };
576 }
577 let now = Instant::now();
578 if now < cooldown_until {
579 return RouteBindAdmission::Refused {
580 consecutive_timeouts: state.consecutive_timeouts,
581 retry_in: cooldown_until - now,
582 probe_in_flight: false,
583 };
584 }
585 state.probe_in_flight = true;
586 admitted(true)
587 }
588
589 fn record_accepted(&self, module_id: &str) -> bool {
590 self.lock()
591 .remove(module_id)
592 .is_some_and(|state| state.cooldown_until.is_some())
593 }
594
595 fn record_timeout(
596 &self,
597 module_id: &str,
598 threshold: u32,
599 cooldown: Duration,
600 ) -> Option<BreakerOpened> {
601 let mut modules = self.lock();
602 let state = modules.entry(module_id.to_string()).or_default();
603 let was_open = state.cooldown_until.is_some();
604 let was_probe = state.probe_in_flight;
605 state.probe_in_flight = false;
606 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
607 if state.consecutive_timeouts < threshold {
608 return None;
609 }
610 state.cooldown_until = Some(Instant::now() + cooldown);
611 Some(BreakerOpened {
612 consecutive_timeouts: state.consecutive_timeouts,
613 reopened_after_probe: was_open && was_probe,
614 })
615 }
616
617 fn record_inconclusive(&self, module_id: &str) {
618 if let Some(state) = self.lock().get_mut(module_id) {
619 state.probe_in_flight = false;
620 }
621 }
622
623 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
638 self.lock()
639 .remove(module_id)
640 .map(|state| state.consecutive_timeouts)
641 .filter(|discarded| *discarded > 0)
642 }
643
644 fn open_snapshot(&self) -> Option<serde_json::Value> {
651 let now = Instant::now();
652 let modules = self.lock();
653 let open = modules
654 .iter()
655 .filter_map(|(module_id, state)| {
656 let cooldown_until = state.cooldown_until?;
657 Some((
658 module_id.clone(),
659 serde_json::json!({
660 "consecutive_timeouts": state.consecutive_timeouts,
661 "cooldown_remaining_ms":
662 cooldown_until.saturating_duration_since(now).as_millis() as u64,
663 "probe_in_flight": state.probe_in_flight,
664 }),
665 ))
666 })
667 .collect::<serde_json::Map<String, serde_json::Value>>();
668 (!open.is_empty()).then_some(serde_json::Value::Object(open))
669 }
670}
671
672impl ControlHandler {
673 pub fn new(registry: Arc<Registry>) -> Self {
674 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
675 }
676
677 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
678 let counters = forwarding.counters();
679 let route_bind_breakers = forwarding.route_bind_breakers();
683 let route_bind_concurrency = forwarding.route_bind_concurrency();
684 Self {
685 registry,
686 forwarding,
687 process_liveness: None,
688 supervisor: SupervisorHandle::new(),
689 subc_capabilities: Arc::from([
690 CAP_MANIFEST_REGISTRATION.to_string(),
691 CAP_CHANNEL_LIFECYCLE.to_string(),
692 CAP_PING_PONG.to_string(),
693 CAP_SESSION_ATTACH.to_string(),
694 CAP_ADMISSION_FACTS_RELAY.to_string(),
695 ]),
696 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
697 route_bind_relay_timeouts: BTreeMap::new(),
698 route_bind_breakers,
699 route_bind_concurrency,
700 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
701 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
702 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
703 storage_config: None,
704 machine_id: None,
705 admission_facts_carrier_module_id: None,
706 admission_facts_targets: None,
707 rescan: None,
708 connected_clients: ConnectedClients::new(),
709 counters,
710 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
711 daemon_provenance: DaemonProvenanceFacts::default(),
712 #[cfg(test)]
713 control_dispatch_delay: None,
714 #[cfg(test)]
715 provenance_probe_override: None,
716 }
717 }
718
719 pub fn with_storage_config(
722 mut self,
723 storage_config: Option<crate::daemon_config::StorageConfig>,
724 ) -> Self {
725 self.storage_config = storage_config;
726 self
727 }
728
729 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
732 self.machine_id = machine_id;
733 self
734 }
735
736 pub fn with_admission_facts_config(
740 mut self,
741 carrier_module_id: Option<String>,
742 targets: Option<Vec<String>>,
743 ) -> Self {
744 self.admission_facts_carrier_module_id = carrier_module_id;
745 self.admission_facts_targets = targets;
746 self
747 }
748
749 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
752 self.route_bind_relay_timeout = timeout;
753 self
754 }
755
756 pub fn with_route_bind_relay_timeouts(
762 mut self,
763 timeouts: impl IntoIterator<Item = (String, Duration)>,
764 ) -> Self {
765 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
766 self
767 }
768
769 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
775 self.route_bind_relay_timeouts
776 .get(module_id)
777 .copied()
778 .unwrap_or(self.route_bind_relay_timeout)
779 }
780
781 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
789 self.route_bind_breaker_threshold = threshold.max(1);
790 self.route_bind_breaker_cooldown = cooldown;
791 self
792 }
793
794 #[cfg(test)]
795 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
796 self.health_probe_timeout = timeout;
797 self
798 }
799
800 #[cfg(test)]
801 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
802 self.control_dispatch_delay = Some(delay);
803 self
804 }
805
806 pub fn with_process_liveness(
807 mut self,
808 process_liveness: Arc<dyn ModuleProcessLiveness>,
809 ) -> Self {
810 self.process_liveness = Some(process_liveness);
811 self
812 }
813
814 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
815 self.supervisor = supervisor;
816 self
817 }
818
819 pub fn with_daemon_provenance(
820 mut self,
821 pid: u32,
822 started_at_ms: u64,
823 executable_path: Option<PathBuf>,
824 build_git_sha: Option<String>,
825 build_lock_digest: Option<String>,
826 ) -> Self {
827 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
828 let process_start_time = process_start_time(pid);
829 self.daemon_provenance = DaemonProvenanceFacts {
830 build: DaemonBuildProvenance {
831 build_git_sha,
832 build_lock_digest,
833 },
834 pid: Some(pid),
835 started_at_ms: Some(started_at_ms),
836 start_clock: None,
837 executable_path,
838 executable_identity,
839 process_start_time,
840 probe: ExecutableIdentityProbe::default(),
841 };
842 self
843 }
844
845 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
846 self.daemon_provenance.start_clock = Some(clock);
847 self
848 }
849
850 #[cfg(test)]
851 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
852 self.provenance_probe_override = Some(result);
853 self
854 }
855
856 pub fn with_capability_config(
860 self,
861 modules: impl IntoIterator<Item = (String, bool)>,
862 reserved_capabilities: BTreeMap<String, String>,
863 ) -> Self {
864 self.capability_evaluator
865 .configure(modules, reserved_capabilities);
866 self
867 }
868
869 pub fn with_supervisor_rescan(
870 mut self,
871 supervisor: Supervisor,
872 config_path: impl Into<PathBuf>,
873 configured_port: Option<u16>,
874 ) -> Self {
875 self.rescan = Some(SupervisorRescanContext {
876 supervisor,
877 config_path: config_path.into(),
878 configured_port,
879 storage_config: self.storage_config.clone(),
880 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
881 admission_facts_targets: self.admission_facts_targets.clone(),
882 });
883 self
884 }
885
886 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
887 self.connected_clients = connected_clients;
888 self
889 }
890
891 pub fn forwarding(&self) -> Arc<ForwardingTable> {
892 Arc::clone(&self.forwarding)
893 }
894
895 pub(crate) fn counters(&self) -> DaemonCounters {
896 self.counters.clone()
897 }
898
899 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
902 tokio::spawn(async move {
903 loop {
904 self.capability_evaluator
905 .wait_for_change_or_deadline()
906 .await;
907 self.refresh_capability_requirements();
908 }
909 });
910 }
911
912 fn runtime_capability_snapshot(
913 &self,
914 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
915 let runtime = self
916 .supervisor
917 .list()
918 .into_iter()
919 .map(|module| {
920 let status = module.status().map_err(|err| {
921 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
922 })?;
923 Ok(RuntimeModule {
924 module_id: status.module_id,
925 state: status.state,
926 enabled: status.enabled,
927 })
928 })
929 .collect::<Result<Vec<_>, RouterError>>()?;
930 let (_, registrations) = self.registry.list_modules().map_err(|err| {
931 RouterError::backend(
932 0,
933 0,
934 format!("failed to list capability registrations: {err}"),
935 )
936 })?;
937 let registrations = registrations
938 .into_iter()
939 .map(|registration| RegisteredModule {
940 module_id: registration.manifest.module_id,
941 module_version: registration.manifest.module_version,
942 capabilities: registration.manifest.capabilities,
943 })
944 .collect();
945 Ok((runtime, registrations))
946 }
947
948 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
955 let cached_registration = RegisteredModule {
956 module_id: registration.manifest.module_id.clone(),
957 module_version: registration.manifest.module_version.clone(),
958 capabilities: registration.manifest.capabilities.clone(),
959 };
960 if self.capability_evaluator.record_hello(&cached_registration) {
961 warn!(
962 module_id = %cached_registration.module_id,
963 "capability claims drifted from the cached manifest"
964 );
965 }
966 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
967 self.enforce_capability_denies();
968 }
969 self.refresh_capability_requirements();
970 }
971
972 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
976 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
977 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
978 self.supervisor.set_swap_promotion_observer(observer);
979 }
980
981 pub fn refresh_capability_requirements(&self) {
982 match self.runtime_capability_snapshot() {
983 Ok((runtime, registrations)) => {
984 log_requirement_events(
985 self.capability_evaluator
986 .evaluate_now(&runtime, ®istrations),
987 );
988 }
989 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
990 }
991 }
992
993 fn enforce_capability_denies(&self) {
997 let (_, registrations) = match self.registry.list_modules() {
998 Ok(snapshot) => snapshot,
999 Err(err) => {
1000 warn!(error = %err, "failed to read registrations for capability deny census");
1001 return;
1002 }
1003 };
1004 let manifests = registrations
1005 .into_iter()
1006 .map(|registration| {
1007 (
1008 registration.manifest.module_id.clone(),
1009 registration.manifest,
1010 )
1011 })
1012 .collect::<BTreeMap<_, _>>();
1013 let census = match self.forwarding.route_census(None) {
1014 Ok(census) => census,
1015 Err(err) => {
1016 warn!(error = %err, "failed to read route census for capability deny enforcement");
1017 return;
1018 }
1019 };
1020
1021 for (target_module_id, routes) in census {
1022 let Some(target_manifest) = manifests.get(&target_module_id) else {
1023 continue;
1024 };
1025 let mut closed_routes = Vec::new();
1026 let mut module_goodbyes = Vec::new();
1027 for route in routes {
1028 let Principal::Reserved {
1029 module_id: opening_module_id,
1030 } = &route.principal
1031 else {
1032 continue;
1033 };
1034 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1035 continue;
1036 };
1037 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1038 continue;
1039 };
1040
1041 match self.forwarding.release_client_route(
1042 route.goodbye_target.connection_id,
1043 route.goodbye_target.channel,
1044 route.goodbye_target.epoch,
1045 ) {
1046 Ok(RouteRelease::Removed(module_goodbye)) => {
1047 warn!(
1048 opening_module_id,
1049 target_module_id,
1050 capability,
1051 "force-closing route because an attested capability deny edge now matches"
1052 );
1053 closed_routes.push(route);
1054 module_goodbyes.push(module_goodbye);
1055 }
1056 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1057 Err(err) => warn!(
1058 opening_module_id,
1059 target_module_id,
1060 capability,
1061 error = %err,
1062 "failed to force-close capability-denied route"
1063 ),
1064 }
1065 }
1066
1067 if closed_routes.is_empty() {
1068 continue;
1069 }
1070 send_route_control_pushes(
1071 &self.forwarding,
1072 closed_routes,
1073 ClientControlPush::RouteClosed {
1074 module_id: target_module_id,
1075 reason: RouteCloseReason::CapabilityDenied,
1076 drained: false,
1077 abandoned: 0,
1078 excluded_subscriptions: 0,
1079 terminal: Some(false),
1080 },
1081 );
1082 self.emit_route_goodbyes(module_goodbyes);
1083 }
1084 }
1085
1086 fn not_ready_reason(
1092 &self,
1093 registration: &crate::registry::ModuleRegistration,
1094 ) -> Option<NotReadyReason> {
1095 if !registration.ready {
1096 return Some(NotReadyReason {
1097 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1098 capability: None,
1099 });
1100 }
1101 self.first_unprovided_required_capability(registration)
1102 .map(|capability| NotReadyReason {
1103 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1104 capability: Some(capability),
1105 })
1106 }
1107
1108 fn first_unprovided_required_capability(
1126 &self,
1127 registration: &crate::registry::ModuleRegistration,
1128 ) -> Option<String> {
1129 let required = registration
1130 .manifest
1131 .capabilities
1132 .iter()
1133 .flat_map(|declarations| declarations.requires.iter())
1134 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1135 .map(|requirement| requirement.capability.as_str())
1136 .collect::<BTreeSet<_>>();
1137 if required.is_empty() {
1138 return None;
1139 }
1140 let module_id = registration.manifest.module_id.as_str();
1141 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1142 if required
1143 .iter()
1144 .any(|capability| verdict(capability).is_none())
1145 {
1146 self.refresh_capability_requirements();
1147 }
1148 required
1149 .into_iter()
1150 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1151 .map(str::to_string)
1152 }
1153
1154 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1155 self.capability_evaluator
1156 .statuses()
1157 .into_iter()
1158 .map(capability_requirement_status)
1159 .collect()
1160 }
1161
1162 fn deregister_connection(
1169 &self,
1170 connection_id: ConnectionId,
1171 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1172 self.registry.deregister_connection(connection_id)
1173 }
1174
1175 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1176 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1177 return None;
1178 }
1179 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1180 parse_client_control_request(&frame.body)
1181 else {
1182 return None;
1183 };
1184 Some(target_module_id(&target).to_string())
1185 }
1186
1187 pub(crate) fn route_open_capacity_refusal(
1188 &self,
1189 ctx: &RouteCtx,
1190 frame: &Frame,
1191 target_module_id: &str,
1192 limit: usize,
1193 ) -> Result<Frame, RouterError> {
1194 self.route_open_admission_refusal_frame(
1195 ctx,
1196 frame,
1197 target_module_id,
1198 format!(
1199 "connection already has {limit} route.open binds in flight; retry after one settles"
1200 ),
1201 )
1202 }
1203
1204 fn route_open_admission_refusal_frame(
1212 &self,
1213 ctx: &RouteCtx,
1214 frame: &Frame,
1215 target_module_id: &str,
1216 message: impl Into<String>,
1217 ) -> Result<Frame, RouterError> {
1218 self.route_open_refusal_frame(
1219 ctx,
1220 frame,
1221 target_module_id,
1222 "open_admission_full",
1223 error_codes::TARGET_UNAVAILABLE,
1224 message,
1225 )
1226 }
1227
1228 #[cfg(test)]
1234 pub fn handle_control(
1235 &self,
1236 connection_id: ConnectionId,
1237 frame: Frame,
1238 ) -> Result<Vec<Frame>, RouterError> {
1239 match frame.header.ty {
1240 FrameType::Ping => Ok(vec![pong(&frame)?]),
1241 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1242 FrameType::Goodbye => self.handle_goodbye(connection_id),
1243 ty => Ok(vec![control_error_frame(
1244 &frame,
1245 "unsupported_control_frame",
1246 format!("unsupported channel-0 frame {ty:?}"),
1247 )?]),
1248 }
1249 }
1250
1251 pub async fn handle_control_frame(
1252 &self,
1253 ctx: &RouteCtx,
1254 frame: Frame,
1255 ) -> Result<Vec<Frame>, RouterError> {
1256 self.handle_control_frame_timed(ctx, frame, None).await
1257 }
1258
1259 pub(crate) async fn handle_control_frame_timed(
1260 &self,
1261 ctx: &RouteCtx,
1262 frame: Frame,
1263 dispatch_started_at: Option<StdInstant>,
1264 ) -> Result<Vec<Frame>, RouterError> {
1265 match frame.header.ty {
1266 FrameType::Ping => Ok(vec![pong(&frame)?]),
1267 FrameType::Hello => {
1268 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1269 }
1270 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1271 FrameType::Cancel => {
1272 if self
1273 .supervisor
1274 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1275 {
1276 Ok(Vec::new())
1277 } else {
1278 Ok(vec![control_error_frame(
1279 &frame,
1280 "unknown_subscription",
1281 "no supervisor spawn subscription has this correlation id",
1282 )?])
1283 }
1284 }
1285 FrameType::Request => {
1286 if self
1287 .forwarding
1288 .module_endpoint_for_connection(ctx.connection_id)
1289 .map_err(RouterError::Forwarding)?
1290 .is_some()
1291 {
1292 if !is_known_module_request_op(&frame.body) {
1293 return Ok(vec![control_error_frame(
1294 &frame,
1295 "unsupported_control_frame",
1296 "module-originated channel-0 REQUEST is not supported",
1297 )?]);
1298 }
1299 let request = match parse_module_control_request_from_module(&frame.body) {
1300 Ok(request) => request,
1301 Err((err, ControlRequestBodyError::UnknownOp)) => {
1302 return Ok(vec![control_error_frame(
1303 &frame,
1304 "unsupported_control_frame",
1305 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1306 )?])
1307 }
1308 Err((err, ControlRequestBodyError::InvalidBody)) => {
1309 return Ok(vec![control_error_frame(
1310 &frame,
1311 "invalid_control_body",
1312 format!("malformed module control body: {err}"),
1313 )?])
1314 }
1315 };
1316 let op = module_control_request_op(&request);
1317 let corr = frame.header.corr;
1318 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1319 let result =
1320 self.handle_module_control_request(ctx.connection_id, frame, request);
1321 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1322 return result;
1323 }
1324
1325 if is_known_module_request_op(&frame.body) {
1326 return Ok(vec![control_error_frame(
1327 &frame,
1328 "not_registered",
1329 "catalog.update requires an active module registration owned by this connection",
1330 )?]);
1331 }
1332
1333 let request = match parse_client_control_request(&frame.body) {
1334 Ok(request) => request,
1335 Err((err, ControlRequestBodyError::UnknownOp)) => {
1336 return Ok(vec![control_error_frame(
1337 &frame,
1338 "unknown_control_op",
1339 format!("unknown client control op: {err}"),
1340 )?])
1341 }
1342 Err((err, ControlRequestBodyError::InvalidBody)) => {
1343 return Ok(vec![control_error_frame(
1344 &frame,
1345 "invalid_control_body",
1346 format!("malformed client control body: {err}"),
1347 )?])
1348 }
1349 };
1350 let op = client_control_request_op(&request);
1351 let corr = frame.header.corr;
1352 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1353 #[cfg(test)]
1354 if let Some(delay) = self.control_dispatch_delay {
1355 tokio::time::sleep(delay).await;
1356 }
1357 let result = self
1358 .handle_client_control_request(ctx, frame, request)
1359 .await;
1360 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1361 result
1362 }
1363 FrameType::Push => {
1364 let Some(endpoint) = self
1365 .forwarding
1366 .module_endpoint_for_connection(ctx.connection_id)
1367 .map_err(RouterError::Forwarding)?
1368 else {
1369 return Ok(vec![control_error_frame(
1370 &frame,
1371 "unsupported_control_frame",
1372 "client-originated channel-0 PUSH is not supported",
1373 )?]);
1374 };
1375 self.handle_status_update(endpoint, frame)
1376 }
1377 FrameType::Response | FrameType::Error
1378 if self
1379 .forwarding
1380 .module_endpoint_for_connection(ctx.connection_id)
1381 .map_err(RouterError::Forwarding)?
1382 .is_some() =>
1383 {
1384 self.handle_module_relay_response(ctx.connection_id, frame)
1385 }
1386 ty => Ok(vec![control_error_frame(
1387 &frame,
1388 "unsupported_control_frame",
1389 format!("unsupported channel-0 frame {ty:?}"),
1390 )?]),
1391 }
1392 }
1393
1394 pub fn cleanup_connection(
1395 &self,
1396 connection_id: ConnectionId,
1397 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1398 let crash_closed = self
1399 .registry
1400 .get_module_by_connection(connection_id)?
1401 .and_then(|registration| {
1402 self.forwarding
1403 .module_endpoint_for_connection(connection_id)
1404 .ok()
1405 .flatten()
1406 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1407 .map(|routes| (registration.manifest.module_id, routes))
1408 });
1409 if let Some((module_id, routes)) = crash_closed {
1410 let terminal = match self.supervisor.get(&module_id) {
1411 None => false,
1412 Some(module) => match module.will_recover_after_connection_loss() {
1413 Ok(will_recover) => !will_recover,
1414 Err(err) => {
1415 warn!(
1416 %module_id,
1417 error = %err,
1418 "failed to read crash recovery verdict; reporting non-terminal conservatively"
1419 );
1420 false
1421 }
1422 },
1423 };
1424 let reason = match self.forwarding.is_daemon_draining() {
1428 Ok(true) => RouteCloseReason::Restart,
1429 Ok(false) => RouteCloseReason::Crash,
1430 Err(err) => {
1431 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
1432 RouteCloseReason::Crash
1433 }
1434 };
1435 send_route_control_pushes(
1436 &self.forwarding,
1437 routes,
1438 ClientControlPush::RouteClosed {
1439 module_id,
1440 reason,
1441 drained: false,
1442 abandoned: 0,
1443 excluded_subscriptions: 0,
1444 terminal: Some(terminal),
1445 },
1446 );
1447 }
1448 let registrations = self.deregister_connection(connection_id);
1449 if let Ok(released_routes) = self.forwarding.cleanup_connection(connection_id) {
1450 self.emit_route_goodbyes(released_routes);
1451 }
1452 if matches!(®istrations, Ok(r) if !r.is_empty()) {
1456 crate::supervise::notify_registration_release();
1457 self.capability_evaluator.wake_deadline_loop();
1458 self.refresh_capability_requirements();
1459 }
1460 self.supervisor.remove_spawn_subscribers(connection_id);
1461 registrations
1462 }
1463
1464 pub(crate) fn handle_route_goodbye(
1465 &self,
1466 connection_id: ConnectionId,
1467 route_channel: u16,
1468 route_epoch: u32,
1469 ) -> Result<bool, RouterError> {
1470 debug!(
1471 connection_id = connection_id.get(),
1472 route_channel, route_epoch, "handling route GOODBYE"
1473 );
1474 let RouteRelease::Removed(released_route) = self
1475 .forwarding
1476 .release_client_route(connection_id, route_channel, route_epoch)
1477 .map_err(RouterError::Forwarding)?
1478 else {
1479 return Ok(false);
1480 };
1481 self.emit_route_goodbyes(vec![released_route]);
1482 Ok(true)
1483 }
1484
1485 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
1486 for released in released_routes {
1487 let frame = match Frame::build_with_version(
1488 released.negotiated_ver,
1489 FrameType::Goodbye,
1490 control_flags(),
1491 released.channel,
1492 released.epoch,
1493 0,
1494 Vec::new(),
1495 ) {
1496 Ok(frame) => frame,
1497 Err(err) => {
1498 warn!(
1499 route_channel = released.channel,
1500 error = %err,
1501 "failed to build route GOODBYE frame"
1502 );
1503 continue;
1504 }
1505 };
1506 if !released.close_on_delivery_failure() {
1507 crate::forwarding::send_module_route_goodbye(
1508 &self.counters,
1509 &released.sink,
1510 frame,
1511 released.module_id.as_deref(),
1512 "client route released",
1513 );
1514 continue;
1515 }
1516 if let Err(err) = released.sink.try_send(frame) {
1517 warn!(
1518 target_connection_id = released.connection_id.get(),
1519 route_channel = released.channel,
1520 error = %err,
1521 "route GOODBYE was not delivered to client; closing target connection"
1522 );
1523 if self
1524 .forwarding
1525 .escalate_client_delivery_failure(
1526 released.connection_id,
1527 released.channel,
1528 released.epoch,
1529 CloseReason::new(
1530 "route_goodbye_delivery_failed",
1531 format!(
1532 "failed to enqueue route GOODBYE for channel {}: {err}",
1533 released.channel
1534 ),
1535 ),
1536 crate::forwarding::UndeliveredFrame {
1537 module_id: released.module_id.as_deref(),
1538 sink: &released.sink,
1539 },
1540 )
1541 .unwrap_or(false)
1542 {
1543 self.counters.increment_goodbye_relay_client_failed();
1544 }
1545 }
1546 }
1547 }
1548
1549 fn send_abandoned_route_bind_goodbye(
1561 &self,
1562 module_sink: &crate::FrameSink,
1563 negotiated_ver: u8,
1564 module_channel: u16,
1565 module_epoch: u32,
1566 ) {
1567 let frame = match Frame::build_with_version(
1568 negotiated_ver,
1569 FrameType::Goodbye,
1570 control_flags(),
1571 module_channel,
1572 module_epoch,
1573 0,
1574 Vec::new(),
1575 ) {
1576 Ok(frame) => frame,
1577 Err(err) => {
1578 warn!(
1579 route_channel = module_channel,
1580 error = %err,
1581 "failed to build GOODBYE for abandoned route.bind"
1582 );
1583 return;
1584 }
1585 };
1586 crate::forwarding::send_module_route_goodbye(
1587 &self.counters,
1588 module_sink,
1589 frame,
1590 None,
1591 "abandoned route.bind",
1592 );
1593 }
1594
1595 fn handle_hello(
1596 &self,
1597 connection_id: ConnectionId,
1598 sink: Option<crate::FrameSink>,
1599 frame: Frame,
1600 ) -> Result<Vec<Frame>, RouterError> {
1601 debug!(
1602 connection_id = connection_id.get(),
1603 corr = frame.header.corr,
1604 "handling HELLO"
1605 );
1606 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
1607 Ok(value) => value,
1608 Err(err) => {
1609 return Ok(vec![control_error_frame(
1610 &frame,
1611 "invalid_hello",
1612 format!("malformed HELLO body: {err}"),
1613 )?])
1614 }
1615 };
1616 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
1617 return Ok(vec![control_error_frame(
1618 &frame,
1619 "invalid_capability_grammar",
1620 err.to_string(),
1621 )?]);
1622 }
1623 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
1624 return Ok(vec![control_error_frame(
1625 &frame,
1626 "invalid_manifest",
1627 err.to_string(),
1628 )?]);
1629 }
1630 if let Some(provenance) = hello_value
1631 .get("manifest")
1632 .and_then(|manifest| manifest.get("provenance"))
1633 {
1634 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
1635 return Ok(vec![control_error_frame(
1636 &frame,
1637 "invalid_manifest",
1638 format!("malformed manifest provenance: {err}"),
1639 )?]);
1640 }
1641 }
1642 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
1643 Ok(hello) => hello,
1644 Err(err) => {
1645 return Ok(vec![control_error_frame(
1646 &frame,
1647 "invalid_hello",
1648 format!("malformed HELLO body: {err}"),
1649 )?])
1650 }
1651 };
1652
1653 if hello.protocol_ver != hello.manifest.protocol_ver {
1654 return Ok(vec![control_error_frame(
1655 &frame,
1656 "invalid_manifest",
1657 format!(
1658 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
1659 hello.protocol_ver, hello.manifest.protocol_ver
1660 ),
1661 )?]);
1662 }
1663
1664 if hello.manifest.module_id.trim().is_empty() {
1665 return Ok(vec![control_error_frame(
1666 &frame,
1667 "invalid_manifest",
1668 "manifest module_id must not be empty",
1669 )?]);
1670 }
1671
1672 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
1673 Ok(negotiated_ver) => negotiated_ver,
1674 Err(message) => {
1675 return Ok(vec![control_error_frame(
1676 &frame,
1677 "version_unsupported",
1678 message,
1679 )?])
1680 }
1681 };
1682
1683 let swap_admission = self
1691 .supervisor
1692 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
1693 if swap_admission == SwapHelloAdmission::Refused {
1694 warn!(
1695 module_id = %hello.manifest.module_id,
1696 connection_id = connection_id.get(),
1697 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
1698 );
1699 return Ok(vec![control_error_frame(
1700 &frame,
1701 "swap_token_invalid",
1702 format!(
1703 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
1704 hello.manifest.module_id
1705 ),
1706 )?]);
1707 }
1708 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
1709
1710 if let Some(rejection) = (!swap_candidate)
1718 .then(|| {
1719 self.supervisor.reserved_hello_rejection(
1720 &hello.manifest.module_id,
1721 hello.launch_nonce.as_deref(),
1722 )
1723 })
1724 .flatten()
1725 {
1726 let message = match rejection {
1727 ReservedHelloRejection::Exact { module_id } => format!(
1728 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
1729 ),
1730 ReservedHelloRejection::Prefix {
1731 prefix,
1732 owner_module_id,
1733 } => format!(
1734 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
1735 hello.manifest.module_id
1736 ),
1737 };
1738 return Ok(vec![control_error_frame(
1739 &frame,
1740 "reserved_module",
1741 message,
1742 )?]);
1743 }
1744
1745 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
1746 &hello.manifest.module_id,
1747 hello.manifest.capabilities.as_ref(),
1748 );
1749 if let Some(refusal) = reserved_capability_refusals.first() {
1750 let capability = refusal.capability.clone();
1751 let bound_module = refusal.claimants[0].clone();
1752 log_duplicate_claim_events(reserved_capability_refusals);
1753 return Ok(vec![control_error_frame(
1754 &frame,
1755 "reserved_capability",
1756 format!(
1757 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
1758 capability, bound_module, hello.manifest.module_id
1759 ),
1760 )?]);
1761 }
1762
1763 if self
1766 .forwarding
1767 .connection_has_client_routes(connection_id)
1768 .map_err(RouterError::Forwarding)?
1769 {
1770 return Ok(vec![control_error_frame(
1771 &frame,
1772 "invalid_hello",
1773 "connection has open client routes and cannot also register as a module",
1774 )?]);
1775 }
1776
1777 let control_ops = effective_module_control_ops(hello.control_ops);
1778 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
1781 if swap_candidate {
1782 return self.register_swap_candidate(
1783 connection_id,
1784 sink,
1785 &frame,
1786 hello.manifest,
1787 negotiated_ver,
1788 control_ops,
1789 hello_ack,
1790 );
1791 }
1792 let registration = match self.registry.register_with_control_ops(
1793 hello.manifest,
1794 negotiated_ver,
1795 connection_id,
1796 control_ops,
1797 ) {
1798 Ok(registration) => registration,
1799 Err(RegistryError::DuplicateModuleId { module_id }) => {
1800 return Ok(vec![control_error_frame(
1801 &frame,
1802 "duplicate_module_id",
1803 format!(
1804 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
1805 ),
1806 )?])
1807 }
1808 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
1809 return Ok(vec![control_error_frame(
1810 &frame,
1811 "invalid_module_id",
1812 err.to_string(),
1813 )?])
1814 }
1815 Err(err) => {
1816 return Ok(vec![control_error_frame(
1817 &frame,
1818 "registry_error",
1819 err.to_string(),
1820 )?])
1821 }
1822 };
1823
1824 let reply = if let Some(sink) = sink {
1825 let concurrency = manifest_concurrency(®istration.manifest);
1841 if let Err(err) = self.forwarding.register_module_connection_acked(
1842 connection_id,
1843 registration.manifest.module_id.clone(),
1844 negotiated_ver,
1845 concurrency,
1846 sink,
1847 hello_ack,
1848 ) {
1849 if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
1853 crate::supervise::notify_registration_release();
1854 }
1855 return Ok(vec![control_error_frame(
1856 &frame,
1857 forwarding_error_code(&err),
1858 err.to_string(),
1859 )?]);
1860 }
1861 Vec::new()
1862 } else {
1863 vec![hello_ack]
1866 };
1867
1868 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
1877 info!(
1878 module_id = %registration.manifest.module_id,
1879 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
1880 );
1881 }
1882
1883 self.apply_registration_capabilities(®istration);
1884
1885 info!(
1886 module_id = %registration.manifest.module_id,
1887 module_version = %registration.manifest.module_version,
1888 negotiated_ver,
1889 routable_provider = manifest_provides_routable_role(®istration.manifest),
1890 connection_id = connection_id.get(),
1891 "module registered"
1892 );
1893
1894 Ok(reply)
1895 }
1896
1897 #[allow(clippy::too_many_arguments)]
1907 fn register_swap_candidate(
1908 &self,
1909 connection_id: ConnectionId,
1910 sink: Option<crate::FrameSink>,
1911 frame: &Frame,
1912 manifest: ModuleManifest,
1913 negotiated_ver: u8,
1914 control_ops: Vec<String>,
1915 hello_ack: Frame,
1916 ) -> Result<Vec<Frame>, RouterError> {
1917 let module_id = manifest.module_id.clone();
1918 let registration = match self.registry.register_candidate_with_control_ops(
1919 manifest,
1920 negotiated_ver,
1921 connection_id,
1922 control_ops,
1923 ) {
1924 Ok(registration) => registration,
1925 Err(RegistryError::DuplicateModuleId { module_id }) => {
1926 return Ok(vec![control_error_frame(
1927 frame,
1928 "duplicate_module_id",
1929 format!(
1930 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
1931 ),
1932 )?])
1933 }
1934 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
1935 return Ok(vec![control_error_frame(
1936 frame,
1937 "invalid_module_id",
1938 err.to_string(),
1939 )?])
1940 }
1941 Err(err) => {
1942 return Ok(vec![control_error_frame(
1943 frame,
1944 "registry_error",
1945 err.to_string(),
1946 )?])
1947 }
1948 };
1949 let reply = if let Some(sink) = sink {
1950 let concurrency = manifest_concurrency(®istration.manifest);
1954 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
1955 connection_id,
1956 module_id.clone(),
1957 negotiated_ver,
1958 concurrency,
1959 sink,
1960 hello_ack,
1961 ) {
1962 if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
1963 crate::supervise::notify_registration_release();
1964 }
1965 return Ok(vec![control_error_frame(
1966 frame,
1967 forwarding_error_code(&err),
1968 err.to_string(),
1969 )?]);
1970 }
1971 Vec::new()
1972 } else {
1973 vec![hello_ack]
1974 };
1975 self.supervisor.mark_swap_candidate_admitted(&module_id);
1976 info!(
1977 module_id = %module_id,
1978 module_version = %registration.manifest.module_version,
1979 negotiated_ver,
1980 ready = registration.ready,
1981 connection_id = connection_id.get(),
1982 "swap candidate registered; not routable until cutover"
1983 );
1984 Ok(reply)
1985 }
1986
1987 fn build_hello_ack(
1988 &self,
1989 frame: &Frame,
1990 negotiated_ver: u8,
1991 module_id: &str,
1992 ) -> Result<Frame, RouterError> {
1993 let ack = ModuleHelloAckBody {
1994 negotiated_ver,
1995 subc_ops: module_subc_ops(),
1996 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
1997 storage: self
1998 .storage_config
1999 .as_ref()
2000 .map(|cfg| cfg.descriptor_for(module_id)),
2001 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2002 };
2003 let body = serde_json::to_vec(&ack).map_err(|err| {
2004 RouterError::backend(
2005 0,
2006 frame.header.corr,
2007 format!("failed to encode HELLO_ACK: {err}"),
2008 )
2009 })?;
2010
2011 Frame::build_with_version(
2012 negotiated_ver,
2013 FrameType::HelloAck,
2014 control_flags(),
2015 0,
2016 0,
2017 frame.header.corr,
2018 body,
2019 )
2020 .map_err(RouterError::FrameBuild)
2021 }
2022
2023 async fn handle_client_control_request(
2024 &self,
2025 ctx: &RouteCtx,
2026 frame: Frame,
2027 request: ClientControlRequest,
2028 ) -> Result<Vec<Frame>, RouterError> {
2029 match request {
2030 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2031 ClientControlRequest::CatalogList { module_id } => {
2032 self.handle_catalog_list(frame, module_id)
2033 }
2034 ClientControlRequest::RouteOpen {
2035 target,
2036 identity,
2037 consumer_identity,
2038 consumer_capabilities,
2039 admission_facts,
2040 } => {
2041 self.handle_route_open(
2042 ctx,
2043 frame,
2044 RouteOpenRequest {
2045 target,
2046 identity,
2047 consumer_identity,
2048 consumer_capabilities,
2049 admission_facts,
2050 },
2051 )
2052 .await
2053 }
2054 ClientControlRequest::RoutePoll {
2055 route_channel,
2056 route_epoch,
2057 kind,
2058 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2059 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2060 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2061 self.handle_supervisor_spawn_snapshot(frame)
2062 }
2063 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2064 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2065 }
2066 ClientControlRequest::SupervisorRestart {
2067 module_id,
2068 drain_timeout_ms,
2069 } => {
2070 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms)
2071 .await
2072 }
2073 ClientControlRequest::SupervisorSwap {
2074 module_id,
2075 ready_timeout_ms,
2076 } => {
2077 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms)
2078 .await
2079 }
2080 ClientControlRequest::SupervisorReload { module_id } => {
2081 self.handle_supervisor_reload(frame, module_id).await
2082 }
2083 ClientControlRequest::SupervisorRescan { preview } => {
2084 self.handle_supervisor_rescan(frame, preview).await
2085 }
2086 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2087 self.handle_supervisor_release_reserved(frame, module_id)
2088 .await
2089 }
2090 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2091 self.handle_supervisor_set_enabled(frame, module_id, enabled)
2092 .await
2093 }
2094 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2095 self.handle_supervisor_health_probe(frame, module_id).await
2096 }
2097 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2098 ClientControlRequest::SupervisorRoutes { module_id } => {
2099 self.handle_supervisor_routes(frame, module_id)
2100 }
2101 ClientControlRequest::SupervisorProvenance { module_id } => {
2102 self.handle_supervisor_provenance(frame, module_id).await
2103 }
2104 ClientControlRequest::SupervisorStderrTail {
2105 module_id,
2106 max_lines,
2107 max_bytes,
2108 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2109 ClientControlRequest::SupervisorTerminals { module_id } => {
2110 self.handle_supervisor_terminals(frame, module_id).await
2111 }
2112 }
2113 }
2114
2115 fn handle_module_control_request(
2116 &self,
2117 connection_id: ConnectionId,
2118 frame: Frame,
2119 request: ModuleControlRequestFromModule,
2120 ) -> Result<Vec<Frame>, RouterError> {
2121 match request {
2122 ModuleControlRequestFromModule::CatalogUpdate {
2123 provides,
2124 capabilities,
2125 ready,
2126 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2127 ModuleControlRequestFromModule::LiveRoots {} => {
2128 let registered = self
2129 .registry
2130 .get_module_by_connection(connection_id)
2131 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2132 let Some(registration) = registered else {
2133 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2134 };
2135 let response = self
2136 .forwarding
2137 .live_roots(®istration.manifest.module_id)
2138 .map_err(RouterError::Forwarding)?;
2139 Ok(vec![control_response_body_frame(
2140 &frame,
2141 &response,
2142 "ModuleControlResponseToModule::LiveRoots",
2143 )?])
2144 }
2145 }
2146 }
2147
2148 fn handle_catalog_update(
2149 &self,
2150 connection_id: ConnectionId,
2151 frame: Frame,
2152 provides: Vec<ProviderRole>,
2153 capabilities: Option<CapabilityDeclarations>,
2154 ready: Option<bool>,
2155 ) -> Result<Vec<Frame>, RouterError> {
2156 self.refresh_capability_requirements();
2157 let Some(registration) = self
2158 .registry
2159 .get_module_by_connection(connection_id)
2160 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2161 else {
2162 return Ok(vec![control_error_frame(
2163 &frame,
2164 "not_registered",
2165 "catalog.update requires an active module registration owned by this connection",
2166 )?]);
2167 };
2168
2169 if let Some(message) =
2170 catalog_update_frozen_field_message(®istration.manifest, &provides)
2171 {
2172 return Ok(vec![control_error_frame(
2173 &frame,
2174 "catalog_update_frozen_field",
2175 message,
2176 )?]);
2177 }
2178
2179 let mut candidate = registration.manifest.clone();
2180 candidate.provides = provides.clone();
2181 candidate.capabilities = capabilities
2182 .clone()
2183 .or_else(|| registration.manifest.capabilities.clone());
2184 if let Err(err) = candidate.validate_capability_grammar() {
2185 return Ok(vec![control_error_frame(
2186 &frame,
2187 "invalid_capability_grammar",
2188 err.to_string(),
2189 )?]);
2190 }
2191
2192 let updated = self
2193 .registry
2194 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
2195 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2196 if updated.is_none() {
2197 return Ok(vec![control_error_frame(
2198 &frame,
2199 "not_registered",
2200 "catalog.update requires an active module registration owned by this connection",
2201 )?]);
2202 }
2203 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
2204 log_duplicate_claim_events(
2205 self.capability_evaluator
2206 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
2207 );
2208 }
2209 if capability_census_trigger(
2210 registration.manifest.capabilities.as_ref(),
2211 updated
2212 .as_ref()
2213 .and_then(|entry| entry.manifest.capabilities.as_ref()),
2214 ) {
2215 self.enforce_capability_denies();
2216 }
2217 self.refresh_capability_requirements();
2218
2219 let response = ModuleControlResponseToModule::CatalogUpdate {};
2220 control_response_body_frame(
2221 &frame,
2222 &response,
2223 "ModuleControlResponseToModule::CatalogUpdate",
2224 )
2225 .map(|frame| vec![frame])
2226 }
2227
2228 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
2229 self.refresh_capability_requirements();
2230 let mut counters = self.counters.snapshot();
2236 if let (Ok((connections_with_routes, max)), Some(obj)) = (
2237 self.forwarding.client_route_concentration(),
2238 counters.as_object_mut(),
2239 ) {
2240 obj.insert(
2241 "client_connections_with_routes".into(),
2242 connections_with_routes.into(),
2243 );
2244 obj.insert("max_routes_on_one_connection".into(), max.into());
2245 }
2246 if let (Some(open_breakers), Some(obj)) = (
2252 self.route_bind_breakers.open_snapshot(),
2253 counters.as_object_mut(),
2254 ) {
2255 obj.insert("route_bind_breakers_open".into(), open_breakers);
2256 }
2257 let response = ClientControlResponse::ServerDescribe {
2258 protocol_ver: PROTOCOL_VERSION,
2259 subc_ops: subc_ops(),
2260 capabilities: self.subc_capabilities.as_ref().to_vec(),
2261 connected_clients: self.connected_clients.count(),
2262 counters: Some(counters),
2263 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
2264 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
2265 capability_requirements: self.capability_requirement_statuses(),
2266 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2267 };
2268 Ok(vec![control_response_body_frame(
2269 &frame,
2270 &response,
2271 "ClientControlResponse::ServerDescribe",
2272 )?])
2273 }
2274
2275 fn handle_catalog_list(
2276 &self,
2277 frame: Frame,
2278 module_id: Option<String>,
2279 ) -> Result<Vec<Frame>, RouterError> {
2280 let (generation, modules) = self.registry.list_modules().map_err(|err| {
2281 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
2282 })?;
2283 let entries = modules
2284 .into_iter()
2285 .filter(|registration| {
2286 module_id
2287 .as_deref()
2288 .map(|wanted| registration.manifest.module_id == wanted)
2289 .unwrap_or(true)
2290 })
2291 .map(|registration| {
2292 let not_ready = self.not_ready_reason(®istration);
2293 let roles = registration.manifest.provides;
2294 CatalogEntry {
2295 module_id: registration.manifest.module_id,
2296 ready: not_ready.is_none(),
2297 not_ready,
2298 module_version: Some(registration.manifest.module_version),
2299 roles,
2300 control_ops: registration.control_ops,
2301 capabilities: registration.manifest.capabilities,
2302 self_signals: registration.manifest.self_signals,
2303 }
2304 })
2305 .collect();
2306 let response = ClientControlResponse::CatalogList {
2307 generation,
2308 modules: entries,
2309 subc_ops: subc_ops(),
2310 };
2311 Ok(vec![control_response_body_frame(
2312 &frame,
2313 &response,
2314 "ClientControlResponse::CatalogList",
2315 )?])
2316 }
2317
2318 fn route_open_principal(
2319 &self,
2320 frame: &Frame,
2321 consumer_identity: Option<ConsumerIdentity>,
2322 ) -> Result<Result<Principal, Frame>, RouterError> {
2323 let Some(consumer_identity) = consumer_identity else {
2324 return Ok(Ok(Principal::Direct));
2325 };
2326
2327 if self.supervisor.spawned_consumer_authorized(
2328 &consumer_identity.module_id,
2329 &consumer_identity.launch_nonce,
2330 ) {
2331 return Ok(Ok(Principal::Reserved {
2332 module_id: consumer_identity.module_id,
2333 }));
2334 }
2335
2336 Ok(Err(control_error_frame(
2337 frame,
2338 "bad_consumer_identity",
2339 format!(
2340 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
2341 consumer_identity.module_id
2342 ),
2343 )?))
2344 }
2345
2346 fn route_open_refusal_frame(
2352 &self,
2353 ctx: &RouteCtx,
2354 frame: &Frame,
2355 module_id: &str,
2356 reason: &'static str,
2357 code: &'static str,
2358 message: impl Into<String>,
2359 ) -> Result<Frame, RouterError> {
2360 self.observe_route_open_refusal(ctx, module_id, reason, code);
2361 control_error_frame(frame, code, message.into())
2362 }
2363
2364 fn route_open_breaker_refusal_frame(
2382 &self,
2383 ctx: &RouteCtx,
2384 frame: &Frame,
2385 module_id: &str,
2386 consecutive_timeouts: u32,
2387 retry_in: Duration,
2388 probe_in_flight: bool,
2389 ) -> Result<Frame, RouterError> {
2390 self.counters
2391 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
2392 debug!(
2393 target: "control",
2394 code = "module_timeout",
2395 module_id = ?module_id,
2396 connection_id = ctx.connection_id.get(),
2397 consecutive_timeouts,
2398 retry_in_ms = retry_in.as_millis() as u64,
2399 probe_in_flight,
2400 "route.open refused by open bind-relay breaker"
2401 );
2402 let detail = if probe_in_flight {
2403 "one probe bind is already in flight; retry once it settles".to_string()
2404 } else {
2405 format!("not relaying for another {retry_in:?}")
2406 };
2407 control_error_frame(
2408 frame,
2409 "module_timeout",
2410 format!(
2411 "module_id '{module_id}' failed {consecutive_timeouts} consecutive route.bind \
2412 relays; {detail}"
2413 ),
2414 )
2415 }
2416
2417 fn observe_route_open_refusal(
2430 &self,
2431 ctx: &RouteCtx,
2432 module_id: &str,
2433 reason: &'static str,
2434 code: &'static str,
2435 ) {
2436 self.counters.increment_route_open_refused(code);
2437 info!(
2438 target: "control",
2439 code,
2440 reason,
2441 module_id = ?module_id,
2442 connection_id = ctx.connection_id.get(),
2443 "route.open refused"
2444 );
2445 }
2446
2447 fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
2492 self.counters.increment_route_open_accepted(principal);
2493 info!(
2494 target: "control",
2495 principal,
2496 module_id,
2497 connection_id = ctx.connection_id.get(),
2498 "route.open accepted"
2499 );
2500 }
2501
2502 fn supervised_absent_route_open_refusal_frame(
2503 &self,
2504 ctx: &RouteCtx,
2505 frame: &Frame,
2506 module_id: &str,
2507 code: &'static str,
2508 status: &crate::supervise::ModuleStatus,
2509 ) -> Result<Frame, RouterError> {
2510 self.counters.increment_route_open_refused(code);
2511 info!(
2512 target: "control",
2513 code,
2514 reason = "supervised_not_registered",
2515 module_id = ?module_id,
2516 connection_id = ctx.connection_id.get(),
2517 state = %status.state,
2518 enabled = status.enabled,
2519 live = status.live,
2520 "route.open refused"
2521 );
2522 control_error_frame(
2523 frame,
2524 code,
2525 format!(
2526 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
2527 status.state, status.enabled, status.live
2528 ),
2529 )
2530 }
2531
2532 async fn handle_route_open(
2533 &self,
2534 ctx: &RouteCtx,
2535 frame: Frame,
2536 request: RouteOpenRequest,
2537 ) -> Result<Vec<Frame>, RouterError> {
2538 let RouteOpenRequest {
2539 target,
2540 mut identity,
2541 consumer_identity,
2542 consumer_capabilities,
2543 admission_facts,
2544 } = request;
2545 let target_module_id = target_module_id(&target).to_string();
2546 debug!(
2547 connection_id = ctx.connection_id.get(),
2548 corr = frame.header.corr,
2549 module_id = %target_module_id,
2550 "handling route.open"
2551 );
2552
2553 let Some(registration) = self
2572 .registry
2573 .get_module(&target_module_id)
2574 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2575 else {
2576 if let Some((status, warming)) =
2577 self.supervisor_status(&target_module_id, frame.header.corr)?
2578 {
2579 if status.protocol == ModuleProtocol::None {
2586 return Ok(vec![self.route_open_refusal_frame(
2587 ctx,
2588 &frame,
2589 &target_module_id,
2590 "protocol_none",
2591 error_codes::MODULE_NO_PROTOCOL,
2592 format!(
2593 "module_id '{target_module_id}' is declared protocol: none; \
2594 it speaks no subc wire and serves no routes"
2595 ),
2596 )?]);
2597 }
2598 let code = if warming {
2599 "module_warming"
2600 } else {
2601 "target_unavailable"
2602 };
2603 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
2604 ctx,
2605 &frame,
2606 &target_module_id,
2607 code,
2608 &status,
2609 )?]);
2610 }
2611 if let Some(removed_ago_ms) =
2612 self.supervisor.removal_tombstone_age_ms(&target_module_id)
2613 {
2614 return Ok(vec![self.route_open_refusal_frame(
2615 ctx,
2616 &frame,
2617 &target_module_id,
2618 "removed",
2619 error_codes::MODULE_REMOVED,
2620 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
2621 )?]);
2622 }
2623 return Ok(vec![self.route_open_refusal_frame(
2624 ctx,
2625 &frame,
2626 &target_module_id,
2627 "not_registered",
2628 error_codes::UNKNOWN_MODULE,
2629 format!("module_id '{target_module_id}' is not registered"),
2630 )?]);
2631 };
2632
2633 if !registration.ready {
2637 self.counters
2638 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
2639 info!(
2640 target: "control",
2641 code = error_codes::MODULE_WARMING,
2642 module_id = ?target_module_id,
2643 connection_id = ctx.connection_id.get(),
2644 reason = "declared_not_ready",
2645 "route.open refused"
2646 );
2647 return Ok(vec![control_error_body_frame(
2648 &frame,
2649 ErrorBody {
2650 code: error_codes::MODULE_WARMING.to_string(),
2651 message: format!(
2652 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
2653 ),
2654 detail: Some(serde_json::json!({
2655 "reason": "declared_not_ready"
2656 })),
2657 },
2658 )?]);
2659 }
2660
2661 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
2680 self.counters
2681 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
2682 info!(
2683 target: "control",
2684 code = error_codes::MODULE_WARMING,
2685 module_id = ?target_module_id,
2686 connection_id = ctx.connection_id.get(),
2687 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
2688 capability = %capability,
2689 "route.open refused"
2690 );
2691 return Ok(vec![control_error_body_frame(
2692 &frame,
2693 ErrorBody {
2694 code: error_codes::MODULE_WARMING.to_string(),
2695 message: format!(
2696 "module_id '{target_module_id}' requires capability '{capability}', \
2697 which no registered module provides; retry"
2698 ),
2699 detail: Some(serde_json::json!({
2700 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
2701 "capability": capability,
2702 })),
2703 },
2704 )?]);
2705 }
2706
2707 if !target_has_required_role(&target, ®istration.manifest.provides) {
2708 return Ok(vec![self.route_open_refusal_frame(
2709 ctx,
2710 &frame,
2711 &target_module_id,
2712 "role_not_provided",
2713 "target_unavailable",
2714 format!("module_id '{target_module_id}' does not provide the requested target"),
2715 )?]);
2716 }
2717
2718 if registration.state != ChannelState::Active {
2719 return Ok(vec![self.route_open_refusal_frame(
2720 ctx,
2721 &frame,
2722 &target_module_id,
2723 "registration_not_active",
2724 "target_unavailable",
2725 format!("module_id '{target_module_id}' is not active"),
2726 )?]);
2727 }
2728
2729 if self
2730 .forwarding
2731 .module_is_draining(&target_module_id)
2732 .map_err(RouterError::Forwarding)?
2733 {
2734 return Ok(vec![self.route_open_refusal_frame(
2735 ctx,
2736 &frame,
2737 &target_module_id,
2738 "reloading",
2739 "module_reloading",
2740 format!("module_id '{target_module_id}' is reloading"),
2741 )?]);
2742 }
2743
2744 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
2745 process_liveness.process_live(&target_module_id) == Some(false)
2746 }) {
2747 if process_liveness.process_replacing(&target_module_id) {
2754 return Ok(vec![self.route_open_refusal_frame(
2755 ctx,
2756 &frame,
2757 &target_module_id,
2758 "reloading",
2759 "module_reloading",
2760 format!("module_id '{target_module_id}' is reloading"),
2761 )?]);
2762 }
2763 return Ok(vec![self.route_open_refusal_frame(
2764 ctx,
2765 &frame,
2766 &target_module_id,
2767 "supervisor_not_live",
2768 "target_unavailable",
2769 format!("module_id '{target_module_id}' is not live"),
2770 )?]);
2771 }
2772
2773 if !self
2774 .forwarding
2775 .has_live_module_connection(&target_module_id)
2776 .map_err(RouterError::Forwarding)?
2777 {
2778 return Ok(vec![self.route_open_refusal_frame(
2779 ctx,
2780 &frame,
2781 &target_module_id,
2782 "no_forwarding_connection",
2783 "target_unavailable",
2784 format!("module_id '{target_module_id}' has no live forwarding connection"),
2785 )?]);
2786 }
2787
2788 if let Some(error) =
2789 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
2790 {
2791 self.observe_route_open_refusal(
2792 ctx,
2793 &target_module_id,
2794 "op_not_allowed",
2795 "op_not_allowed",
2796 );
2797 return Ok(vec![error]);
2798 }
2799
2800 let principal = match self.route_open_principal(&frame, consumer_identity)? {
2801 Ok(principal) => principal,
2802 Err(error) => {
2803 self.observe_route_open_refusal(
2804 ctx,
2805 &target_module_id,
2806 "bad_consumer_identity",
2807 "bad_consumer_identity",
2808 );
2809 return Ok(vec![error]);
2810 }
2811 };
2812
2813 if let Principal::Reserved {
2817 module_id: opening_module_id,
2818 } = &principal
2819 {
2820 if let Some(opening_registration) = self
2821 .registry
2822 .get_module(opening_module_id)
2823 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2824 {
2825 if let Some(capability) =
2826 denied_capability(&opening_registration.manifest, ®istration.manifest)
2827 {
2828 warn!(
2829 opening_module_id,
2830 target_module_id,
2831 capability,
2832 "refusing route.open because an attested capability deny edge matches"
2833 );
2834 return Ok(vec![self.route_open_refusal_frame(
2835 ctx,
2836 &frame,
2837 &target_module_id,
2838 "capability_deny_edge",
2839 "capability_forbidden",
2840 format!(
2841 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
2842 ),
2843 )?]);
2844 }
2845 }
2846 }
2847
2848 if admission_facts.is_some() {
2849 let carrier_matches = matches!(
2850 &principal,
2851 Principal::Reserved { module_id }
2852 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
2853 );
2854 if !carrier_matches {
2855 return Ok(vec![self.route_open_refusal_frame(
2856 ctx,
2857 &frame,
2858 &target_module_id,
2859 "admission_facts_carrier_not_permitted",
2860 "admission_facts_not_permitted",
2861 "admission facts may only be carried by the configured reserved module",
2862 )?]);
2863 }
2864
2865 let target_allowed = self
2866 .admission_facts_targets
2867 .as_ref()
2868 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
2869 if !target_allowed {
2870 return Ok(vec![self.route_open_refusal_frame(
2871 ctx,
2872 &frame,
2873 &target_module_id,
2874 "admission_facts_target_not_listed",
2875 "admission_facts_target_not_allowed",
2876 format!(
2877 "admission facts are not permitted for target module_id '{target_module_id}'"
2878 ),
2879 )?]);
2880 }
2881
2882 }
2886
2887 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
2909 Ok(project_root) => project_root,
2910 Err(err) => {
2911 return Ok(vec![control_error_frame(
2912 &frame,
2913 "invalid_project_root",
2914 err.to_string(),
2915 )?])
2916 }
2917 };
2918 identity.project_root = project_root.as_path().to_path_buf();
2919
2920 let _concurrency_guard = match self
2931 .route_bind_concurrency
2932 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
2933 {
2934 Ok(guard) => guard,
2935 Err(in_flight) => {
2936 return Ok(vec![self.route_open_admission_refusal_frame(
2937 ctx,
2938 &frame,
2939 &target_module_id,
2940 format!(
2941 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
2942 ),
2943 )?]);
2944 }
2945 };
2946
2947 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
2950 RouteBindAdmission::Admitted { guard, probe } => {
2951 if probe {
2952 info!(
2953 module_id = %target_module_id,
2954 connection_id = ctx.connection_id.get(),
2955 "route.bind breaker half-open: admitting one probe"
2956 );
2957 }
2958 guard
2959 }
2960 RouteBindAdmission::Refused {
2961 consecutive_timeouts,
2962 retry_in,
2963 probe_in_flight,
2964 } => {
2965 return Ok(vec![self.route_open_breaker_refusal_frame(
2966 ctx,
2967 &frame,
2968 &target_module_id,
2969 consecutive_timeouts,
2970 retry_in,
2971 probe_in_flight,
2972 )?]);
2973 }
2974 };
2975
2976 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
2981 let relay_deadline = Instant::now() + route_bind_relay_timeout;
2982 let pending = match self
2983 .forwarding
2984 .begin_route_bind_relay_for(
2985 ctx.connection_id,
2986 ctx.egress.clone(),
2987 response_version(&frame),
2988 frame.header.corr,
2989 &target_module_id,
2990 principal.clone(),
2991 Some(project_root),
2992 relay_deadline,
2993 )
2994 .await
2995 {
2996 Ok(pending) => pending,
2997 Err(err) => {
2998 return Ok(vec![self.route_open_refusal_frame(
2999 ctx,
3000 &frame,
3001 &target_module_id,
3002 "relay_reservation_failed",
3003 forwarding_error_code(&err),
3004 err.to_string(),
3005 )?])
3006 }
3007 };
3008 let crate::forwarding::PendingRouteBindRelay {
3009 endpoint,
3010 module_sink,
3011 negotiated_ver,
3012 client_channel,
3013 client_epoch,
3014 module_channel,
3015 module_epoch,
3016 corr: relay_corr,
3017 receiver,
3018 } = pending;
3019 let mut reservation =
3020 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
3021
3022 debug!(
3023 connection_id = ctx.connection_id.get(),
3024 client_channel,
3025 client_epoch,
3026 module_channel,
3027 module_epoch,
3028 "reserved route handle pair"
3029 );
3030 let principal_label = match &principal {
3033 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
3034 Principal::Direct => "direct".to_string(),
3035 other => format!("{other:?}"),
3036 };
3037 let relay = ModuleControlRequest::RouteBind {
3038 route_channel: module_channel,
3039 epoch: module_epoch,
3040 target,
3041 identity,
3042 principal: Some(principal),
3043 consumer_capabilities,
3044 admission_facts,
3045 };
3046 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
3047 RouterError::backend(
3048 0,
3049 frame.header.corr,
3050 format!("failed to encode route.bind request: {err}"),
3051 )
3052 })?;
3053 let relay_frame = Frame::build_with_version(
3054 negotiated_ver,
3055 FrameType::Request,
3056 control_flags(),
3057 0,
3058 0,
3059 relay_corr,
3060 relay_body,
3061 )
3062 .map_err(RouterError::FrameBuild)?;
3063
3064 if let Err(err) = module_sink.send(relay_frame).await {
3065 reservation.release_and_disarm();
3066 return Ok(vec![self.route_open_refusal_frame(
3067 ctx,
3068 &frame,
3069 &target_module_id,
3070 "relay_send_failed",
3071 "target_unavailable",
3072 err.to_string(),
3073 )?]);
3074 }
3075
3076 if !self
3077 .forwarding
3078 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
3079 .map_err(RouterError::Forwarding)?
3080 {
3081 self.send_abandoned_route_bind_goodbye(
3082 &module_sink,
3083 negotiated_ver,
3084 module_channel,
3085 module_epoch,
3086 );
3087 }
3088
3089 match timeout_at(relay_deadline, receiver).await {
3090 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
3091 reservation.disarm();
3092 if breaker.record_accepted() {
3093 info!(
3094 module_id = %target_module_id,
3095 "route.bind breaker closed: the probe was accepted"
3096 );
3097 }
3098 self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
3099 Ok(Vec::new())
3100 }
3101 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
3102 reservation.release_and_disarm();
3103 breaker.record_inconclusive();
3107 self.counters
3108 .increment_route_open_refused("module_rejected");
3109 info!(
3110 target: "control",
3111 code = "module_rejected",
3112 module_code = ?body.code,
3113 module_id = ?target_module_id,
3114 connection_id = ctx.connection_id.get(),
3115 "route.open refused"
3116 );
3117 Ok(vec![control_error_body_frame(&frame, body)?])
3118 }
3119 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
3120 reservation.release_and_disarm();
3121 breaker.record_inconclusive();
3122 tracing::warn!(
3129 module_id = %target_module_id,
3130 "route.bind relay abandoned: {message}"
3131 );
3132 Ok(vec![self.route_open_refusal_frame(
3133 ctx,
3134 &frame,
3135 &target_module_id,
3136 "relay_abandoned",
3137 "target_unavailable",
3138 message,
3139 )?])
3140 }
3141 Ok(Err(_)) => {
3142 reservation.release_and_disarm();
3143 breaker.record_inconclusive();
3144 Ok(vec![self.route_open_refusal_frame(
3145 ctx,
3146 &frame,
3147 &target_module_id,
3148 "relay_waiter_canceled",
3149 "target_unavailable",
3150 "route.bind relay waiter was canceled before the module responded",
3151 )?])
3152 }
3153 Err(_) => {
3154 reservation.release_and_disarm();
3155 if let Some(opened) = breaker.record_timeout(
3159 self.route_bind_breaker_threshold,
3160 self.route_bind_breaker_cooldown,
3161 ) {
3162 warn!(
3163 module_id = %target_module_id,
3164 consecutive_timeouts = opened.consecutive_timeouts,
3165 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
3166 reopened_after_probe = opened.reopened_after_probe,
3167 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
3168 );
3169 }
3170 tracing::warn!(
3178 module_id = %target_module_id,
3179 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
3180 "route.bind relay timed out: module did not ack within budget"
3181 );
3182 Ok(vec![self.route_open_refusal_frame(
3183 ctx,
3184 &frame,
3185 &target_module_id,
3186 "relay_timed_out",
3187 "module_timeout",
3188 format!(
3189 "module_id '{target_module_id}' did not answer route.bind within {:?}",
3190 route_bind_relay_timeout
3191 ),
3192 )?])
3193 }
3194 }
3195 }
3196
3197 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3198 let response = ClientControlResponse::SupervisorSpawnSnapshot {
3199 snapshot: self.supervisor.spawn_snapshot(),
3200 };
3201 Ok(vec![control_response_body_frame(
3202 &frame,
3203 &response,
3204 "ClientControlResponse::SupervisorSpawnSnapshot",
3205 )?])
3206 }
3207
3208 fn handle_supervisor_spawn_subscribe(
3209 &self,
3210 ctx: &RouteCtx,
3211 frame: Frame,
3212 since: Option<SpawnCursor>,
3213 ) -> Result<Vec<Frame>, RouterError> {
3214 match self.supervisor.subscribe_spawns(
3215 ctx.connection_id,
3216 frame.header.corr,
3217 response_version(&frame),
3218 since,
3219 ctx.egress.clone(),
3220 ) {
3221 Ok(()) => Ok(Vec::new()),
3222 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
3223 Ok(vec![control_error_body_frame(
3224 &frame,
3225 ErrorBody {
3226 code: "spawn_cursor_incarnation_mismatch".to_string(),
3227 message: "spawn cursor belongs to a different daemon incarnation"
3228 .to_string(),
3229 detail: Some(serde_json::json!({
3230 "current_daemon_incarnation": current
3231 })),
3232 },
3233 )?])
3234 }
3235 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
3236 &frame,
3237 ErrorBody {
3238 code: "spawn_cursor_too_old".to_string(),
3239 message: "spawn cursor predates the retained event ring".to_string(),
3240 detail: Some(serde_json::json!({
3241 "oldest_retained_cursor": oldest
3242 })),
3243 },
3244 )?]),
3245 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
3246 0,
3247 frame.header.corr,
3248 format!("failed to open supervisor spawn subscription: {error}"),
3249 )),
3250 }
3251 }
3252
3253 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3254 let generation = self
3255 .registry
3256 .generation()
3257 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3258 let mut modules = Vec::new();
3259 for module in self.supervisor.list() {
3260 let status = module.status_for_control("list").map_err(|err| {
3261 RouterError::backend(
3262 0,
3263 frame.header.corr,
3264 format!("failed to read supervisor status: {err}"),
3265 )
3266 })?;
3267 let (configured, _) = module.configuration().map_err(|err| {
3268 RouterError::backend(
3269 0,
3270 frame.header.corr,
3271 format!("failed to read module configuration: {err}"),
3272 )
3273 })?;
3274 let image = module.running_image_agreement().await;
3276 let pending_reload = Some(reload_verdict(
3277 &configured.program,
3278 status.spawned_from.as_deref(),
3279 image,
3280 ));
3281 modules.push(SupervisorEntry {
3282 module_id: status.module_id,
3283 state: status.state.to_string(),
3284 enabled: status.enabled,
3285 live: status.live,
3286 protocol: status.protocol,
3287 health: status.health.status,
3288 pending_reload,
3289 last_probe_ms: status.health.last_probe_ms,
3290 last_exit_code: status.last_exit.as_ref().and_then(|e| e.code),
3291 last_exit_signal: status.last_exit.as_ref().and_then(|e| e.signal),
3292 last_exit_ms: status.last_exit.as_ref().map(|e| e.at_ms),
3293 last_exit_kind: status.last_exit.as_ref().map(|e| e.kind.into()),
3294 restart_count: Some(status.restart_count),
3295 max_restarts: Some(status.max_restarts),
3296 lifetime_restarts: Some(status.lifetime_restarts),
3297 spawn_generation: Some(status.spawn_generation),
3298 restart_window_secs: Some(status.restart_window.as_secs()),
3299 drain_timeout_ms: Some(status.drain_timeout.as_millis() as u64),
3300 restart_backoff_ms: Some(status.restart_backoff.as_millis() as u64),
3301 restart_max_backoff_ms: Some(status.restart_max_backoff.as_millis() as u64),
3302 });
3303 }
3304 let response = ClientControlResponse::SupervisorList {
3305 generation,
3306 modules,
3307 };
3308 Ok(vec![control_response_body_frame(
3309 &frame,
3310 &response,
3311 "ClientControlResponse::SupervisorList",
3312 )?])
3313 }
3314
3315 fn handle_supervisor_stderr_tail(
3316 &self,
3317 frame: Frame,
3318 module_id: String,
3319 max_lines: Option<u32>,
3320 max_bytes: Option<u32>,
3321 ) -> Result<Vec<Frame>, RouterError> {
3322 let Some(module) = self.supervisor.get(&module_id) else {
3323 return Ok(vec![control_error_frame(
3324 &frame,
3325 "unknown_module",
3326 format!("module_id '{module_id}' is not supervised"),
3327 )?]);
3328 };
3329
3330 let snapshot = module.stderr_tail(
3331 max_lines.map(|value| value as usize),
3332 max_bytes.map(|value| value as usize),
3333 );
3334
3335 let response = ClientControlResponse::SupervisorStderrTail {
3336 module_id,
3337 tail: StderrTail {
3338 capture: match snapshot.capture {
3339 CaptureState::Captured => StderrCaptureState::Captured,
3340 CaptureState::Incomplete { reason } => {
3341 StderrCaptureState::Incomplete { reason }
3342 }
3343 CaptureState::NotCaptured { reason } => {
3344 StderrCaptureState::NotCaptured { reason }
3345 }
3346 },
3347 entries: snapshot
3348 .entries
3349 .into_iter()
3350 .map(|entry| match entry {
3351 TailEntry::Line { text, truncated } => {
3352 StderrTailEntry::Line { text, truncated }
3353 }
3354 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
3355 })
3356 .collect(),
3357 dropped_lines: snapshot.dropped_lines,
3358 },
3359 };
3360 Ok(vec![control_response_body_frame(
3361 &frame,
3362 &response,
3363 "ClientControlResponse::SupervisorStderrTail",
3364 )?])
3365 }
3366
3367 async fn handle_supervisor_terminals(
3368 &self,
3369 frame: Frame,
3370 module_id: String,
3371 ) -> Result<Vec<Frame>, RouterError> {
3372 let Some(module) = self.supervisor.get(&module_id) else {
3373 return Ok(vec![control_error_frame(
3374 &frame,
3375 "unknown_module",
3376 format!("module_id '{module_id}' is not supervised"),
3377 )?]);
3378 };
3379
3380 let terminals = module
3383 .read_durable_terminal_history()
3384 .await
3385 .map_err(|error| {
3386 RouterError::backend(
3387 0,
3388 frame.header.corr,
3389 format!("failed to read terminal history: {error}"),
3390 )
3391 })?;
3392 let response = ClientControlResponse::SupervisorTerminals {
3393 module_id,
3394 terminals,
3395 };
3396 Ok(vec![control_response_body_frame(
3397 &frame,
3398 &response,
3399 "ClientControlResponse::SupervisorTerminals",
3400 )?])
3401 }
3402
3403 fn handle_supervisor_routes(
3404 &self,
3405 frame: Frame,
3406 module_id: Option<String>,
3407 ) -> Result<Vec<Frame>, RouterError> {
3408 let modules = self
3409 .forwarding
3410 .route_census(module_id.as_deref())
3411 .map_err(RouterError::Forwarding)?
3412 .into_iter()
3413 .map(|(module_id, routes)| SupervisorRouteModule {
3414 module_id,
3415 routes: routes
3416 .into_iter()
3417 .map(|route| SupervisorRoute {
3418 consumer: match route.principal {
3419 Principal::Reserved { module_id } => {
3420 SupervisorRouteConsumer::Reserved { module_id }
3421 }
3422 Principal::Direct | Principal::Unverified => {
3423 SupervisorRouteConsumer::Direct {
3424 connection_id: route.goodbye_target.connection_id.get(),
3425 }
3426 }
3427 },
3428 age_ms: Instant::now()
3429 .saturating_duration_since(route.bound_at)
3430 .as_millis()
3431 .try_into()
3432 .unwrap_or(u64::MAX),
3433 draining: route.draining,
3434 drain_reason: route.drain_reason,
3435 })
3436 .collect(),
3437 })
3438 .collect();
3439 let response = ClientControlResponse::SupervisorRoutes { modules };
3440 Ok(vec![control_response_body_frame(
3441 &frame,
3442 &response,
3443 "ClientControlResponse::SupervisorRoutes",
3444 )?])
3445 }
3446
3447 async fn handle_supervisor_provenance(
3448 &self,
3449 frame: Frame,
3450 module_id: Option<String>,
3451 ) -> Result<Vec<Frame>, RouterError> {
3452 let mut selected = if let Some(module_id) = module_id {
3453 let Some(module) = self.supervisor.get(&module_id) else {
3454 return Ok(vec![control_error_frame(
3455 &frame,
3456 "unknown_module",
3457 format!("module_id '{module_id}' is not supervised"),
3458 )?]);
3459 };
3460 vec![module]
3461 } else {
3462 self.supervisor.list()
3463 };
3464
3465 let mut modules = Vec::with_capacity(selected.len());
3466 for module in selected.drain(..) {
3467 let status = module.status().map_err(|err| {
3468 RouterError::backend(
3469 0,
3470 frame.header.corr,
3471 format!("failed to read supervisor status: {err}"),
3472 )
3473 })?;
3474 let module_declared = self
3475 .registry
3476 .get_module(&status.module_id)
3477 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3478 .and_then(|registration| registration.manifest.provenance)
3479 .map(|build| ModuleDeclaredProvenance::Reported { build })
3480 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
3481 #[cfg(test)]
3482 let running_image = match &self.provenance_probe_override {
3483 Some(result) => result.clone(),
3484 None => module.running_image_agreement().await,
3485 };
3486 #[cfg(not(test))]
3487 let running_image = module.running_image_agreement().await;
3488 modules.push(SupervisorModuleProvenance {
3489 module_id: status.module_id,
3490 module_declared,
3491 daemon_observed: SupervisorObservedProcess {
3492 pid: status.pid,
3493 spawned_at_ms: status.spawned_at_ms,
3494 spawned_from: status.spawned_from,
3495 running_image,
3496 },
3497 });
3498 }
3499 let daemon = SupervisorDaemonProvenance {
3500 daemon_build: self.daemon_provenance.build.clone(),
3501 daemon_observed: DaemonObservedProcess {
3502 pid: self.daemon_provenance.pid,
3503 started_at_ms: self
3504 .daemon_provenance
3505 .start_clock
3506 .map(|clock| clock.started_at_ms())
3507 .or(self.daemon_provenance.started_at_ms),
3508 running_image: self
3509 .daemon_provenance
3510 .probe
3511 .observe(
3512 self.daemon_provenance.pid,
3513 self.daemon_provenance.executable_path.as_deref(),
3514 self.daemon_provenance.executable_identity,
3515 self.daemon_provenance.process_start_time,
3516 )
3517 .await,
3518 },
3519 };
3520 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
3521 Ok(vec![control_response_body_frame(
3522 &frame,
3523 &response,
3524 "ClientControlResponse::SupervisorProvenance",
3525 )?])
3526 }
3527
3528 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3529 self.refresh_capability_requirements();
3530 let generation = self
3531 .registry
3532 .generation()
3533 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3534 let modules = self
3535 .supervisor
3536 .list()
3537 .into_iter()
3538 .map(|module| {
3539 let status = module.status_for_control("health").map_err(|err| {
3540 RouterError::backend(
3541 0,
3542 frame.header.corr,
3543 format!("failed to read supervisor health: {err}"),
3544 )
3545 })?;
3546 let module_id = status.module_id;
3547 let capability_detail = self
3548 .capability_evaluator
3549 .required_problem_detail(&module_id);
3550 Ok(SupervisorHealthEntry {
3551 module_id,
3552 status: status.health.status,
3553 detail: append_capability_problem_detail(
3554 status.health.detail,
3555 capability_detail,
3556 ),
3557 metrics: status.health.metrics,
3558 consecutive_failures: status.health.consecutive_failures,
3559 late_answer_count: status.health.late_answer_count,
3560 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
3561 last_action: status.health.last_action,
3562 last_action_ms: status.health.last_action_ms,
3563 last_probe_ms: status.health.last_probe_ms,
3564 })
3565 })
3566 .collect::<Result<Vec<_>, RouterError>>()?;
3567 let response = ClientControlResponse::SupervisorHealth {
3568 generation,
3569 modules,
3570 };
3571 Ok(vec![control_response_body_frame(
3572 &frame,
3573 &response,
3574 "ClientControlResponse::SupervisorHealth",
3575 )?])
3576 }
3577
3578 async fn handle_supervisor_restart(
3579 &self,
3580 frame: Frame,
3581 module_id: String,
3582 drain_timeout_ms: Option<u64>,
3583 ) -> Result<Vec<Frame>, RouterError> {
3584 let operation_lock = self.supervisor.operation_lock();
3585 let _operation_guard = operation_lock.lock().await;
3586 let Some(module) = self.supervisor.get(&module_id) else {
3587 return Ok(vec![control_error_frame(
3588 &frame,
3589 "unknown_module",
3590 format!("module_id '{module_id}' is not supervised"),
3591 )?]);
3592 };
3593
3594 if let Err(err) = module.restart(drain_timeout_ms).await {
3595 let (code, message) = match err {
3596 crate::supervise::SuperviseError::Disabled { .. } => {
3597 ("module_disabled", err.to_string())
3598 }
3599 crate::supervise::SuperviseError::SwapInProgress { .. } => {
3600 ("swap_in_progress", err.to_string())
3601 }
3602 _ => (
3603 "target_unavailable",
3604 format!("failed to restart module_id '{module_id}': {err}"),
3605 ),
3606 };
3607 return Ok(vec![control_error_frame(&frame, code, message)?]);
3608 }
3609
3610 let response = ClientControlResponse::SupervisorAck {
3611 module_id,
3612 applied: true,
3613 };
3614 Ok(vec![control_response_body_frame(
3615 &frame,
3616 &response,
3617 "ClientControlResponse::SupervisorAck",
3618 )?])
3619 }
3620
3621 async fn handle_supervisor_swap(
3625 &self,
3626 frame: Frame,
3627 module_id: String,
3628 ready_timeout_ms: Option<u64>,
3629 ) -> Result<Vec<Frame>, RouterError> {
3630 let module = {
3637 let operation_lock = self.supervisor.operation_lock();
3638 let _operation_guard = operation_lock.lock().await;
3639 self.supervisor.get(&module_id)
3640 };
3641 let Some(module) = module else {
3642 return Ok(vec![control_error_frame(
3643 &frame,
3644 "unknown_module",
3645 format!("module_id '{module_id}' is not supervised"),
3646 )?]);
3647 };
3648
3649 if let Err(err) = module
3650 .swap(ready_timeout_ms.map(Duration::from_millis))
3651 .await
3652 {
3653 use crate::supervise::SuperviseError;
3654 let message = err.to_string();
3655 let error = match err {
3656 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
3657 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
3658 code: "swap_refused".to_string(),
3659 message,
3660 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
3661 },
3662 SuperviseError::SwapFailed {
3663 arm,
3664 candidate_exit,
3665 ..
3666 } => ErrorBody {
3667 code: "swap_failed".to_string(),
3668 message,
3669 detail: Some(serde_json::json!({
3670 "arm": arm.as_str(),
3671 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
3672 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
3673 })),
3674 },
3675 _ => ErrorBody::new(
3676 "target_unavailable",
3677 format!("failed to swap module_id '{module_id}': {message}"),
3678 ),
3679 };
3680 return Ok(vec![control_error_body_frame(&frame, error)?]);
3681 }
3682
3683 let response = ClientControlResponse::SupervisorAck {
3684 module_id,
3685 applied: true,
3686 };
3687 Ok(vec![control_response_body_frame(
3688 &frame,
3689 &response,
3690 "ClientControlResponse::SupervisorAck",
3691 )?])
3692 }
3693
3694 async fn handle_supervisor_reload(
3695 &self,
3696 frame: Frame,
3697 module_id: String,
3698 ) -> Result<Vec<Frame>, RouterError> {
3699 let operation_lock = self.supervisor.operation_lock();
3700 let _operation_guard = operation_lock.lock().await;
3701 let Some(module) = self.supervisor.get(&module_id) else {
3702 return Ok(vec![control_error_frame(
3703 &frame,
3704 "unknown_module",
3705 format!("module_id '{module_id}' is not supervised"),
3706 )?]);
3707 };
3708
3709 if let Err(err) = module.reload().await {
3710 let (code, message) = match err {
3711 crate::supervise::SuperviseError::Disabled { .. } => {
3712 ("module_disabled", err.to_string())
3713 }
3714 crate::supervise::SuperviseError::SwapInProgress { .. } => {
3715 ("swap_in_progress", err.to_string())
3716 }
3717 _ => (
3718 "reload_failed",
3719 format!("failed to reload module_id '{module_id}': {err}"),
3720 ),
3721 };
3722 return Ok(vec![control_error_frame(&frame, code, message)?]);
3723 }
3724
3725 let response = ClientControlResponse::SupervisorAck {
3726 module_id,
3727 applied: true,
3728 };
3729 Ok(vec![control_response_body_frame(
3730 &frame,
3731 &response,
3732 "ClientControlResponse::SupervisorAck",
3733 )?])
3734 }
3735
3736 async fn handle_supervisor_rescan(
3737 &self,
3738 frame: Frame,
3739 preview: bool,
3740 ) -> Result<Vec<Frame>, RouterError> {
3741 let Some(context) = self.rescan.clone() else {
3742 return Ok(vec![control_error_frame(
3743 &frame,
3744 "rescan_unavailable",
3745 "the daemon was not started with a reloadable config path".to_string(),
3746 )?]);
3747 };
3748
3749 let operation_lock = self.supervisor.operation_lock();
3750 let _operation_guard = operation_lock.lock().await;
3751 let loaded = match crate::daemon_config::load(&context.config_path) {
3752 Ok(config) => config,
3753 Err(err) => {
3754 return Ok(vec![control_error_frame(
3755 &frame,
3756 "invalid_daemon_config",
3757 format!("supervisor rescan rejected daemon config: {err}"),
3758 )?])
3759 }
3760 };
3761 let Some(config) = loaded else {
3769 return Ok(vec![control_error_frame(
3770 &frame,
3771 "invalid_daemon_config",
3772 format!(
3773 "daemon config not found at {}; refusing to rescan (an absent config would \
3774 retire every supervised module)",
3775 context.config_path.display()
3776 ),
3777 )?]);
3778 };
3779 let (
3780 configured_port,
3781 storage_config,
3782 admission_facts_carrier_module_id,
3783 admission_facts_targets,
3784 modules,
3785 reserved_capabilities,
3786 ) = (
3787 config.port,
3788 config.storage,
3789 config.admission_facts_carrier_module_id,
3790 config.admission_facts_targets,
3791 config.modules,
3792 config.reserved_capabilities,
3793 );
3794
3795 let mut restart_required = Vec::new();
3804 for section in RestartRequiredSection::ALL {
3805 let changed = match section {
3806 RestartRequiredSection::Port => configured_port != context.configured_port,
3807 RestartRequiredSection::Storage => storage_config != context.storage_config,
3808 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
3809 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
3810 }
3811 RestartRequiredSection::AdmissionFactsTargets => {
3812 admission_facts_targets != context.admission_facts_targets
3813 }
3814 };
3815 if changed {
3816 restart_required.push(section.label().to_string());
3817 }
3818 }
3819 if !restart_required.is_empty() {
3820 warn!(
3821 config_path = %context.config_path.display(),
3822 sections = %restart_required.join(", "),
3823 "daemon config changed outside the modules section; restart the daemon to apply those changes"
3824 );
3825 }
3826
3827 for configured in &modules {
3828 if let Err(err) = validate_spec(&configured.module_spec()) {
3829 return Ok(vec![control_error_frame(
3830 &frame,
3831 "invalid_daemon_config",
3832 format!("supervisor rescan rejected daemon config: {err}"),
3833 )?]);
3834 }
3835 }
3836
3837 let configured_capabilities = modules
3838 .iter()
3839 .map(|module| (module.module_id.clone(), module.enabled))
3840 .collect::<Vec<_>>();
3841 let preview_capability_warnings = if preview {
3842 let (_, registrations) = self.runtime_capability_snapshot()?;
3843 let current_modules = self
3844 .supervisor
3845 .list()
3846 .into_iter()
3847 .map(|module| module.module_id().to_string())
3848 .collect::<BTreeSet<_>>();
3849 let resulting_modules = configured_capabilities.clone();
3850 let removed = current_modules
3851 .into_iter()
3852 .filter(|module_id| {
3853 !resulting_modules
3854 .iter()
3855 .any(|(configured_id, _)| configured_id == module_id)
3856 })
3857 .collect::<Vec<_>>();
3858 self.capability_evaluator.preview_removal_warnings(
3859 resulting_modules,
3860 &removed,
3861 ®istrations,
3862 )
3863 } else {
3864 Vec::new()
3865 };
3866 let result = match self
3867 .reconcile_supervised_modules(&context.supervisor, modules, preview)
3868 .await
3869 {
3870 Ok(result) => result,
3871 Err(message) => {
3872 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
3873 }
3874 };
3875 if !preview {
3876 self.capability_evaluator
3877 .configure(configured_capabilities, reserved_capabilities);
3878 self.capability_evaluator.wake_deadline_loop();
3879 self.refresh_capability_requirements();
3880 }
3881 let mut result = result;
3882 result.restart_required = restart_required;
3883 result.capability_warnings = preview_capability_warnings;
3884 let response = ClientControlResponse::SupervisorRescan { result };
3885 Ok(vec![control_response_body_frame(
3886 &frame,
3887 &response,
3888 "ClientControlResponse::SupervisorRescan",
3889 )?])
3890 }
3891
3892 async fn handle_supervisor_release_reserved(
3893 &self,
3894 frame: Frame,
3895 module_id: String,
3896 ) -> Result<Vec<Frame>, RouterError> {
3897 let Some(context) = self.rescan.clone() else {
3898 return Ok(vec![control_error_frame(
3899 &frame,
3900 "release_unavailable",
3901 "reserved-id release requires a daemon started with a reloadable config path",
3902 )?]);
3903 };
3904 let operation_lock = self.supervisor.operation_lock();
3905 let _operation_guard = operation_lock.lock().await;
3906 let loaded = match crate::daemon_config::load(&context.config_path) {
3907 Ok(Some(config)) => config,
3908 Ok(None) => {
3909 return Ok(vec![control_error_frame(
3910 &frame,
3911 "invalid_daemon_config",
3912 format!(
3913 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
3914 context.config_path.display()
3915 ),
3916 )?])
3917 }
3918 Err(err) => {
3919 return Ok(vec![control_error_frame(
3920 &frame,
3921 "invalid_daemon_config",
3922 format!("unable to verify reserved-id release against daemon config: {err}"),
3923 )?])
3924 }
3925 };
3926 if loaded
3927 .modules
3928 .iter()
3929 .any(|configured| configured.module_id == module_id)
3930 {
3931 return Ok(vec![control_error_frame(
3932 &frame,
3933 "reserved_module_configured",
3934 format!(
3935 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
3936 ),
3937 )?]);
3938 }
3939 if !self.supervisor.release_retained_reserved_gate(&module_id) {
3940 return Ok(vec![control_error_frame(
3941 &frame,
3942 "reserved_gate_not_retained",
3943 format!(
3944 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
3945 ),
3946 )?]);
3947 }
3948
3949 let response = ClientControlResponse::SupervisorAck {
3950 module_id,
3951 applied: true,
3952 };
3953 Ok(vec![control_response_body_frame(
3954 &frame,
3955 &response,
3956 "ClientControlResponse::SupervisorAck",
3957 )?])
3958 }
3959
3960 async fn reconcile_supervised_modules(
3969 &self,
3970 supervisor: &Supervisor,
3971 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
3972 preview: bool,
3973 ) -> Result<SupervisorRescanResult, String> {
3974 let mut current = BTreeMap::new();
3975 for module in self.supervisor.list() {
3976 let (spec, health) = module.configuration().map_err(|err| {
3977 format!(
3978 "failed to read configuration for module_id '{}': {err}",
3979 module.module_id()
3980 )
3981 })?;
3982 let enabled = module
3983 .status()
3984 .map_err(|err| {
3985 format!(
3986 "failed to read status for module_id '{}': {err}",
3987 module.module_id()
3988 )
3989 })?
3990 .enabled;
3991 current.insert(
3992 module.module_id().to_string(),
3993 (module, spec, health, enabled),
3994 );
3995 }
3996 let configured = configured_modules
3997 .into_iter()
3998 .map(|module| (module.module_id.clone(), module))
3999 .collect::<BTreeMap<_, _>>();
4000
4001 let added = configured
4002 .keys()
4003 .filter(|module_id| !current.contains_key(*module_id))
4004 .cloned()
4005 .collect::<Vec<_>>();
4006 let removed = current
4007 .keys()
4008 .filter(|module_id| !configured.contains_key(*module_id))
4009 .cloned()
4010 .collect::<Vec<_>>();
4011 let mut changed_pending_reload = Vec::new();
4012 let mut configuration_changes = BTreeSet::new();
4013 let mut enabled_changes = BTreeSet::new();
4014 let mut unchanged = 0_u32;
4015
4016 for (module_id, configured_module) in &configured {
4017 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
4018 else {
4019 continue;
4020 };
4021 let configuration_changed = *current_spec != configured_module.module_spec()
4022 || *current_health != configured_module.health;
4023 let enabled_changed = *current_enabled != configured_module.enabled;
4024 if configuration_changed {
4025 configuration_changes.insert(module_id.clone());
4026 changed_pending_reload.push(module_id.clone());
4027 }
4028 if enabled_changed {
4029 enabled_changes.insert(module_id.clone());
4030 }
4031 if !configuration_changed && !enabled_changed {
4032 unchanged = unchanged.saturating_add(1);
4033 }
4034 }
4035
4036 if preview {
4042 return Ok(SupervisorRescanResult {
4043 added,
4044 removed,
4045 changed_pending_reload,
4046 enabled_changes: enabled_changes.iter().cloned().collect(),
4047 unchanged,
4048 preview: true,
4049 restart_required: Vec::new(),
4053 capability_warnings: Vec::new(),
4054 });
4055 }
4056
4057 for module_id in &removed {
4058 let module = ¤t
4059 .get(module_id)
4060 .expect("removed module came from current supervisor state")
4061 .0;
4062 module.retire().await.map_err(|err| {
4063 format!("failed to retire module_id '{module_id}' during rescan: {err}")
4064 })?;
4065 self.supervisor.record_rescan_removal(module_id);
4099 self.supervisor.retire(module_id);
4100 }
4101
4102 for module_id in configured.keys() {
4103 let Some((module, _, _, _)) = current.get(module_id) else {
4104 continue;
4105 };
4106 let configured_module = configured
4107 .get(module_id)
4108 .expect("configured module id came from configured map");
4109 if configuration_changes.contains(module_id) {
4110 module
4111 .update_configuration(
4112 configured_module.module_spec(),
4113 configured_module.health,
4114 configured_module.drain_timeout_ms,
4115 )
4116 .await
4117 .map_err(|err| {
4118 format!(
4119 "failed to update module_id '{module_id}' configuration during rescan: {err}"
4120 )
4121 })?;
4122 }
4123 if enabled_changes.contains(module_id) {
4124 module
4125 .set_enabled(configured_module.enabled)
4126 .await
4127 .map_err(|err| {
4128 format!(
4129 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
4130 configured_module.enabled
4131 )
4132 })?;
4133 }
4134 }
4135
4136 for module_id in &added {
4137 let configured_module = configured
4138 .get(module_id)
4139 .expect("added module id came from configured map");
4140 supervisor
4141 .supervise_configured_with_health(
4142 configured_module.module_spec(),
4143 configured_module.enabled,
4144 configured_module.health,
4145 configured_module.drain_timeout_ms,
4146 configured_module.restart,
4147 )
4148 .map_err(|err| {
4149 format!("failed to add module_id '{module_id}' during rescan: {err}")
4150 })?;
4151 }
4152
4153 Ok(SupervisorRescanResult {
4154 added,
4155 removed,
4156 changed_pending_reload,
4157 enabled_changes: enabled_changes.iter().cloned().collect(),
4158 unchanged,
4159 preview: false,
4160 restart_required: Vec::new(),
4163 capability_warnings: Vec::new(),
4164 })
4165 }
4166
4167 async fn handle_supervisor_set_enabled(
4168 &self,
4169 frame: Frame,
4170 module_id: String,
4171 enabled: bool,
4172 ) -> Result<Vec<Frame>, RouterError> {
4173 let operation_lock = self.supervisor.operation_lock();
4174 let _operation_guard = operation_lock.lock().await;
4175 let Some(module) = self.supervisor.get(&module_id) else {
4176 return Ok(vec![control_error_frame(
4177 &frame,
4178 "unknown_module",
4179 format!("module_id '{module_id}' is not supervised"),
4180 )?]);
4181 };
4182
4183 let applied = match module.set_enabled(enabled).await {
4184 Ok(applied) => applied,
4185 Err(err) => {
4186 return Ok(vec![control_error_frame(
4187 &frame,
4188 "target_unavailable",
4189 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
4190 )?])
4191 }
4192 };
4193
4194 self.capability_evaluator.wake_deadline_loop();
4195 self.refresh_capability_requirements();
4196 let response = ClientControlResponse::SupervisorAck { module_id, applied };
4197 Ok(vec![control_response_body_frame(
4198 &frame,
4199 &response,
4200 "ClientControlResponse::SupervisorAck",
4201 )?])
4202 }
4203
4204 async fn handle_supervisor_health_probe(
4205 &self,
4206 frame: Frame,
4207 module_id: String,
4208 ) -> Result<Vec<Frame>, RouterError> {
4209 self.refresh_capability_requirements();
4210 let Some(registration) = self
4211 .registry
4212 .get_module(&module_id)
4213 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4214 else {
4215 return Ok(vec![control_error_frame(
4216 &frame,
4217 "unknown_module",
4218 format!("module_id '{module_id}' is not registered"),
4219 )?]);
4220 };
4221
4222 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
4235 {
4236 return Ok(vec![control_error_frame(
4237 &frame,
4238 "health_not_advertised",
4239 format!("module_id '{module_id}' did not advertise health.check"),
4240 )?]);
4241 }
4242
4243 let deadline = Instant::now() + self.health_probe_timeout;
4244 let pending = match self.forwarding.begin_module_control_rpc_for(
4245 &module_id,
4246 MODULE_CONTROL_OP_HEALTH_CHECK,
4247 deadline,
4248 ) {
4249 Ok(pending) => pending,
4250 Err(err) => {
4251 return Ok(vec![control_error_frame(
4252 &frame,
4253 forwarding_error_code(&err),
4254 err.to_string(),
4255 )?])
4256 }
4257 };
4258
4259 let PendingModuleControlRpc {
4260 endpoint,
4261 module_sink,
4262 negotiated_ver,
4263 corr: probe_corr,
4264 receiver,
4265 } = pending;
4266 let mut guard =
4267 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
4268 let probe_body =
4269 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
4270 RouterError::backend(
4271 0,
4272 frame.header.corr,
4273 format!("failed to encode health.check request: {err}"),
4274 )
4275 })?;
4276 let probe_frame = Frame::build_with_version(
4277 negotiated_ver,
4278 FrameType::Request,
4279 control_flags(),
4280 0,
4281 0,
4282 probe_corr,
4283 probe_body,
4284 )
4285 .map_err(RouterError::FrameBuild)?;
4286
4287 if let Err(err) = module_sink.send(probe_frame).await {
4288 return Ok(vec![control_error_frame(
4289 &frame,
4290 "target_unavailable",
4291 err.to_string(),
4292 )?]);
4293 }
4294
4295 match timeout_at(deadline, receiver).await {
4296 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
4297 guard.disarm();
4298 let Some(report) = response.health_report() else {
4299 return Ok(vec![control_error_frame(
4300 &frame,
4301 "invalid_control_body",
4302 "health.check RPC returned a non-health response",
4303 )?]);
4304 };
4305 let HealthReport {
4310 status,
4311 detail,
4312 metrics,
4313 } = report;
4314 let capability_detail = self
4315 .capability_evaluator
4316 .required_problem_detail(&module_id);
4317 let response = ClientControlResponse::SupervisorHealthProbe {
4318 module_id,
4319 status,
4320 detail: append_capability_problem_detail(detail, capability_detail),
4321 metrics,
4322 };
4323 Ok(vec![control_response_body_frame(
4324 &frame,
4325 &response,
4326 "ClientControlResponse::SupervisorHealthProbe",
4327 )?])
4328 }
4329 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
4330 guard.disarm();
4331 Ok(vec![control_error_body_frame(&frame, body)?])
4332 }
4333 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
4334 guard.disarm();
4335 Ok(vec![control_error_frame(
4336 &frame,
4337 "target_unavailable",
4338 message,
4339 )?])
4340 }
4341 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
4342 guard.disarm();
4343 Ok(vec![control_error_frame(
4344 &frame,
4345 "invalid_control_body",
4346 message,
4347 )?])
4348 }
4349 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
4350 guard.disarm();
4351 Ok(vec![control_error_frame(
4352 &frame,
4353 "invalid_control_body",
4354 format!("expected module-control op '{expected}', got '{actual}'"),
4355 )?])
4356 }
4357 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
4358 guard.disarm();
4359 Ok(vec![control_error_frame(
4360 &frame,
4361 "module_timeout",
4362 format!(
4363 "module_id '{module_id}' answered health.check after {:?}",
4364 self.health_probe_timeout
4365 ),
4366 )?])
4367 }
4368 Ok(Err(_)) => Ok(vec![control_error_frame(
4369 &frame,
4370 "target_unavailable",
4371 "health.check waiter was canceled before the module responded",
4372 )?]),
4373 Err(_) => Ok(vec![control_error_frame(
4374 &frame,
4375 "module_timeout",
4376 format!(
4377 "module_id '{module_id}' did not answer health.check within {:?}",
4378 self.health_probe_timeout
4379 ),
4380 )?]),
4381 }
4382 }
4383
4384 fn supervisor_status(
4385 &self,
4386 module_id: &str,
4387 corr: u64,
4388 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
4389 self.supervisor
4390 .get(module_id)
4391 .map(|module| {
4392 let warming = module.is_warming_for_control("status").map_err(|err| {
4393 RouterError::backend(
4394 0,
4395 corr,
4396 format!(
4397 "failed to read supervisor warming state for module_id '{module_id}': {err}"
4398 ),
4399 )
4400 })?;
4401 module.status_for_control("status").map_err(|err| {
4402 RouterError::backend(
4403 0,
4404 corr,
4405 format!(
4406 "failed to read supervisor status for module_id '{module_id}': {err}"
4407 ),
4408 )
4409 }).map(|status| (status, warming))
4410 })
4411 .transpose()
4412 }
4413
4414 fn guard_module_control_op(
4415 &self,
4416 frame: &Frame,
4417 module_id: &str,
4418 op: &str,
4419 ) -> Result<Option<Frame>, RouterError> {
4420 if self.module_grants_op(module_id, op, frame.header.corr)? {
4421 return Ok(None);
4422 }
4423
4424 Ok(Some(control_error_frame(
4425 frame,
4426 "op_not_allowed",
4427 format!("module_id '{module_id}' did not grant control op '{op}'"),
4428 )?))
4429 }
4430
4431 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
4432 let Some(registration) = self
4433 .registry
4434 .get_module(module_id)
4435 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
4436 else {
4437 return Ok(false);
4438 };
4439 Ok(module_registration_grants_op(®istration.control_ops, op))
4440 }
4441
4442 fn handle_status_update(
4443 &self,
4444 endpoint: ModuleEndpointId,
4445 frame: Frame,
4446 ) -> Result<Vec<Frame>, RouterError> {
4447 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
4448 Ok(update) => update,
4449 Err(err) => {
4450 if is_known_module_push_op(&frame.body) {
4455 return Ok(vec![control_error_frame(
4456 &frame,
4457 "invalid_control_body",
4458 format!("malformed module control push body: {err}"),
4459 )?]);
4460 }
4461 return Ok(Vec::new());
4462 }
4463 };
4464
4465 match update {
4466 ModuleControlPush::RouteStatus {
4467 route_channel,
4468 route_epoch,
4469 status,
4470 } => {
4471 self.forwarding
4472 .cache_status(endpoint, route_channel, route_epoch, status)
4473 .map_err(RouterError::Forwarding)?;
4474 }
4475 }
4476 Ok(Vec::new())
4477 }
4478
4479 fn handle_route_poll(
4480 &self,
4481 ctx: &RouteCtx,
4482 frame: Frame,
4483 route_channel: u16,
4484 route_epoch: u32,
4485 kind: PollKind,
4486 ) -> Result<Vec<Frame>, RouterError> {
4487 let snapshot = self
4488 .forwarding
4489 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
4490 .map_err(RouterError::Forwarding)?;
4491 let response = match (kind, snapshot) {
4492 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
4493 ClientControlResponse::RoutePoll {
4494 route_channel,
4495 route_epoch,
4496 status,
4497 live: None,
4498 }
4499 }
4500 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
4501 route_channel,
4502 route_epoch,
4503 status: None,
4504 live: None,
4505 },
4506 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
4507 let live = self
4533 .process_liveness
4534 .as_ref()
4535 .and_then(|source| source.process_live(&module_id))
4536 .unwrap_or(true);
4537 ClientControlResponse::RoutePoll {
4538 route_channel,
4539 route_epoch,
4540 status: None,
4541 live: Some(live),
4542 }
4543 }
4544 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
4545 route_channel,
4546 route_epoch,
4547 status: None,
4548 live: Some(false),
4549 },
4550 };
4551
4552 Ok(vec![control_response_body_frame(
4553 &frame,
4554 &response,
4555 "ClientControlResponse::RoutePoll",
4556 )?])
4557 }
4558
4559 pub(crate) fn observe_module_control_completion(
4560 &self,
4561 completion: ModuleControlRpcCompletion,
4562 ) -> bool {
4563 match completion {
4564 ModuleControlRpcCompletion::Unknown => false,
4565 ModuleControlRpcCompletion::Settled => true,
4566 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
4567 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
4568 info!(
4569 module_id = %module_id,
4570 latency_ms,
4571 "late health.check answer proves the module is alive"
4572 );
4573 match self
4574 .supervisor
4575 .record_late_health_answer(&module_id, latency_ms)
4576 {
4577 Ok(true) => {}
4578 Ok(false) => debug!(
4579 module_id = %module_id,
4580 latency_ms,
4581 "late health.check answer has no active supervisor snapshot"
4582 ),
4583 Err(err) => warn!(
4584 module_id = %module_id,
4585 latency_ms,
4586 error = %err,
4587 "failed to record late health.check answer"
4588 ),
4589 }
4590 true
4591 }
4592 }
4593 }
4594
4595 fn refuse_to_end_module_connection_for_a_client(
4616 &self,
4617 module_connection_id: ConnectionId,
4618 corr: u64,
4619 err: ForwardingError,
4620 ) -> Result<(), RouterError> {
4621 if let ForwardingError::ConnectionClosing { connection_id } = err {
4622 if connection_id != module_connection_id {
4623 warn!(
4624 module_connection_id = module_connection_id.get(),
4625 client_connection_id = connection_id.get(),
4626 corr,
4627 "dropping a route.bind response for a closing client; the module connection keeps serving"
4628 );
4629 return Ok(());
4630 }
4631 }
4632 Err(RouterError::Forwarding(err))
4633 }
4634
4635 fn handle_module_relay_response(
4636 &self,
4637 connection_id: ConnectionId,
4638 frame: Frame,
4639 ) -> Result<Vec<Frame>, RouterError> {
4640 let mut secondary_error = None;
4641 let outcome = match frame.header.ty {
4642 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
4643 Ok(probe) if probe.op == "route.bind" => {
4644 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
4645 Ok(ModuleControlResponse::RouteBindAck {}) => {
4646 RouteBindRelayOutcome::Accepted
4647 }
4648 Ok(other) => {
4649 let message =
4650 format!("route.bind response carried unexpected body: {other:?}");
4651 secondary_error = Some(control_error_frame(
4652 &frame,
4653 "invalid_control_body",
4654 message.clone(),
4655 )?);
4656 RouteBindRelayOutcome::ModuleGone(message)
4657 }
4658 Err(err) => {
4659 let message = format!("malformed route.bind response body: {err}");
4660 secondary_error = Some(control_error_frame(
4661 &frame,
4662 "invalid_control_body",
4663 message.clone(),
4664 )?);
4665 RouteBindRelayOutcome::ModuleGone(message)
4666 }
4667 }
4668 }
4669 Ok(probe) => {
4670 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
4671 {
4672 Ok(response) => ModuleControlRpcOutcome::Response(response),
4673 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
4674 "malformed {} response body: {err}",
4675 probe.op
4676 )),
4677 };
4678 let completion = self
4679 .forwarding
4680 .complete_module_control_rpc(
4681 connection_id,
4682 frame.header.corr,
4683 Some(&probe.op),
4684 outcome,
4685 )
4686 .map_err(RouterError::Forwarding)?;
4687 if !self.observe_module_control_completion(completion) {
4688 debug!(
4689 connection_id = connection_id.get(),
4690 corr = frame.header.corr,
4691 op = %probe.op,
4692 "dropping late or unknown module-control RPC response"
4693 );
4694 }
4695 return Ok(Vec::new());
4696 }
4697 Err(err) => {
4698 if let Some(expected_op) = self
4699 .forwarding
4700 .pending_module_control_op(connection_id, frame.header.corr)
4701 .map_err(RouterError::Forwarding)?
4702 {
4703 let completion = self
4704 .forwarding
4705 .complete_module_control_rpc(
4706 connection_id,
4707 frame.header.corr,
4708 None,
4709 ModuleControlRpcOutcome::MalformedResponse(format!(
4710 "malformed {expected_op} response body: {err}"
4711 )),
4712 )
4713 .map_err(RouterError::Forwarding)?;
4714 if !self.observe_module_control_completion(completion) {
4715 debug!(
4716 connection_id = connection_id.get(),
4717 corr = frame.header.corr,
4718 "dropping late malformed module-control RPC response"
4719 );
4720 }
4721 return Ok(Vec::new());
4722 }
4723 let message = format!("malformed route.bind response body: {err}");
4724 secondary_error = Some(control_error_frame(
4725 &frame,
4726 "invalid_control_body",
4727 message.clone(),
4728 )?);
4729 RouteBindRelayOutcome::ModuleGone(message)
4730 }
4731 },
4732 FrameType::Error => {
4733 if self
4734 .forwarding
4735 .pending_module_control_op(connection_id, frame.header.corr)
4736 .map_err(RouterError::Forwarding)?
4737 .is_some()
4738 {
4739 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
4740 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
4741 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
4742 "malformed module-control ERROR body: {err}"
4743 )),
4744 };
4745 let completion = self
4746 .forwarding
4747 .complete_module_control_rpc(
4748 connection_id,
4749 frame.header.corr,
4750 None,
4751 outcome,
4752 )
4753 .map_err(RouterError::Forwarding)?;
4754 if !self.observe_module_control_completion(completion) {
4755 debug!(
4756 connection_id = connection_id.get(),
4757 corr = frame.header.corr,
4758 "dropping late or unknown module-control RPC error"
4759 );
4760 }
4761 return Ok(Vec::new());
4762 }
4763 match serde_json::from_slice::<ErrorBody>(&frame.body) {
4764 Ok(body) => RouteBindRelayOutcome::Rejected(body),
4765 Err(err) => {
4766 let message = format!("malformed route.bind ERROR body: {err}");
4767 secondary_error = Some(control_error_frame(
4768 &frame,
4769 "invalid_control_body",
4770 message.clone(),
4771 )?);
4772 RouteBindRelayOutcome::ModuleGone(message)
4773 }
4774 }
4775 }
4776 ty => {
4777 return Ok(vec![control_error_frame(
4778 &frame,
4779 "unsupported_control_frame",
4780 format!("unsupported module channel-0 frame {ty:?}"),
4781 )?])
4782 }
4783 };
4784
4785 let settled =
4786 self.forwarding
4787 .complete_pending_relay(connection_id, frame.header.corr, outcome);
4788 let completion = match settled {
4789 Ok(completion) => completion,
4790 Err(err) => {
4791 self.refuse_to_end_module_connection_for_a_client(
4792 connection_id,
4793 frame.header.corr,
4794 err,
4795 )?;
4796 return Ok(secondary_error.into_iter().collect());
4797 }
4798 };
4799 if let Some(target) = completion.abandoned.as_ref() {
4800 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
4801 }
4802 if !completion.settled {
4803 debug!(
4804 connection_id = connection_id.get(),
4805 corr = frame.header.corr,
4806 frame_type = ?frame.header.ty,
4807 "dropping late or unknown route.bind relay response"
4808 );
4809 }
4810 Ok(secondary_error.into_iter().collect())
4811 }
4812
4813 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
4814 debug!(connection_id = connection_id.get(), "handling GOODBYE");
4815 let registrations = self
4816 .deregister_connection(connection_id)
4817 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
4818 let released_routes = self
4819 .forwarding
4820 .cleanup_connection(connection_id)
4821 .map_err(RouterError::Forwarding)?;
4822 self.emit_route_goodbyes(released_routes);
4823 if !registrations.is_empty() {
4825 crate::supervise::notify_registration_release();
4826 }
4827 Ok(Vec::new())
4828 }
4829}
4830
4831impl Default for ControlHandler {
4832 fn default() -> Self {
4833 Self::new(Arc::new(Registry::default()))
4834 }
4835}
4836
4837impl crate::supervise::SwapPromotionObserver for ControlHandler {
4838 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
4839 self.apply_registration_capabilities(registration);
4840 }
4841}
4842
4843fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
4844 CapabilityRequirementStatus {
4845 consumer: status.consumer,
4846 capability: status.capability,
4847 need: match status.need {
4848 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
4849 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
4850 },
4851 verdict: status.verdict.as_str().to_string(),
4852 episode_seq: status.episode_seq,
4853 config_satisfiable: status.config_satisfiable,
4854 runtime_available: status.runtime_available,
4855 detail: status.detail,
4856 }
4857}
4858
4859fn append_capability_problem_detail(
4860 detail: Option<String>,
4861 capability_detail: Option<String>,
4862) -> Option<String> {
4863 match (detail, capability_detail) {
4864 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
4865 (Some(detail), None) => Some(detail),
4866 (None, Some(capability_detail)) => Some(capability_detail),
4867 (None, None) => None,
4868 }
4869}
4870
4871fn subc_ops() -> Vec<String> {
4872 SUBC_CONTROL_OPS
4873 .iter()
4874 .map(|op| (*op).to_string())
4875 .collect()
4876}
4877
4878fn module_subc_ops() -> Vec<String> {
4879 SUBC_CONTROL_OPS
4880 .iter()
4881 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
4882 .map(|op| (*op).to_string())
4883 .collect()
4884}
4885
4886#[cfg(test)]
4887fn module_baseline_control_ops() -> Vec<String> {
4888 MODULE_BASELINE_CONTROL_OPS
4889 .iter()
4890 .map(|op| (*op).to_string())
4891 .collect()
4892}
4893
4894fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
4895 let mut seen = HashSet::new();
4896 let mut effective = Vec::new();
4897 for op in MODULE_BASELINE_CONTROL_OPS {
4898 if seen.insert((*op).to_string()) {
4899 effective.push((*op).to_string());
4900 }
4901 }
4902 for op in declared.unwrap_or_default() {
4903 if seen.insert(op.clone()) {
4904 effective.push(op);
4905 }
4906 }
4907 effective
4908}
4909
4910fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
4911 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
4912}
4913
4914fn target_module_id(target: &RouteTarget) -> &str {
4915 match target {
4916 RouteTarget::ToolProvider { module_id }
4917 | RouteTarget::ManagementSurface { module_id }
4918 | RouteTarget::InternalService { module_id, .. } => module_id,
4919 }
4920}
4921
4922fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
4923 roles.iter().any(|role| match (target, role) {
4924 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
4925 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
4926 (
4927 RouteTarget::InternalService { service_id, .. },
4928 ProviderRole::InternalService {
4929 service_id: provided,
4930 ..
4931 },
4932 ) => service_id == provided,
4933 _ => false,
4934 })
4935}
4936
4937fn is_routable_role(role: &ProviderRole) -> bool {
4938 matches!(
4939 role,
4940 ProviderRole::ToolProvider { .. }
4941 | ProviderRole::ManagementSurface { .. }
4942 | ProviderRole::InternalService { .. }
4943 )
4944}
4945
4946#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4947enum ControlRequestBodyError {
4948 UnknownOp,
4949 InvalidBody,
4950}
4951
4952#[derive(Debug, Deserialize)]
4953struct ControlOpProbe {
4954 op: String,
4955}
4956
4957const MODULE_PUSH_OPS: &[&str] = &["route.status"];
4960
4961fn is_known_module_push_op(body: &[u8]) -> bool {
4962 serde_json::from_slice::<ControlOpProbe>(body)
4963 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
4964 .unwrap_or(false)
4965}
4966
4967fn is_known_module_request_op(body: &[u8]) -> bool {
4968 serde_json::from_slice::<ControlOpProbe>(body)
4969 .map(|probe| MODULE_TO_SUBC_CONTROL_OPS.contains(&probe.op.as_str()))
4970 .unwrap_or(false)
4971}
4972
4973fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
4974 debug!(
4975 op = %op,
4976 connection_id = connection_id.get(),
4977 corr,
4978 "control dispatch"
4979 );
4980}
4981
4982fn log_slow_control_dispatch(
4983 dispatch_started_at: Option<StdInstant>,
4984 op: &'static str,
4985 connection_id: ConnectionId,
4986 corr: u64,
4987) {
4988 let Some(dispatch_started_at) = dispatch_started_at else {
4989 return;
4990 };
4991 let elapsed = dispatch_started_at.elapsed();
4992 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
4993 warn!(
4994 op = %op,
4995 connection_id = connection_id.get(),
4996 corr,
4997 elapsed_ms = elapsed.as_millis() as u64,
4998 "slow control dispatch"
4999 );
5000 }
5001}
5002
5003fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
5004 match request {
5005 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
5006 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
5007 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
5008 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
5009 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
5010 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
5011 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
5012 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
5013 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
5014 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
5015 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
5016 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
5017 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
5018 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
5019 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
5020 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
5021 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
5022 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
5023 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
5024 }
5025}
5026
5027fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
5028 match request {
5029 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
5030 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
5031 }
5032}
5033
5034fn parse_client_control_request(
5035 body: &[u8],
5036) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
5037 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
5038 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5039 Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
5040 ControlRequestBodyError::InvalidBody
5041 }
5042 Ok(_) => ControlRequestBodyError::UnknownOp,
5043 Err(_) => ControlRequestBodyError::InvalidBody,
5044 };
5045 (err, classification)
5046 })
5047}
5048
5049fn parse_module_control_request_from_module(
5050 body: &[u8],
5051) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
5052 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
5053 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5054 Ok(probe) if MODULE_TO_SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
5055 ControlRequestBodyError::InvalidBody
5056 }
5057 Ok(_) => ControlRequestBodyError::UnknownOp,
5058 Err(_) => ControlRequestBodyError::InvalidBody,
5059 };
5060 (err, classification)
5061 })
5062}
5063
5064#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
5065enum ProviderRoleKind {
5066 ToolProvider,
5067 PipelineStage,
5068 ManagementSurface,
5069 InternalService,
5070}
5071
5072fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
5073 match role {
5074 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
5075 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
5076 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
5077 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
5078 }
5079}
5080
5081fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
5082 roles.iter().map(provider_role_kind).collect()
5083}
5084
5085fn capability_census_trigger(
5089 old: Option<&CapabilityDeclarations>,
5090 new: Option<&CapabilityDeclarations>,
5091) -> bool {
5092 let old_provides = old
5093 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
5094 .unwrap_or_default();
5095 let old_denies = old
5096 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
5097 .unwrap_or_default();
5098 let new = new.cloned().unwrap_or(CapabilityDeclarations {
5099 provides: Vec::new(),
5100 requires: Vec::new(),
5101 must_never_reach: Vec::new(),
5102 });
5103
5104 new.provides
5105 .iter()
5106 .any(|capability| !old_provides.contains(capability))
5107 || new
5108 .must_never_reach
5109 .iter()
5110 .any(|capability| !old_denies.contains(capability))
5111}
5112
5113fn denied_capability<'a>(
5116 opening_manifest: &'a ModuleManifest,
5117 target_manifest: &ModuleManifest,
5118) -> Option<&'a str> {
5119 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
5120 let target_capabilities = target_manifest.capabilities.as_ref()?;
5121 opening_capabilities
5122 .must_never_reach
5123 .iter()
5124 .find(|denied| {
5125 target_capabilities
5126 .provides
5127 .iter()
5128 .any(|provided| provided == *denied)
5129 })
5130 .map(String::as_str)
5131}
5132
5133fn catalog_update_frozen_field_message(
5134 registered: &ModuleManifest,
5135 provides: &[ProviderRole],
5136) -> Option<String> {
5137 let old_has_provides = !registered.provides.is_empty();
5138 let new_has_provides = !provides.is_empty();
5139 if old_has_provides != new_has_provides {
5140 return Some(format!(
5141 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
5142 registered.module_id
5143 ));
5144 }
5145
5146 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
5147 return Some(format!(
5148 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
5149 registered.module_id
5150 ));
5151 }
5152
5153 let registered_concurrency = manifest_concurrency(registered);
5154 let mut candidate = registered.clone();
5155 candidate.provides = provides.to_vec();
5156 let candidate_concurrency = manifest_concurrency(&candidate);
5157 if candidate_concurrency != registered_concurrency {
5158 return Some(format!(
5159 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
5160 registered.module_id, registered_concurrency, candidate_concurrency
5161 ));
5162 }
5163
5164 None
5167}
5168
5169fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
5170 manifest.provides.iter().any(is_routable_role)
5171}
5172
5173fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
5179 manifest
5180 .provides
5181 .iter()
5182 .find_map(|provider| match provider {
5183 ProviderRole::ToolProvider { concurrency, .. }
5184 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
5185 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
5186 })
5187 .unwrap_or(Concurrency::ModuleManaged)
5188}
5189
5190fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
5197 let has_management_surface = manifest
5198 .provides
5199 .iter()
5200 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
5201 if !has_management_surface {
5202 return false;
5203 }
5204 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
5205 return false;
5206 };
5207 let Some(provides) = raw
5208 .get("manifest")
5209 .and_then(|manifest| manifest.get("provides"))
5210 .and_then(serde_json::Value::as_array)
5211 else {
5212 return false;
5213 };
5214 provides.iter().any(|role| {
5219 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
5220 && role.get("concurrency").is_none()
5221 })
5222}
5223
5224fn negotiate_version(peer_version: u8) -> Result<u8, String> {
5225 if peer_version != PROTOCOL_VERSION {
5226 return Err(format!(
5227 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
5228 ));
5229 }
5230 Ok(PROTOCOL_VERSION)
5231}
5232
5233fn pong(frame: &Frame) -> Result<Frame, RouterError> {
5234 Frame::build_with_version(
5235 response_version(frame),
5236 FrameType::Pong,
5237 frame.header.flags,
5238 0,
5239 0,
5240 frame.header.corr,
5241 Vec::new(),
5242 )
5243 .map_err(RouterError::FrameBuild)
5244}
5245
5246fn control_error_frame(
5247 frame: &Frame,
5248 code: &'static str,
5249 message: impl Into<String>,
5250) -> Result<Frame, RouterError> {
5251 control_error_body_frame(
5252 frame,
5253 ErrorBody {
5254 code: code.to_string(),
5255 message: message.into(),
5256 detail: None,
5257 },
5258 )
5259}
5260
5261fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
5262 let body = serde_json::to_vec(&error).map_err(|err| {
5263 RouterError::backend(
5264 0,
5265 frame.header.corr,
5266 format!("failed to encode control ERROR: {err}"),
5267 )
5268 })?;
5269
5270 Frame::build_with_version(
5271 response_version(frame),
5272 FrameType::Error,
5273 control_flags(),
5274 0,
5275 0,
5276 frame.header.corr,
5277 body,
5278 )
5279 .map_err(RouterError::FrameBuild)
5280}
5281
5282fn control_response_body_frame<T: Serialize>(
5283 frame: &Frame,
5284 reply: &T,
5285 label: &'static str,
5286) -> Result<Frame, RouterError> {
5287 let body = serde_json::to_vec(reply).map_err(|err| {
5288 RouterError::backend(
5289 0,
5290 frame.header.corr,
5291 format!("failed to encode {label}: {err}"),
5292 )
5293 })?;
5294
5295 Frame::build_with_version(
5296 response_version(frame),
5297 FrameType::Response,
5298 control_flags(),
5299 0,
5300 0,
5301 frame.header.corr,
5302 body,
5303 )
5304 .map_err(RouterError::FrameBuild)
5305}
5306
5307fn forwarding_error_code(err: &ForwardingError) -> &'static str {
5337 match err {
5338 ForwardingError::NoModuleConnection => "target_unavailable",
5339 ForwardingError::ModuleReloading { .. } => "module_reloading",
5340 ForwardingError::ClientRouteChannelExhausted { .. }
5341 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
5342 ForwardingError::StaleModuleEndpoint
5343 | ForwardingError::UnknownReservation { .. }
5344 | ForwardingError::ConnectionClosing { .. }
5345 | ForwardingError::ClientEgressClosed { .. }
5346 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
5347 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
5350 ForwardingError::RelayCorrelationExhausted
5351 | ForwardingError::RouteOpenBuild(_)
5352 | ForwardingError::Poisoned => "forwarding_error",
5353 }
5354}
5355
5356fn response_version(frame: &Frame) -> u8 {
5357 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
5358 frame.header.ver
5359 } else {
5360 PROTOCOL_VERSION
5361 }
5362}
5363
5364fn control_flags() -> Flags {
5365 Flags::new(false, Priority::Passive, false)
5366}
5367
5368fn send_goodbye_target_best_effort(
5373 counters: &DaemonCounters,
5374 target: &GoodbyeTarget,
5375 context: &'static str,
5376) {
5377 let Ok(frame) = Frame::build_with_version(
5378 target.negotiated_ver,
5379 FrameType::Goodbye,
5380 control_flags(),
5381 target.channel,
5382 target.epoch,
5383 0,
5384 Vec::new(),
5385 ) else {
5386 return;
5387 };
5388 crate::forwarding::send_module_route_goodbye(
5389 counters,
5390 &target.sink,
5391 frame,
5392 target.module_id.as_deref(),
5393 context,
5394 );
5395}
5396
5397pub(crate) fn send_route_control_pushes(
5398 forwarding: &ForwardingTable,
5399 routes: Vec<EndpointRoute>,
5400 push: ClientControlPush,
5401) {
5402 let body = match serde_json::to_vec(&push) {
5403 Ok(body) => body,
5404 Err(err) => {
5405 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
5406 return;
5407 }
5408 };
5409 let mut targets = Vec::new();
5410 for route in routes {
5411 let target = route.goodbye_target;
5412 if let Some(existing) = targets
5413 .iter()
5414 .find(|existing: &&GoodbyeTarget| existing.connection_id == target.connection_id)
5415 {
5416 debug_assert_eq!(
5417 existing.negotiated_ver, target.negotiated_ver,
5418 "one connection cannot negotiate multiple frame versions"
5419 );
5420 continue;
5421 }
5422 targets.push(target);
5423 }
5424 for target in targets {
5425 let frame = match Frame::build_with_version(
5426 target.negotiated_ver,
5427 FrameType::Push,
5428 control_flags(),
5429 0,
5430 0,
5431 0,
5432 body.clone(),
5433 ) {
5434 Ok(frame) => frame,
5435 Err(err) => {
5436 warn!(
5437 route_channel = target.channel,
5438 error = %err,
5439 "failed to build route lifecycle control PUSH frame"
5440 );
5441 continue;
5442 }
5443 };
5444 if let Err(err) = target.sink.try_send(frame) {
5445 if target.close_on_delivery_failure() {
5446 warn!(
5447 target_connection_id = target.connection_id.get(),
5448 route_channel = target.channel,
5449 error = %err,
5450 "route lifecycle control PUSH was not delivered to client; closing target connection"
5451 );
5452 let _ = forwarding.escalate_client_delivery_failure(
5453 target.connection_id,
5454 target.channel,
5455 target.epoch,
5456 CloseReason::new(
5457 "route_lifecycle_push_delivery_failed",
5458 format!(
5459 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
5460 target.channel
5461 ),
5462 ),
5463 crate::forwarding::UndeliveredFrame {
5464 module_id: target.module_id.as_deref(),
5465 sink: &target.sink,
5466 },
5467 );
5468 }
5469 }
5470 }
5471}
5472
5473#[cfg(test)]
5474mod tests {
5475 use std::{
5476 collections::BTreeMap,
5477 fmt,
5478 path::PathBuf,
5479 sync::{Arc, Mutex},
5480 time::Duration,
5481 };
5482
5483 use serde_json::{json, Value};
5484 use subc_protocol::{
5485 manifest::{
5486 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
5487 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
5488 },
5489 session::HealthStatus,
5490 FrameType,
5491 };
5492
5493 use super::*;
5494 use crate::{
5495 forwarding::{DataRoute, DataRouteState},
5496 registry::ChannelState,
5497 router::FrameSink,
5498 stderr_tail::DEFAULT_MAX_LINE_BYTES,
5499 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
5500 test_support::TestTempDir,
5501 RouteCtx, Router,
5502 };
5503 use tokio::{
5504 sync::mpsc,
5505 time::{sleep, Instant},
5506 };
5507 use tracing::{
5508 field::{Field, Visit},
5509 Event, Subscriber,
5510 };
5511 use tracing_subscriber::{layer::Context, prelude::*, Layer};
5512
5513 fn fake_aft_stub_path() -> PathBuf {
5532 let mut path = std::env::current_exe().expect("current_exe available in tests");
5533 path.pop(); path.pop(); path.push(if cfg!(windows) {
5536 "fake-aft-stub.exe"
5537 } else {
5538 "fake-aft-stub"
5539 });
5540 assert!(
5541 path.exists(),
5542 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
5543 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
5544 path.display()
5545 );
5546 path
5547 }
5548
5549 fn client_retries(code: &str) -> bool {
5554 subc_protocol::error_codes::is_retryable_route_open(code)
5555 }
5556
5557 #[test]
5568 fn retryability_of_forwarding_codes_matches_the_failure() {
5569 let transient = [
5572 ForwardingError::NoModuleConnection,
5573 ForwardingError::ModuleReloading {
5574 module_id: "m".into(),
5575 },
5576 ForwardingError::StaleModuleEndpoint,
5577 ForwardingError::UnknownReservation {
5578 client_channel: 1,
5579 module_channel: 1,
5580 },
5581 ForwardingError::ConnectionClosing {
5582 connection_id: ConnectionId::new(1),
5583 },
5584 ForwardingError::ClientEgressClosed {
5585 connection_id: ConnectionId::new(1),
5586 },
5587 ForwardingError::ModuleEgressUnavailable {
5588 connection_id: ConnectionId::new(1),
5589 },
5590 ];
5591 for err in transient {
5592 let code = forwarding_error_code(&err);
5593 assert!(
5594 client_retries(code),
5595 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
5596 );
5597 }
5598
5599 let permanent = [
5604 ForwardingError::ClientRouteChannelExhausted {
5605 connection_id: ConnectionId::new(1),
5606 },
5607 ForwardingError::ModuleRouteChannelExhausted {
5608 endpoint: ModuleEndpointId {
5609 connection_id: ConnectionId::new(1),
5610 generation: 1,
5611 },
5612 },
5613 ForwardingError::RelayCorrelationExhausted,
5614 ForwardingError::RouteOpenBuild("x".into()),
5615 ForwardingError::Poisoned,
5616 ];
5617 for err in permanent {
5618 let code = forwarding_error_code(&err);
5619 assert!(
5620 !client_retries(code),
5621 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
5622 );
5623 }
5624 }
5625
5626 #[tokio::test]
5654 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
5655 let handler = ControlHandler::default();
5656 let frame =
5657 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
5658
5659 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
5663 assert_eq!(
5664 stamped,
5665 Principal::Direct,
5666 "a caller that proved nothing must not be stamped as a supervised module"
5667 );
5668
5669 let forged = handler
5674 .route_open_principal(
5675 &frame,
5676 Some(ConsumerIdentity {
5677 module_id: "aft".to_string(),
5678 launch_nonce: "not-a-real-nonce".to_string(),
5679 }),
5680 )
5681 .unwrap();
5682 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
5683 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
5684 }
5685
5686 #[test]
5703 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
5704 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
5705 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
5706 let ClientControlRequest::RouteOpen {
5707 consumer_identity, ..
5708 } = parsed
5709 else {
5710 panic!("route.open body must parse as RouteOpen");
5711 };
5712 assert_eq!(
5713 consumer_identity,
5714 Some(ConsumerIdentity {
5715 module_id: "aft".to_string(),
5716 launch_nonce: "n".to_string(),
5717 }),
5718 "the wire field name must reach the value route_open_principal reads"
5719 );
5720 }
5721
5722 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
5723 ModuleManifest::builder(module_id, "0.1.0")
5724 .protocol_ver(protocol_ver)
5725 .provides(vec![ProviderRole::ToolProvider {
5726 tools: vec![Tool {
5727 name: "read".to_string(),
5728 description: None,
5729 execution_mode: ExecutionMode::Pure,
5730 schema: json!({"type": "object"}),
5731 }],
5732 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
5733 concurrency: Concurrency::ModuleManaged,
5734 emits_push: true,
5735 sub_supervises: true,
5736 }])
5737 .build()
5738 }
5739
5740 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
5741 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
5742 }
5743
5744 fn hello_frame_with_control_ops(
5745 module_id: &str,
5746 protocol_ver: u8,
5747 corr: u64,
5748 control_ops: Option<Vec<String>>,
5749 ) -> Frame {
5750 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
5751 }
5752
5753 fn hello_frame_with_nonce(
5754 module_id: &str,
5755 protocol_ver: u8,
5756 corr: u64,
5757 launch_nonce: Option<&str>,
5758 ) -> Frame {
5759 hello_frame_full(
5760 module_id,
5761 protocol_ver,
5762 corr,
5763 None,
5764 launch_nonce.map(ToOwned::to_owned),
5765 )
5766 }
5767
5768 fn hello_frame_full(
5769 module_id: &str,
5770 protocol_ver: u8,
5771 corr: u64,
5772 control_ops: Option<Vec<String>>,
5773 launch_nonce: Option<String>,
5774 ) -> Frame {
5775 let body = serde_json::to_vec(&ModuleHelloBody {
5776 manifest: manifest(module_id, protocol_ver),
5777 protocol_ver,
5778 control_ops,
5779 launch_nonce,
5780 })
5781 .unwrap();
5782 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
5783 }
5784
5785 fn non_routable_hello_frame_with_control_ops(
5786 module_id: &str,
5787 corr: u64,
5788 control_ops: Option<Vec<String>>,
5789 ) -> Frame {
5790 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
5791 manifest.provides.clear();
5792 let body = serde_json::to_vec(&ModuleHelloBody {
5793 manifest,
5794 protocol_ver: PROTOCOL_VERSION,
5795 control_ops,
5796 launch_nonce: None,
5797 })
5798 .unwrap();
5799 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
5800 }
5801
5802 fn capability_grammar_hello_frame(
5803 capabilities: Value,
5804 runtime_computed: Option<Value>,
5805 corr: u64,
5806 ) -> Frame {
5807 let mut body = serde_json::to_value(ModuleHelloBody {
5808 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
5809 protocol_ver: PROTOCOL_VERSION,
5810 control_ops: None,
5811 launch_nonce: None,
5812 })
5813 .expect("HELLO body serializes");
5814 body["manifest"]["capabilities"] = capabilities;
5815 if let Some(runtime_computed) = runtime_computed {
5816 body["runtime_computed"] = runtime_computed;
5817 }
5818 Frame::build(
5819 FrameType::Hello,
5820 control_flags(),
5821 0,
5822 0,
5823 corr,
5824 serde_json::to_vec(&body).expect("HELLO body reserializes"),
5825 )
5826 .expect("HELLO frame builds")
5827 }
5828
5829 fn channel_request(channel: u16, corr: u64) -> Frame {
5830 Frame::build(
5831 FrameType::Request,
5832 Flags::new(true, Priority::Interactive, false),
5833 channel,
5834 0,
5835 corr,
5836 b"opaque".to_vec(),
5837 )
5838 .unwrap()
5839 }
5840
5841 fn route_ctx(
5842 connection_id: ConnectionId,
5843 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
5844 let (tx, rx) = mpsc::channel(8);
5845 (
5846 RouteCtx {
5847 connection_id,
5848 egress: FrameSink::new(tx),
5849 },
5850 rx,
5851 )
5852 }
5853
5854 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
5855 serde_json::from_slice(&frame.body).unwrap()
5856 }
5857
5858 async fn hello_via_sink(
5863 handler: &ControlHandler,
5864 ctx: &RouteCtx,
5865 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
5866 hello: Frame,
5867 ) -> Frame {
5868 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
5869 assert!(
5870 replies.is_empty(),
5871 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
5872 );
5873 let ack = rx
5874 .try_recv()
5875 .expect("HELLO_ACK is queued on the module sink")
5876 .frame;
5877 assert_eq!(ack.header.ty, FrameType::HelloAck);
5878 ack
5879 }
5880
5881 fn parse_error(frame: &Frame) -> Value {
5882 serde_json::from_slice(&frame.body).unwrap()
5883 }
5884
5885 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
5886 serde_json::from_slice(&frame.body).unwrap()
5887 }
5888
5889 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
5890 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
5891 route_channel,
5892 route_epoch: 0,
5893 kind,
5894 })
5895 .unwrap();
5896 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
5897 }
5898
5899 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
5900 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
5901 module_id: module_id.to_string(),
5902 })
5903 .unwrap();
5904 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
5905 }
5906
5907 fn route_open_frame(corr: u64, module_id: &str, project_root: TestTempDir) -> Frame {
5908 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
5909 }
5910
5911 fn route_open_frame_with_consumer_capabilities(
5912 corr: u64,
5913 module_id: &str,
5914 project_root: TestTempDir,
5915 consumer_capabilities: Option<Vec<String>>,
5916 ) -> Frame {
5917 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
5918 target: RouteTarget::ToolProvider {
5919 module_id: module_id.to_string(),
5920 },
5921 identity: BindIdentity::new(
5922 project_root.path().to_path_buf(),
5923 "unit".to_string(),
5924 "session".to_string(),
5925 ),
5926 consumer_identity: None,
5927 consumer_capabilities,
5928 admission_facts: None,
5929 })
5930 .unwrap();
5931 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
5932 }
5933
5934 fn route_open_frame_with_admission_facts(
5935 corr: u64,
5936 module_id: &str,
5937 project_root: TestTempDir,
5938 consumer_identity: Option<subc_control::ConsumerIdentity>,
5939 facts: Option<Value>,
5940 ) -> Frame {
5941 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
5942 target: RouteTarget::ToolProvider {
5943 module_id: module_id.to_string(),
5944 },
5945 identity: BindIdentity::new(
5946 project_root.path().to_path_buf(),
5947 "unit".to_string(),
5948 format!("session-{corr}"),
5949 ),
5950 consumer_identity,
5951 consumer_capabilities: None,
5952 admission_facts: facts,
5953 })
5954 .unwrap();
5955 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
5956 }
5957
5958 #[derive(Clone, Default)]
5959 struct EventCapture {
5960 events: Arc<Mutex<Vec<CapturedEvent>>>,
5961 }
5962
5963 #[derive(Clone, Debug)]
5964 struct CapturedEvent {
5965 target: String,
5966 fields: BTreeMap<String, String>,
5967 }
5968
5969 impl EventCapture {
5970 fn events(&self) -> Vec<CapturedEvent> {
5971 self.events.lock().unwrap().clone()
5972 }
5973 }
5974
5975 impl<S> Layer<S> for EventCapture
5976 where
5977 S: Subscriber,
5978 {
5979 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
5980 let mut visitor = EventFieldVisitor::default();
5981 event.record(&mut visitor);
5982 self.events.lock().unwrap().push(CapturedEvent {
5983 target: event.metadata().target().to_string(),
5984 fields: visitor.fields,
5985 });
5986 }
5987 }
5988
5989 #[derive(Default)]
5990 struct EventFieldVisitor {
5991 fields: BTreeMap<String, String>,
5992 }
5993
5994 impl Visit for EventFieldVisitor {
5995 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
5996 self.fields
5997 .insert(field.name().to_string(), format!("{value:?}"));
5998 }
5999 }
6000
6001 fn health_response(corr: u64, status: HealthStatus) -> Frame {
6002 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
6003 status,
6004 detail: Some("warming".to_string()),
6005 metrics: Some(json!({"queue_depth": 3})),
6006 })
6007 .unwrap();
6008 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
6009 }
6010
6011 fn route_bind_ack(corr: u64) -> Frame {
6012 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
6013 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
6014 }
6015
6016 fn unique_project_root(label: &str) -> TestTempDir {
6017 TestTempDir::new(label)
6018 }
6019
6020 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
6021 match parse_route_poll(frame) {
6022 ClientControlResponse::RoutePoll {
6023 status: None,
6024 live: Some(live),
6025 ..
6026 } => assert_eq!(live, expected_live),
6027 other => panic!("unexpected route.poll response: {other:?}"),
6028 }
6029 }
6030
6031 fn bind_liveness_route(
6032 registry: &Registry,
6033 forwarding: &ForwardingTable,
6034 module_id: &str,
6035 ) -> (RouteCtx, u16, u32) {
6036 let module_connection = ConnectionId::new(101);
6037 let client_connection = ConnectionId::new(202);
6038 let registration = registry
6039 .register_with_control_ops(
6040 manifest(module_id, PROTOCOL_VERSION),
6041 PROTOCOL_VERSION,
6042 module_connection,
6043 module_baseline_control_ops(),
6044 )
6045 .unwrap();
6046 let (module_tx, _module_rx) = mpsc::channel(8);
6047 let endpoint = forwarding
6048 .register_module_connection(
6049 module_connection,
6050 module_id.to_string(),
6051 PROTOCOL_VERSION,
6052 manifest_concurrency(®istration.manifest),
6053 FrameSink::new(module_tx),
6054 )
6055 .unwrap();
6056 let (client_ctx, _client_rx) = route_ctx(client_connection);
6057 let pending = forwarding
6058 .begin_route_bind_relay_for_test(
6059 client_connection,
6060 client_ctx.egress.clone(),
6061 1,
6062 module_id,
6063 )
6064 .unwrap();
6065 assert_eq!(pending.endpoint, endpoint);
6066 let route_channel = pending.client_channel;
6067 let route_epoch = pending.client_epoch;
6068 forwarding
6069 .complete_pending_relay(
6070 module_connection,
6071 pending.corr,
6072 RouteBindRelayOutcome::Accepted,
6073 )
6074 .unwrap();
6075 (client_ctx, route_channel, route_epoch)
6076 }
6077
6078 struct FakeProcessLiveness {
6079 live: Option<bool>,
6080 }
6081
6082 impl ModuleProcessLiveness for FakeProcessLiveness {
6083 fn process_live(&self, _module_id: &str) -> Option<bool> {
6084 self.live
6085 }
6086 }
6087
6088 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6089 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
6090 {
6091 let registry = Arc::new(Registry::default());
6092 let supervisor_handle = SupervisorHandle::new();
6093 let supervisor = Supervisor::new(
6094 Arc::clone(®istry),
6095 RestartPolicy::new(1, Duration::from_millis(10)),
6096 )
6097 .with_handle(supervisor_handle.clone());
6098 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
6099 let module = supervisor
6100 .spawn(ModuleSpec {
6101 module_id: "stderr-tail-wire".to_string(),
6102 program: fake_aft_stub_path(),
6103 args: Vec::new(),
6104 env: vec![
6105 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
6106 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
6107 ],
6108 reserved: false,
6109 reserved_prefixes: Vec::new(),
6110 protocol: ModuleProtocol::Subc,
6111 overlap: Default::default(),
6112 })
6113 .unwrap();
6114
6115 let deadline = Instant::now() + Duration::from_secs(5);
6116 loop {
6117 let tail = module.stderr_tail(None, None);
6118 if tail
6119 .entries
6120 .iter()
6121 .any(|entry| matches!(entry, TailEntry::ProcessStart))
6122 && tail.entries.iter().any(|entry| {
6123 matches!(
6124 entry,
6125 TailEntry::Line {
6126 truncated: true,
6127 ..
6128 }
6129 )
6130 })
6131 {
6132 break;
6133 }
6134 assert!(
6135 Instant::now() < deadline,
6136 "module did not produce a truncated line and restart boundary: {tail:?}"
6137 );
6138 sleep(Duration::from_millis(10)).await;
6139 }
6140
6141 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
6142 let request = ClientControlRequest::SupervisorStderrTail {
6143 module_id: "stderr-tail-wire".to_string(),
6144 max_lines: None,
6145 max_bytes: None,
6146 };
6147 let frame = Frame::build(
6148 FrameType::Request,
6149 control_flags(),
6150 0,
6151 0,
6152 1,
6153 serde_json::to_vec(&request).unwrap(),
6154 )
6155 .unwrap();
6156 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6157 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
6158 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
6159 serde_json::from_slice(&responses[0].body).unwrap()
6160 else {
6161 panic!("expected supervisor.stderr_tail response");
6162 };
6163
6164 assert!(
6165 tail.entries
6166 .iter()
6167 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
6168 "the control response lost the restart boundary"
6169 );
6170 let Some(StderrTailEntry::Line { text, truncated }) = tail.entries.iter().find(|entry| {
6171 matches!(
6172 entry,
6173 StderrTailEntry::Line {
6174 truncated: true,
6175 ..
6176 }
6177 )
6178 }) else {
6179 panic!("the control response lost the truncated line");
6180 };
6181 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
6182 assert!(*truncated);
6183 }
6184
6185 #[tokio::test(flavor = "current_thread")]
6190 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
6191 let dir = TestTempDir::new("terminals-off-worker");
6192 let journal_path = dir.join("terminals.jsonl");
6193 let registry = Arc::new(Registry::default());
6194 let supervisor_handle = SupervisorHandle::new();
6195 let supervisor =
6196 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
6197 .with_handle(supervisor_handle.clone())
6198 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
6199 let module = supervisor
6200 .spawn(ModuleSpec {
6201 module_id: "terminal-off-worker".to_string(),
6202 program: fake_aft_stub_path(),
6203 args: Vec::new(),
6204 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
6205 reserved: false,
6206 reserved_prefixes: Vec::new(),
6207 protocol: ModuleProtocol::Subc,
6208 overlap: Default::default(),
6209 })
6210 .unwrap();
6211 let deadline = Instant::now() + Duration::from_secs(5);
6212 while module.terminal_history().entries.len() != 2 {
6213 assert!(Instant::now() < deadline, "module did not record two exits");
6214 sleep(Duration::from_millis(10)).await;
6215 }
6216
6217 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
6218 let handler =
6219 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
6220 let frame = Frame::build(
6221 FrameType::Request,
6222 control_flags(),
6223 0,
6224 0,
6225 1,
6226 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
6227 module_id: "terminal-off-worker".to_string(),
6228 })
6229 .unwrap(),
6230 )
6231 .unwrap();
6232 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6233 let spawned_at = std::time::Instant::now();
6234 let read = tokio::spawn({
6235 let handler = Arc::clone(&handler);
6236 async move { handler.handle_control_frame(&ctx, frame).await }
6237 });
6238 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
6241 .await
6242 .unwrap()
6243 .expect("the history read reached its pause");
6244 let elapsed = spawned_at.elapsed();
6245 assert!(
6246 elapsed < Duration::from_secs(2) && !read.is_finished(),
6247 "this task could not run while the history read was paused \
6248 (resumed after {elapsed:?}, read finished: {})",
6249 read.is_finished()
6250 );
6251
6252 drop(release);
6253 let responses = read.await.unwrap().unwrap();
6254 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
6255 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
6256 panic!("expected supervisor.terminals response");
6257 };
6258 assert_eq!(terminals.entries.len(), 2);
6259 assert_eq!(terminals.journal_skipped_lines, 0);
6260 assert_eq!(terminals.journal_read_errors, 0);
6261 }
6262
6263 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6264 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
6265 let registry = Arc::new(Registry::default());
6266 let supervisor_handle = SupervisorHandle::new();
6267 let supervisor =
6268 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
6269 .with_handle(supervisor_handle.clone());
6270 let module = supervisor
6271 .spawn(ModuleSpec {
6272 module_id: "terminal-golden".to_string(),
6273 program: fake_aft_stub_path(),
6274 args: Vec::new(),
6275 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
6276 reserved: false,
6277 reserved_prefixes: Vec::new(),
6278 protocol: ModuleProtocol::Subc,
6279 overlap: Default::default(),
6280 })
6281 .unwrap();
6282
6283 let deadline = Instant::now() + Duration::from_secs(5);
6284 while module.terminal_history().entries.len() != 2 {
6285 assert!(
6286 Instant::now() < deadline,
6287 "module did not retain two terminal exits: {:?}",
6288 module.terminal_history()
6289 );
6290 sleep(Duration::from_millis(10)).await;
6291 }
6292
6293 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
6294 let request = ClientControlRequest::SupervisorTerminals {
6295 module_id: "terminal-golden".to_string(),
6296 };
6297 let frame = Frame::build(
6298 FrameType::Request,
6299 control_flags(),
6300 0,
6301 0,
6302 1,
6303 serde_json::to_vec(&request).unwrap(),
6304 )
6305 .unwrap();
6306 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6307 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
6308 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
6309 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
6310 panic!("expected supervisor.terminals response");
6311 };
6312 assert_eq!(terminals.entries.len(), 2);
6313 assert_eq!(terminals.dropped, 0);
6314
6315 let mut rendered = serde_json::to_value(response).unwrap();
6316 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
6319 for (index, entry) in rendered["entries"]
6320 .as_array_mut()
6321 .expect("terminal response entries array")
6322 .iter_mut()
6323 .enumerate()
6324 {
6325 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
6326 }
6327
6328 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
6329 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
6330 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
6331 if std::env::var_os("UPDATE_GOLDEN").is_some() {
6332 std::fs::write(&golden_path, &serialized).unwrap();
6333 }
6334 let expected: Value =
6335 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
6336 assert_eq!(rendered, expected);
6337 }
6338
6339 #[test]
6340 fn hello_registers_manifest_and_returns_ack() {
6341 let registry = Arc::new(Registry::default());
6342 let handler = ControlHandler::new(Arc::clone(®istry));
6343 let conn = ConnectionId::new(1);
6344
6345 let responses = handler
6346 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
6347 .unwrap();
6348
6349 assert_eq!(responses.len(), 1);
6350 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
6351 assert_eq!(responses[0].header.channel, 0);
6352 assert_eq!(responses[0].header.corr, 7);
6353 let ack = parse_ack(&responses[0]);
6354 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
6355 assert!(ack
6356 .subc_capabilities
6357 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
6358 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
6359 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
6360 assert!(ack
6361 .subc_ops
6362 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
6363 assert!(ack
6364 .subc_ops
6365 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
6366
6367 let registration = registry.get_module("aft").unwrap().unwrap();
6368 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
6369 assert_eq!(registration.state, ChannelState::Active);
6370 assert_eq!(registration.connection_id, conn);
6371 assert_eq!(registration.control_ops, module_baseline_control_ops());
6372 }
6373
6374 #[test]
6375 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
6376 let invalid_identifiers = [
6377 ("case_change", "credentials-Provider/v1"),
6378 ("leading_zero", "credentials-provider/v01"),
6379 ("trailing_hyphen", "credentials-provider-/v1"),
6380 ("consecutive_hyphens", "credentials--provider/v1"),
6381 ("uppercase", "Credentials-provider/v1"),
6382 ("missing_v", "credentials-provider/1"),
6383 ("whitespace", "credentials provider/v1"),
6384 ("zero_version", "credentials-provider/v0"),
6385 ("out_of_range_version", "credentials-provider/v4294967296"),
6386 (
6387 "overlength_name",
6388 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
6389 ),
6390 ];
6391 let mut cases = invalid_identifiers
6392 .into_iter()
6393 .map(|(name, identifier)| {
6394 (
6395 format!("identifier_{name}"),
6396 "capabilities.provides[0]".to_string(),
6397 identifier.to_string(),
6398 json!({ "provides": [identifier] }),
6399 None,
6400 )
6401 })
6402 .collect::<Vec<_>>();
6403 cases.extend([
6404 (
6405 "unknown_need".to_string(),
6406 "capabilities.requires[0].need".to_string(),
6407 "deferred".to_string(),
6408 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
6409 None,
6410 ),
6411 (
6412 "duplicate_provides".to_string(),
6413 "capabilities.provides[1]".to_string(),
6414 "credentials-provider/v1".to_string(),
6415 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
6416 None,
6417 ),
6418 (
6419 "duplicate_must_never_reach".to_string(),
6420 "capabilities.must_never_reach[1]".to_string(),
6421 "credentials-provider/v1".to_string(),
6422 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
6423 None,
6424 ),
6425 (
6426 "duplicate_requires_same_need".to_string(),
6427 "capabilities.requires[1]".to_string(),
6428 "credentials-provider/v1".to_string(),
6429 json!({ "requires": [
6430 { "capability": "credentials-provider/v1", "need": "required" },
6431 { "capability": "credentials-provider/v1", "need": "required" }
6432 ] }),
6433 None,
6434 ),
6435 (
6436 "duplicate_requires_conflicting_need".to_string(),
6437 "capabilities.requires[1]".to_string(),
6438 "credentials-provider/v1".to_string(),
6439 json!({ "requires": [
6440 { "capability": "credentials-provider/v1", "need": "required" },
6441 { "capability": "credentials-provider/v1", "need": "optional" }
6442 ] }),
6443 None,
6444 ),
6445 (
6446 "capabilities_root_pointer".to_string(),
6447 "runtime_computed[0]".to_string(),
6448 "/capabilities".to_string(),
6449 json!({}),
6450 Some(json!(["/capabilities"])),
6451 ),
6452 (
6453 "capabilities_descendant_pointer".to_string(),
6454 "runtime_computed[0]".to_string(),
6455 "/capabilities/provides".to_string(),
6456 json!({}),
6457 Some(json!(["/capabilities/provides"])),
6458 ),
6459 (
6460 "malformed_pointer_without_leading_slash".to_string(),
6461 "runtime_computed[0]".to_string(),
6462 "capabilities".to_string(),
6463 json!({}),
6464 Some(json!(["capabilities"])),
6465 ),
6466 (
6467 "malformed_pointer_escape".to_string(),
6468 "runtime_computed[0]".to_string(),
6469 "/roles/~2/tools".to_string(),
6470 json!({}),
6471 Some(json!(["/roles/~2/tools"])),
6472 ),
6473 (
6474 "unknown_capabilities_field".to_string(),
6475 "capabilities.future".to_string(),
6476 "<array>".to_string(),
6477 json!({ "future": [] }),
6478 None,
6479 ),
6480 ]);
6481
6482 for (index, (name, field, value, capabilities, runtime_computed)) in
6483 cases.into_iter().enumerate()
6484 {
6485 let registry = Arc::new(Registry::default());
6486 let handler = ControlHandler::new(Arc::clone(®istry));
6487 let response = handler
6488 .handle_control(
6489 ConnectionId::new((index + 1) as u64),
6490 capability_grammar_hello_frame(
6491 capabilities,
6492 runtime_computed,
6493 index as u64 + 1,
6494 ),
6495 )
6496 .expect("invalid HELLO returns a refusal");
6497
6498 assert_eq!(response.len(), 1, "{name} must emit one refusal");
6499 let error = parse_error(&response[0]);
6500 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
6501 let message = error["message"]
6502 .as_str()
6503 .expect("error message is a string");
6504 assert!(
6505 message.contains(&field),
6506 "{name}: field missing from {message}"
6507 );
6508 assert!(
6509 message.contains(&value),
6510 "{name}: value missing from {message}"
6511 );
6512 assert_eq!(
6513 registry
6514 .active_registration_count()
6515 .expect("registry reads"),
6516 0,
6517 "{name}: refused HELLO must not create a catalog entry"
6518 );
6519 }
6520 }
6521
6522 #[test]
6523 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
6524 let registry = Arc::new(Registry::default());
6525 let handler = ControlHandler::new(Arc::clone(®istry));
6526 let capabilities = json!({
6527 "provides": ["credentials-provider/v1"],
6528 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
6529 "must_never_reach": ["federation-transport/v1"]
6530 });
6531 let response = handler
6532 .handle_control(
6533 ConnectionId::new(99),
6534 capability_grammar_hello_frame(
6535 capabilities.clone(),
6536 Some(json!(["/roles/0/tools"])),
6537 99,
6538 ),
6539 )
6540 .expect("valid HELLO registers");
6541 assert_eq!(response[0].header.ty, FrameType::HelloAck);
6542
6543 let request = Frame::build(
6544 FrameType::Request,
6545 control_flags(),
6546 0,
6547 0,
6548 100,
6549 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
6550 .expect("catalog request serializes"),
6551 )
6552 .expect("catalog request frame builds");
6553 let response = handler
6554 .handle_catalog_list(request, None)
6555 .expect("catalog list succeeds");
6556 let ClientControlResponse::CatalogList { modules, .. } =
6557 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
6558 else {
6559 panic!("catalog request must return catalog.list");
6560 };
6561 assert_eq!(modules.len(), 1);
6562 assert_eq!(
6563 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
6564 capabilities
6565 );
6566 }
6567
6568 #[test]
6569 fn catalog_list_mirrors_management_operation_description() {
6570 let registry = Arc::new(Registry::default());
6571 let handler = ControlHandler::new(Arc::clone(®istry));
6572 let description = "List managed records and return their identifiers and metadata.";
6573 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
6574 manifest.provides = vec![ProviderRole::ManagementSurface {
6575 operations: vec![ManagementOperation {
6576 name: "records.list".to_string(),
6577 kind: ManagementOperationKind::Query,
6578 description: Some(description.to_string()),
6579 }],
6580 config_schema: json!({"type": "object"}),
6581 observability: vec![ObservabilitySurface {
6582 name: "records.stats".to_string(),
6583 kind: ObservabilityKind::Snapshot,
6584 }],
6585 identity_scope: vec![IdentityScope::Project],
6586 concurrency: Concurrency::ModuleManaged,
6587 }];
6588 registry
6589 .register_with_control_ops(
6590 manifest,
6591 PROTOCOL_VERSION,
6592 ConnectionId::new(99),
6593 Vec::new(),
6594 )
6595 .expect("described management manifest registers");
6596
6597 let request = Frame::build(
6598 FrameType::Request,
6599 control_flags(),
6600 0,
6601 0,
6602 100,
6603 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
6604 .expect("catalog request serializes"),
6605 )
6606 .expect("catalog request frame builds");
6607 let response = handler
6608 .handle_catalog_list(request, None)
6609 .expect("catalog list succeeds");
6610 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
6611 assert_eq!(
6612 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
6613 "catalog.list must preserve the declared operation description verbatim"
6614 );
6615 }
6616
6617 #[test]
6618 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
6619 let registry = Arc::new(Registry::default());
6620 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
6621 [("vault".to_string(), true), ("squatter".to_string(), true)],
6622 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
6623 );
6624 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
6625 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
6626 provides: vec!["credentials-provider/v1".to_string()],
6627 requires: Vec::new(),
6628 must_never_reach: Vec::new(),
6629 });
6630 let frame = Frame::build(
6631 FrameType::Hello,
6632 control_flags(),
6633 0,
6634 0,
6635 77,
6636 serde_json::to_vec(&ModuleHelloBody {
6637 manifest: squatter,
6638 protocol_ver: PROTOCOL_VERSION,
6639 control_ops: None,
6640 launch_nonce: None,
6641 })
6642 .expect("HELLO serializes"),
6643 )
6644 .expect("HELLO frame builds");
6645 let response = handler
6646 .handle_control(ConnectionId::new(77), frame)
6647 .expect("reserved claim receives a typed refusal");
6648 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
6649 assert_eq!(
6650 registry
6651 .active_registration_count()
6652 .expect("registry reads"),
6653 0,
6654 "a reserved capability refusal must not leave a catalog entry"
6655 );
6656 }
6657
6658 #[test]
6659 fn server_describe_surfaces_required_capability_verdict_fields() {
6660 let registry = Arc::new(Registry::default());
6661 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
6662 [
6663 ("consumer".to_string(), true),
6664 ("provider".to_string(), false),
6665 ],
6666 BTreeMap::new(),
6667 );
6668 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
6669 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
6670 provides: Vec::new(),
6671 requires: vec![subc_protocol::manifest::CapabilityRequirement {
6672 capability: "credentials-provider/v1".to_string(),
6673 need: subc_protocol::manifest::CapabilityNeed::Required,
6674 }],
6675 must_never_reach: Vec::new(),
6676 });
6677 let hello = Frame::build(
6678 FrameType::Hello,
6679 control_flags(),
6680 0,
6681 0,
6682 78,
6683 serde_json::to_vec(&ModuleHelloBody {
6684 manifest: consumer,
6685 protocol_ver: PROTOCOL_VERSION,
6686 control_ops: None,
6687 launch_nonce: None,
6688 })
6689 .expect("HELLO serializes"),
6690 )
6691 .expect("HELLO frame builds");
6692 handler
6693 .handle_control(ConnectionId::new(78), hello)
6694 .expect("consumer registers");
6695 let describe = Frame::build(
6696 FrameType::Request,
6697 control_flags(),
6698 0,
6699 0,
6700 79,
6701 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
6702 .expect("request serializes"),
6703 )
6704 .expect("describe frame builds");
6705 let response = handler
6706 .handle_server_describe(describe)
6707 .expect("server.describe succeeds");
6708 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
6709 let requirement = &rendered["capability_requirements"][0];
6710 assert_eq!(requirement["consumer"], "consumer");
6711 assert_eq!(requirement["verdict"], "never_provided");
6712 assert_eq!(requirement["episode_seq"], 1);
6713 assert_eq!(requirement["config_satisfiable"], false);
6714 assert_eq!(requirement["runtime_available"], false);
6715 assert!(requirement["detail"]
6716 .as_str()
6717 .expect("detail string")
6718 .contains("credentials-provider/v1"));
6719 }
6720
6721 #[test]
6722 fn catalog_list_omits_capabilities_for_legacy_manifest() {
6723 let registry = Arc::new(Registry::default());
6724 let handler = ControlHandler::new(Arc::clone(®istry));
6725 let hello = handler
6726 .handle_control(
6727 ConnectionId::new(101),
6728 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
6729 )
6730 .expect("legacy HELLO registers");
6731 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
6732
6733 let request = Frame::build(
6734 FrameType::Request,
6735 control_flags(),
6736 0,
6737 0,
6738 102,
6739 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
6740 .expect("catalog request serializes"),
6741 )
6742 .expect("catalog request frame builds");
6743 let response = handler
6744 .handle_catalog_list(request, None)
6745 .expect("catalog list succeeds");
6746 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
6747 assert!(
6748 body["modules"][0].get("capabilities").is_none(),
6749 "legacy manifest must retain an absent capabilities field on catalog.list"
6750 );
6751 }
6752
6753 #[test]
6754 fn hello_ack_omits_storage_when_no_storage_config() {
6755 let registry = Arc::new(Registry::default());
6756 let handler = ControlHandler::new(Arc::clone(®istry));
6757 let responses = handler
6758 .handle_control(
6759 ConnectionId::new(1),
6760 hello_frame("aft", PROTOCOL_VERSION, 7),
6761 )
6762 .unwrap();
6763 let ack = parse_ack(&responses[0]);
6764 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
6765 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
6766 }
6767
6768 #[tokio::test]
6769 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
6770 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
6771 let registry = Arc::new(Registry::default());
6772 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
6773 let responses = handler
6774 .handle_control(
6775 ConnectionId::new(1),
6776 hello_frame("aft", PROTOCOL_VERSION, 7),
6777 )
6778 .unwrap();
6779 let ack = parse_ack(&responses[0]);
6780 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
6781
6782 let described = handler
6783 .handle_control_frame(
6784 &route_ctx(ConnectionId::new(2)).0,
6785 Frame::build(
6786 FrameType::Request,
6787 control_flags(),
6788 0,
6789 0,
6790 9,
6791 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
6792 )
6793 .unwrap(),
6794 )
6795 .await
6796 .unwrap();
6797 let ClientControlResponse::ServerDescribe { machine_id, .. } =
6798 serde_json::from_slice(&described[0].body).unwrap()
6799 else {
6800 panic!("server.describe answered with another shape");
6801 };
6802 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
6803 }
6804
6805 #[test]
6806 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
6807 let registry = Arc::new(Registry::default());
6810 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
6811 crate::daemon_config::StorageConfig::Sqlite {
6812 data_home: std::path::PathBuf::from("/data"),
6813 },
6814 ));
6815
6816 let responses = handler
6817 .handle_control(
6818 ConnectionId::new(1),
6819 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
6820 )
6821 .unwrap();
6822 let ack = parse_ack(&responses[0]);
6823 assert_eq!(
6824 ack.storage,
6825 Some(serde_json::json!({
6826 "module_id": "alfonso-routing",
6827 "storage_namespace": "default",
6828 "isolation": { "kind": "module" },
6829 "backend": {
6830 "backend": "sqlite",
6831 "path": "/data/cortexkit/alfonso-routing/store.db"
6832 }
6833 })),
6834 "the delivered descriptor is the module's own sqlite store path"
6835 );
6836 }
6837
6838 #[test]
6839 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
6840 let registry = Arc::new(Registry::default());
6841 let handler = ControlHandler::new(Arc::clone(®istry));
6842 let conn = ConnectionId::new(1);
6843 let responses = handler
6844 .handle_control(
6845 conn,
6846 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
6847 )
6848 .unwrap();
6849 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
6850 let registration = registry.get_module("aft").unwrap().unwrap();
6851 assert_eq!(registration.control_ops, module_baseline_control_ops());
6852
6853 let frame =
6854 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
6855 assert!(handler
6856 .guard_module_control_op(&frame, "aft", "route.bind")
6857 .unwrap()
6858 .is_none());
6859 let error = handler
6860 .guard_module_control_op(&frame, "aft", "test.synthetic")
6861 .unwrap()
6862 .expect("synthetic ungranted op should be rejected");
6863 assert_eq!(error.header.ty, FrameType::Error);
6864 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
6865 }
6866
6867 #[test]
6868 fn hello_control_ops_some_adds_optional_grants() {
6869 let registry = Arc::new(Registry::default());
6870 let handler = ControlHandler::new(Arc::clone(®istry));
6871 handler
6872 .handle_control(
6873 ConnectionId::new(1),
6874 hello_frame_with_control_ops(
6875 "aft",
6876 PROTOCOL_VERSION,
6877 7,
6878 Some(vec![
6879 "future.synthetic".to_string(),
6880 "route.bind".to_string(),
6881 ]),
6882 ),
6883 )
6884 .unwrap();
6885 let registration = registry.get_module("aft").unwrap().unwrap();
6886 assert_eq!(
6887 registration.control_ops,
6888 vec![
6889 "route.bind".to_string(),
6890 "route.status".to_string(),
6891 "future.synthetic".to_string(),
6892 ]
6893 );
6894 let frame =
6895 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
6896 assert!(handler
6897 .guard_module_control_op(&frame, "aft", "future.synthetic")
6898 .unwrap()
6899 .is_none());
6900 }
6901
6902 #[tokio::test]
6903 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
6904 let registry = Arc::new(Registry::default());
6905 let forwarding = Arc::new(ForwardingTable::default());
6906 let handler =
6907 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
6908 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
6909 hello_via_sink(
6910 &handler,
6911 &module_ctx,
6912 &mut module_rx,
6913 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
6914 )
6915 .await;
6916
6917 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
6918 let responses = handler
6919 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
6920 .await
6921 .unwrap();
6922 assert_eq!(responses.len(), 1);
6923 assert_eq!(responses[0].header.ty, FrameType::Error);
6924 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
6925 assert!(module_rx.try_recv().is_err());
6926 }
6927
6928 #[tokio::test]
6929 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
6930 let registry = Arc::new(Registry::default());
6931 let forwarding = Arc::new(ForwardingTable::default());
6932 let handler =
6933 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
6934 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
6935 hello_via_sink(
6936 &handler,
6937 &module_ctx,
6938 &mut module_rx,
6939 hello_frame_with_control_ops(
6940 "aft",
6941 PROTOCOL_VERSION,
6942 7,
6943 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
6944 ),
6945 )
6946 .await;
6947
6948 let project_root = unique_project_root("demux");
6949 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
6950 let route_handler = handler.clone();
6951 let route_task = tokio::spawn(async move {
6952 route_handler
6953 .handle_control_frame(
6954 &route_client_ctx,
6955 route_open_frame(100, "aft", project_root),
6956 )
6957 .await
6958 .unwrap()
6959 });
6960 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
6961 .await
6962 .unwrap()
6963 .unwrap();
6964 assert!(matches!(
6965 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
6966 ModuleControlRequest::RouteBind { .. }
6967 ));
6968
6969 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
6970 let health_handler = handler.clone();
6971 let health_task = tokio::spawn(async move {
6972 health_handler
6973 .handle_control_frame(
6974 &health_client_ctx,
6975 supervisor_health_probe_frame(101, "aft"),
6976 )
6977 .await
6978 .unwrap()
6979 });
6980 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
6981 .await
6982 .unwrap()
6983 .unwrap();
6984 assert_eq!(
6985 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
6986 ModuleControlRequest::HealthCheck {}
6987 );
6988
6989 handler
6990 .handle_control_frame(
6991 &module_ctx,
6992 health_response(health_frame.header.corr, HealthStatus::Degraded),
6993 )
6994 .await
6995 .unwrap();
6996 let health_response = health_task.await.unwrap();
6997 assert_eq!(health_response.len(), 1);
6998 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
6999 ClientControlResponse::SupervisorHealthProbe {
7000 module_id,
7001 status,
7002 detail,
7003 metrics,
7004 } => {
7005 assert_eq!(module_id, "aft");
7006 assert_eq!(status, HealthStatus::Degraded);
7007 assert_eq!(detail.as_deref(), Some("warming"));
7008 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
7009 }
7010 other => panic!("unexpected health response: {other:?}"),
7011 }
7012
7013 handler
7014 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
7015 .await
7016 .unwrap();
7017 let route_response = route_task.await.unwrap();
7018 assert!(route_response.is_empty());
7019 let published = route_client_rx.recv().await.unwrap();
7020 assert!(matches!(
7021 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
7022 ClientControlResponse::RouteOpen { .. }
7023 ));
7024 }
7025
7026 async fn relay_route_open(
7031 handler: &ControlHandler,
7032 client_connection: ConnectionId,
7033 client_egress: &FrameSink,
7034 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
7035 corr: u64,
7036 module_id: &str,
7037 project_root_label: &str,
7038 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
7039 let ctx = RouteCtx {
7040 connection_id: client_connection,
7041 egress: client_egress.clone(),
7042 };
7043 let handler = handler.clone();
7044 let project_root = unique_project_root(project_root_label);
7045 let module_id = module_id.to_string();
7046 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
7047 let task = tokio::spawn(async move {
7048 let _guard = tracing::dispatcher::set_default(&dispatch);
7049 handler
7050 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
7051 .await
7052 .unwrap()
7053 });
7054 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
7055 .await
7056 .expect("module receives the relayed route.bind")
7057 .expect("module egress is open");
7058 (task, bind.frame)
7059 }
7060
7061 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
7062 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
7063 ModuleControlRequest::RouteBind {
7064 route_channel,
7065 epoch,
7066 ..
7067 } => (route_channel, epoch),
7068 other => panic!("expected a route.bind request, got {other:?}"),
7069 }
7070 }
7071
7072 fn published_route(frame: &Frame) -> (u16, u32) {
7073 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
7074 ClientControlResponse::RouteOpen {
7075 route_channel,
7076 route_epoch,
7077 } => (route_channel, route_epoch),
7078 other => panic!("expected a route.open response, got {other:?}"),
7079 }
7080 }
7081
7082 #[tokio::test]
7102 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
7103 let registry = Arc::new(Registry::default());
7104 let forwarding = Arc::new(ForwardingTable::default());
7105 let handler =
7106 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
7107
7108 let module_connection = ConnectionId::new(30);
7109 let (module_ctx, mut module_rx) = route_ctx(module_connection);
7110 hello_via_sink(
7111 &handler,
7112 &module_ctx,
7113 &mut module_rx,
7114 hello_frame("aft", PROTOCOL_VERSION, 7),
7115 )
7116 .await;
7117
7118 let dying_client = ConnectionId::new(31);
7119 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
7120
7121 let (first_task, first_bind) = relay_route_open(
7124 &handler,
7125 dying_client,
7126 &dying_ctx.egress,
7127 &mut module_rx,
7128 100,
7129 "aft",
7130 "closing-first",
7131 )
7132 .await;
7133 handler
7134 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
7135 .await
7136 .unwrap();
7137 assert!(first_task.await.unwrap().is_empty());
7138 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
7139
7140 let (second_task, second_bind) = relay_route_open(
7142 &handler,
7143 dying_client,
7144 &dying_ctx.egress,
7145 &mut module_rx,
7146 101,
7147 "aft",
7148 "closing-second",
7149 )
7150 .await;
7151 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
7152
7153 assert!(forwarding
7156 .escalate_client_delivery_failure(
7157 dying_client,
7158 first_channel,
7159 first_epoch,
7160 CloseReason::new(
7161 "module_to_client_delivery_failed",
7162 "client egress refused a module frame",
7163 ),
7164 crate::forwarding::UndeliveredFrame {
7165 module_id: None,
7166 sink: &dying_ctx.egress,
7167 },
7168 )
7169 .unwrap());
7170 assert!(!dying_ctx.egress.is_closed());
7171
7172 let ack = handler
7174 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
7175 .await;
7176 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
7177 if module_loop_error.is_some() {
7178 handler.cleanup_connection(module_connection).unwrap();
7182 }
7183 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7187 .await
7188 .ok()
7189 .flatten();
7190
7191 assert!(
7193 registry
7194 .get_module_by_connection(module_connection)
7195 .unwrap()
7196 .is_some(),
7197 "one client's closing connection ended the shared module connection: \
7198 {module_loop_error:?}"
7199 );
7200 let cotenant = ConnectionId::new(32);
7202 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
7203 let (cotenant_task, cotenant_bind) = relay_route_open(
7204 &handler,
7205 cotenant,
7206 &cotenant_ctx.egress,
7207 &mut module_rx,
7208 102,
7209 "aft",
7210 "closing-cotenant",
7211 )
7212 .await;
7213 handler
7214 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
7215 .await
7216 .unwrap();
7217 assert!(cotenant_task.await.unwrap().is_empty());
7218 let (cotenant_channel, cotenant_epoch) =
7219 published_route(&cotenant_rx.recv().await.unwrap());
7220 assert!(matches!(
7221 forwarding
7222 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
7223 .unwrap(),
7224 DataRoute::Client(DataRouteState::Bound(_))
7225 ));
7226
7227 let goodbye = post_ack_module_frame
7230 .expect("module receives a GOODBYE for the abandoned route channel");
7231 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
7232 assert_eq!(goodbye.header.channel, abandoned_channel);
7233 assert_eq!(goodbye.header.epoch, abandoned_epoch);
7234
7235 assert!(dying_rx.try_recv().is_err());
7239 let second_response = second_task.await.unwrap();
7240 assert_eq!(second_response.len(), 1);
7241 assert_eq!(
7242 parse_error(&second_response[0])["code"],
7243 "target_unavailable"
7244 );
7245 }
7246
7247 #[test]
7254 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
7255 let handler = ControlHandler::default();
7256 let module_connection = ConnectionId::new(30);
7257 let client_connection = ConnectionId::new(31);
7258
7259 handler
7260 .refuse_to_end_module_connection_for_a_client(
7261 module_connection,
7262 77,
7263 ForwardingError::ConnectionClosing {
7264 connection_id: client_connection,
7265 },
7266 )
7267 .expect("a closing client must never end the module connection");
7268
7269 assert!(matches!(
7270 handler.refuse_to_end_module_connection_for_a_client(
7271 module_connection,
7272 78,
7273 ForwardingError::ConnectionClosing {
7274 connection_id: module_connection,
7275 },
7276 ),
7277 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
7278 connection_id
7279 })) if connection_id == module_connection
7280 ));
7281 assert!(matches!(
7282 handler.refuse_to_end_module_connection_for_a_client(
7283 module_connection,
7284 79,
7285 ForwardingError::Poisoned,
7286 ),
7287 Err(RouterError::Forwarding(ForwardingError::Poisoned))
7288 ));
7289 assert!(matches!(
7290 handler.refuse_to_end_module_connection_for_a_client(
7291 module_connection,
7292 80,
7293 ForwardingError::StaleModuleEndpoint,
7294 ),
7295 Err(RouterError::Forwarding(
7296 ForwardingError::StaleModuleEndpoint
7297 ))
7298 ));
7299 }
7300
7301 #[tokio::test]
7314 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
7315 let registry = Arc::new(Registry::default());
7316 let forwarding = Arc::new(ForwardingTable::default());
7317 let supervisor = SupervisorHandle::new();
7318 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7319 let handler =
7320 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
7321 .with_supervisor(supervisor);
7322
7323 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
7324 hello_via_sink(
7325 &handler,
7326 &target_ctx,
7327 &mut target_rx,
7328 hello_frame("target", PROTOCOL_VERSION, 1),
7329 )
7330 .await;
7331
7332 let wrong_nonce = handler
7336 .handle_control_frame(
7337 &route_ctx(ConnectionId::new(91)).0,
7338 route_open_frame_with_admission_facts(
7339 20,
7340 "target",
7341 unique_project_root("admission-facts"),
7342 Some(subc_control::ConsumerIdentity {
7343 module_id: "fed".to_string(),
7344 launch_nonce: "not-the-real-nonce".to_string(),
7345 }),
7346 None,
7347 ),
7348 )
7349 .await
7350 .unwrap();
7351 assert_eq!(
7352 parse_error(&wrong_nonce[0])["code"],
7353 "bad_consumer_identity",
7354 "a mismatched launch nonce must be refused, not stamped Reserved"
7355 );
7356
7357 let never_spawned = handler
7361 .handle_control_frame(
7362 &route_ctx(ConnectionId::new(92)).0,
7363 route_open_frame_with_admission_facts(
7364 21,
7365 "target",
7366 unique_project_root("admission-facts"),
7367 Some(subc_control::ConsumerIdentity {
7368 module_id: "never-spawned".to_string(),
7369 launch_nonce: "any-nonce".to_string(),
7370 }),
7371 None,
7372 ),
7373 )
7374 .await
7375 .unwrap();
7376 assert_eq!(
7377 parse_error(&never_spawned[0])["code"],
7378 "bad_consumer_identity",
7379 "an unspawned module_id must be refused rather than accepted for lack of a record"
7380 );
7381 }
7382
7383 #[tokio::test]
7406 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
7407 let registry = Arc::new(Registry::default());
7408 let forwarding = Arc::new(ForwardingTable::default());
7409 let supervisor = SupervisorHandle::new();
7410 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7411 let handler =
7412 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
7413 .with_supervisor(supervisor);
7414
7415 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
7416 hello_via_sink(
7417 &handler,
7418 &target_ctx,
7419 &mut target_rx,
7420 hello_frame("target", PROTOCOL_VERSION, 1),
7421 )
7422 .await;
7423
7424 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
7425 let route_handler = handler.clone();
7426 let route_task = tokio::spawn(async move {
7427 route_handler
7428 .handle_control_frame(
7429 &client_ctx,
7430 route_open_frame_with_admission_facts(
7431 30,
7432 "target",
7433 unique_project_root("admission-facts"),
7434 Some(subc_control::ConsumerIdentity {
7435 module_id: "fed".to_string(),
7436 launch_nonce: "fed-nonce".to_string(),
7437 }),
7438 None,
7439 ),
7440 )
7441 .await
7442 .unwrap()
7443 });
7444
7445 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7452 .await
7453 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
7454 .expect("module control channel closed before route.bind");
7455 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
7456 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
7457 panic!("expected route.bind")
7458 };
7459 assert_eq!(
7460 principal,
7461 Some(Principal::Reserved {
7462 module_id: "fed".to_string()
7463 }),
7464 "a correctly attested consumer must be stamped Reserved for its own id"
7465 );
7466
7467 handler
7468 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
7469 .await
7470 .unwrap();
7471 assert!(route_task.await.unwrap().is_empty());
7472 assert!(
7473 matches!(
7474 serde_json::from_slice::<ClientControlResponse>(
7475 &client_rx.recv().await.unwrap().body
7476 )
7477 .unwrap(),
7478 ClientControlResponse::RouteOpen { .. }
7479 ),
7480 "the route must actually open, not merely avoid an error"
7481 );
7482 }
7483
7484 #[tokio::test(start_paused = true)]
7485 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
7486 let registry = Arc::new(Registry::default());
7487 let forwarding = Arc::new(ForwardingTable::default());
7488 let supervisor = SupervisorHandle::new();
7489 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7490 let handler =
7491 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
7492 .with_supervisor(supervisor);
7493
7494 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
7495 hello_via_sink(
7496 &handler,
7497 &target_ctx,
7498 &mut target_rx,
7499 hello_frame("target", PROTOCOL_VERSION, 1),
7500 )
7501 .await;
7502
7503 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
7504 let direct_handler = handler.clone();
7505 let direct_open = tokio::spawn(async move {
7506 direct_handler
7507 .handle_control_frame(
7508 &direct_ctx,
7509 route_open_frame(2, "target", unique_project_root("route-census-direct")),
7510 )
7511 .await
7512 .unwrap()
7513 });
7514 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7515 .await
7516 .expect("no direct route.bind within 5s")
7517 .expect("target control channel closed before direct route.bind");
7518 handler
7519 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
7520 .await
7521 .unwrap();
7522 assert!(direct_open.await.unwrap().is_empty());
7523 let _ = direct_rx.recv().await.unwrap();
7524
7525 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
7526 let reserved_handler = handler.clone();
7527 let reserved_open = tokio::spawn(async move {
7528 reserved_handler
7529 .handle_control_frame(
7530 &reserved_ctx,
7531 route_open_frame_with_admission_facts(
7532 3,
7533 "target",
7534 unique_project_root("admission-facts"),
7535 Some(ConsumerIdentity {
7536 module_id: "fed".to_string(),
7537 launch_nonce: "fed-nonce".to_string(),
7538 }),
7539 None,
7540 ),
7541 )
7542 .await
7543 .unwrap()
7544 });
7545 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7546 .await
7547 .expect("no reserved route.bind within 5s")
7548 .expect("target control channel closed before reserved route.bind");
7549 handler
7550 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
7551 .await
7552 .unwrap();
7553 assert!(reserved_open.await.unwrap().is_empty());
7554 let _ = reserved_rx.recv().await.unwrap();
7555
7556 forwarding
7557 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
7558 .unwrap();
7559 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
7560 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
7561 module_id: Some("target".to_string()),
7562 })
7563 .unwrap();
7564 let census_frame =
7565 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
7566 let response = handler
7567 .handle_control_frame(&census_ctx, census_frame)
7568 .await
7569 .unwrap()
7570 .pop()
7571 .unwrap();
7572 let actual: Value = serde_json::from_slice(&response.body).unwrap();
7573 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
7574 assert!(matches!(
7575 decoded,
7576 ClientControlResponse::SupervisorRoutes { .. }
7577 ));
7578 let routes = actual["modules"][0]["routes"].as_array().unwrap();
7579 assert_eq!(routes.len(), 2);
7580 assert!(routes.iter().all(|route| route["draining"] == true));
7581 assert!(
7584 routes.iter().all(|route| route["drain_reason"] == "reload"),
7585 "draining routes must name the drain's reason: {routes:?}"
7586 );
7587 assert!(routes.iter().any(|route| {
7588 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
7589 }));
7590 assert!(routes.iter().any(|route| {
7591 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
7592 }));
7593
7594 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
7595 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
7596 if std::env::var_os("UPDATE_GOLDEN").is_some() {
7597 std::fs::write(
7598 &golden_path,
7599 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
7600 )
7601 .unwrap();
7602 }
7603 let expected: Value =
7604 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
7605 assert_eq!(actual, expected);
7606 }
7607
7608 async fn query_live_roots(
7609 handler: &ControlHandler,
7610 module_ctx: &RouteCtx,
7611 ) -> ModuleControlResponseToModule {
7612 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
7613 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
7614 let response = handler
7615 .handle_control_frame(module_ctx, frame)
7616 .await
7617 .unwrap()
7618 .pop()
7619 .unwrap();
7620 serde_json::from_slice(&response.body).unwrap()
7621 }
7622
7623 #[tokio::test(start_paused = true)]
7624 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
7625 let registry = Arc::new(Registry::default());
7626 let forwarding = Arc::new(ForwardingTable::default());
7627 let handler = ControlHandler::with_forwarding(registry, forwarding);
7628 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
7629 hello_via_sink(
7630 &handler,
7631 &target_ctx,
7632 &mut target_rx,
7633 hello_frame("target", PROTOCOL_VERSION, 1),
7634 )
7635 .await;
7636 let root = unique_project_root("live-roots-known");
7637 let path = ProjectRootId::from_path_allowing_missing(root.path())
7638 .unwrap()
7639 .as_path()
7640 .to_path_buf();
7641 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
7642 let open_handler = handler.clone();
7643 let opened = tokio::spawn(async move {
7644 open_handler
7645 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
7646 .await
7647 .unwrap()
7648 });
7649 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7650 .await
7651 .unwrap()
7652 .unwrap();
7653 handler
7654 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
7655 .await
7656 .unwrap();
7657 assert!(opened.await.unwrap().is_empty());
7658 let _ = client_rx.recv().await.unwrap();
7659
7660 let root = unique_project_root("live-roots-pending");
7661 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
7662 .unwrap()
7663 .as_path()
7664 .to_path_buf();
7665 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
7666 let open_handler = handler.clone();
7667 let pending = tokio::spawn(async move {
7668 open_handler
7669 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
7670 .await
7671 .unwrap()
7672 });
7673 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7674 .await
7675 .unwrap()
7676 .unwrap();
7677 let actual = query_live_roots(&handler, &target_ctx).await;
7678 let ModuleControlResponseToModule::LiveRoots {
7679 roots,
7680 unknown_root_bindings,
7681 total_bindings,
7682 } = actual
7683 else {
7684 panic!("expected live roots")
7685 };
7686 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
7687 assert_eq!(unknown_root_bindings, 0);
7688 assert_eq!(
7689 roots.len(),
7690 2,
7691 "root-known arm must retain each canonical root"
7692 );
7693 assert_eq!(
7694 total_bindings,
7695 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
7696 );
7697 let counts = roots
7698 .iter()
7699 .map(|root| (root.project_root.clone(), root.bound, root.pending))
7700 .collect::<Vec<_>>();
7701 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
7702 expected.sort_by(|a, b| a.0.cmp(&b.0));
7703 assert_eq!(
7704 counts, expected,
7705 "roots must sort by path and count pending separately"
7706 );
7707 handler
7708 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
7709 .await
7710 .unwrap();
7711 assert!(pending.await.unwrap().is_empty());
7712 }
7713
7714 #[tokio::test(start_paused = true)]
7715 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
7716 let forwarding = Arc::new(ForwardingTable::default());
7717 let handler =
7718 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
7719 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
7720 hello_via_sink(
7721 &handler,
7722 &target_ctx,
7723 &mut target_rx,
7724 hello_frame("target", PROTOCOL_VERSION, 1),
7725 )
7726 .await;
7727 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
7728 let pending = forwarding
7729 .begin_route_bind_relay_for_test(
7730 client_ctx.connection_id,
7731 client_ctx.egress.clone(),
7732 2,
7733 "target",
7734 )
7735 .unwrap();
7736 forwarding
7737 .complete_pending_relay(
7738 target_ctx.connection_id,
7739 pending.corr,
7740 RouteBindRelayOutcome::Accepted,
7741 )
7742 .unwrap();
7743 let actual = query_live_roots(&handler, &target_ctx).await;
7744 let ModuleControlResponseToModule::LiveRoots {
7745 roots,
7746 unknown_root_bindings,
7747 total_bindings,
7748 } = actual
7749 else {
7750 panic!("expected live roots")
7751 };
7752 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
7753 assert_eq!(
7754 unknown_root_bindings, 1,
7755 "unknown-root arm must not read as no bindings"
7756 );
7757 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
7758 assert_eq!(
7759 total_bindings,
7760 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
7761 );
7762 }
7763
7764 #[tokio::test(start_paused = true)]
7772 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
7773 let forwarding = Arc::new(ForwardingTable::default());
7774 let handler =
7775 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
7776 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
7777 let replies = handler
7778 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
7779 .await
7780 .unwrap();
7781 let queued_by_hello = module_rx.len();
7782
7783 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
7784 let open_handler = handler.clone();
7785 let open = tokio::spawn(async move {
7786 open_handler
7787 .handle_control_frame(
7788 &client_ctx,
7789 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
7790 )
7791 .await
7792 .unwrap()
7793 });
7794 let mut spins = 0;
7796 while module_rx.len() == queued_by_hello {
7797 spins += 1;
7798 assert!(spins < 10_000, "route.open never queued a route.bind");
7799 tokio::task::yield_now().await;
7800 }
7801
7802 for reply in replies {
7804 module_ctx.egress.send(reply).await.unwrap();
7805 }
7806
7807 let first = module_rx.recv().await.unwrap().frame;
7808 assert_eq!(
7809 first.header.ty,
7810 FrameType::HelloAck,
7811 "the first frame a registering module reads must be its HELLO_ACK"
7812 );
7813 assert_eq!(first.header.corr, 7);
7814 let second = module_rx.recv().await.unwrap().frame;
7815 assert_eq!(second.header.ty, FrameType::Request);
7816 assert!(
7817 matches!(
7818 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
7819 ModuleControlRequest::RouteBind { .. }
7820 ),
7821 "the route.bind follows the ack"
7822 );
7823 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
7824
7825 handler
7826 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
7827 .await
7828 .unwrap();
7829 assert!(open.await.unwrap().is_empty());
7830 let _ = client_rx.recv().await.unwrap();
7831 }
7832
7833 #[tokio::test(start_paused = true)]
7834 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
7835 let handler = ControlHandler::with_forwarding(
7836 Arc::new(Registry::default()),
7837 Arc::new(ForwardingTable::default()),
7838 );
7839 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
7840 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
7841 hello_via_sink(
7842 &handler,
7843 &first_ctx,
7844 &mut first_rx,
7845 hello_frame("first", PROTOCOL_VERSION, 1),
7846 )
7847 .await;
7848 hello_via_sink(
7849 &handler,
7850 &second_ctx,
7851 &mut second_rx,
7852 hello_frame("second", PROTOCOL_VERSION, 2),
7853 )
7854 .await;
7855 let root = unique_project_root("second-only");
7856 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
7857 let cloned = handler.clone();
7858 let open = tokio::spawn(async move {
7859 cloned
7860 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
7861 .await
7862 .unwrap()
7863 });
7864 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
7865 .await
7866 .unwrap()
7867 .unwrap();
7868 let first = query_live_roots(&handler, &first_ctx).await;
7869 let second = query_live_roots(&handler, &second_ctx).await;
7870 assert!(
7871 matches!(
7872 first,
7873 ModuleControlResponseToModule::LiveRoots {
7874 total_bindings: 0,
7875 ..
7876 }
7877 ),
7878 "cross-module scope must not expose another module's roots"
7879 );
7880 assert!(
7881 matches!(
7882 second,
7883 ModuleControlResponseToModule::LiveRoots {
7884 total_bindings: 1,
7885 ..
7886 }
7887 ),
7888 "second module must see its pending route"
7889 );
7890 handler
7891 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
7892 .await
7893 .unwrap();
7894 assert!(open.await.unwrap().is_empty());
7895 }
7896
7897 #[tokio::test(start_paused = true)]
7898 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
7899 let handler = ControlHandler::with_forwarding(
7900 Arc::new(Registry::default()),
7901 Arc::new(ForwardingTable::default()),
7902 );
7903 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
7904 hello_via_sink(
7905 &handler,
7906 &target_ctx,
7907 &mut target_rx,
7908 hello_frame("target", PROTOCOL_VERSION, 1),
7909 )
7910 .await;
7911 let actual = query_live_roots(&handler, &target_ctx).await;
7912 let ModuleControlResponseToModule::LiveRoots {
7913 roots,
7914 unknown_root_bindings,
7915 total_bindings,
7916 } = actual
7917 else {
7918 panic!("expected live roots")
7919 };
7920 assert!(roots.is_empty());
7921 assert_eq!(unknown_root_bindings, 0);
7922 assert_eq!(total_bindings, 0);
7923 assert_eq!(
7924 total_bindings,
7925 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
7926 );
7927 }
7928
7929 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
7936 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
7937 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
7938 let text = std::fs::read_to_string(&path)
7939 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
7940 let vectors: Vec<(String, Value)> = text
7941 .lines()
7942 .filter(|line| !line.trim().is_empty())
7943 .map(|line| {
7944 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
7945 let id = entry["corpus_id"]
7946 .as_str()
7947 .expect("every vector carries a corpus_id")
7948 .to_string();
7949 (id, entry["package"].clone())
7950 })
7951 .collect();
7952 assert_eq!(
7956 vectors.len(),
7957 3,
7958 "vendored fed corpus changed size; re-sync from subc-federation"
7959 );
7960
7961 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
7976 let richest = vectors
7977 .iter()
7978 .filter_map(|(_, package)| package.as_object())
7979 .map(|object| {
7980 object
7981 .keys()
7982 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
7983 .count()
7984 })
7985 .max()
7986 .unwrap_or(0);
7987 assert!(
7988 richest >= 2,
7989 "vendored corpus no longer carries a package with unmodelled fields, \
7990 so the relay test can no longer distinguish a verbatim relay from a lossy one"
7991 );
7992
7993 vectors
7994 }
7995
7996 #[tokio::test]
8004 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
8005 for (corpus_id, package) in fed_admission_facts_vectors() {
8006 let registry = Arc::new(Registry::default());
8007 let forwarding = Arc::new(ForwardingTable::default());
8008 let supervisor = SupervisorHandle::new();
8009 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8010 let handler =
8011 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8012 .with_supervisor(supervisor)
8013 .with_admission_facts_config(
8014 Some("fed".to_string()),
8015 Some(vec!["target".to_string()]),
8016 );
8017
8018 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
8019 hello_via_sink(
8020 &handler,
8021 &target_ctx,
8022 &mut target_rx,
8023 hello_frame("target", PROTOCOL_VERSION, 1),
8024 )
8025 .await;
8026
8027 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
8028 let route_handler = handler.clone();
8029 let expected = package.clone();
8030 let route_task = tokio::spawn(async move {
8031 route_handler
8032 .handle_control_frame(
8033 &client_ctx,
8034 route_open_frame_with_admission_facts(
8035 20,
8036 "target",
8037 unique_project_root("admission-facts"),
8038 Some(subc_control::ConsumerIdentity {
8039 module_id: "fed".to_string(),
8040 launch_nonce: "fed-nonce".to_string(),
8041 }),
8042 Some(package),
8043 ),
8044 )
8045 .await
8046 .unwrap()
8047 });
8048
8049 let bind_frame = target_rx.recv().await.unwrap();
8050 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8051 let ModuleControlRequest::RouteBind {
8052 admission_facts, ..
8053 } = bind
8054 else {
8055 panic!("{corpus_id}: expected route.bind")
8056 };
8057 assert_eq!(
8058 admission_facts,
8059 Some(expected),
8060 "{corpus_id}: relay must not add, drop or reshape any field"
8061 );
8062
8063 handler
8064 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8065 .await
8066 .unwrap();
8067 route_task.await.unwrap();
8068 }
8069 }
8070
8071 #[tokio::test]
8072 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
8073 let registry = Arc::new(Registry::default());
8074 let forwarding = Arc::new(ForwardingTable::default());
8075 let supervisor = SupervisorHandle::new();
8076 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8077 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
8078 let handler =
8079 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8080 .with_supervisor(supervisor)
8081 .with_admission_facts_config(
8082 Some("fed".to_string()),
8083 Some(vec!["target".to_string()]),
8084 );
8085
8086 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
8087 hello_via_sink(
8088 &handler,
8089 &target_ctx,
8090 &mut target_rx,
8091 hello_frame("target", PROTOCOL_VERSION, 1),
8092 )
8093 .await;
8094 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
8095 hello_via_sink(
8096 &handler,
8097 &other_ctx,
8098 &mut other_rx,
8099 hello_frame("other", PROTOCOL_VERSION, 2),
8100 )
8101 .await;
8102
8103 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
8104 let expected_facts = facts.clone();
8105 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
8106 let route_handler = handler.clone();
8107 let route_task = tokio::spawn(async move {
8108 route_handler
8109 .handle_control_frame(
8110 &client_ctx,
8111 route_open_frame_with_admission_facts(
8112 10,
8113 "target",
8114 unique_project_root("admission-facts"),
8115 Some(subc_control::ConsumerIdentity {
8116 module_id: "fed".to_string(),
8117 launch_nonce: "fed-nonce".to_string(),
8118 }),
8119 Some(facts.clone()),
8120 ),
8121 )
8122 .await
8123 .unwrap()
8124 });
8125 let bind_frame = target_rx.recv().await.unwrap();
8126 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8127 let ModuleControlRequest::RouteBind {
8128 admission_facts, ..
8129 } = bind
8130 else {
8131 panic!("expected route.bind")
8132 };
8133 assert_eq!(admission_facts, Some(expected_facts));
8134 handler
8135 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8136 .await
8137 .unwrap();
8138 assert!(route_task.await.unwrap().is_empty());
8139 assert!(matches!(
8140 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
8141 .unwrap(),
8142 ClientControlResponse::RouteOpen { .. }
8143 ));
8144
8145 let direct = handler
8146 .handle_control_frame(
8147 &route_ctx(ConnectionId::new(73)).0,
8148 route_open_frame_with_admission_facts(
8149 11,
8150 "target",
8151 unique_project_root("admission-facts"),
8152 None,
8153 Some(json!({"x": 1})),
8154 ),
8155 )
8156 .await
8157 .unwrap();
8158 assert_eq!(
8159 parse_error(&direct[0])["code"],
8160 "admission_facts_not_permitted"
8161 );
8162
8163 let different_reserved = handler
8164 .handle_control_frame(
8165 &route_ctx(ConnectionId::new(77)).0,
8166 route_open_frame_with_admission_facts(
8167 15,
8168 "target",
8169 unique_project_root("admission-facts"),
8170 Some(subc_control::ConsumerIdentity {
8171 module_id: "other".to_string(),
8172 launch_nonce: "other-nonce".to_string(),
8173 }),
8174 Some(json!({"x": 1})),
8175 ),
8176 )
8177 .await
8178 .unwrap();
8179 assert_eq!(
8180 parse_error(&different_reserved[0])["code"],
8181 "admission_facts_not_permitted"
8182 );
8183
8184 let other_target = handler
8185 .handle_control_frame(
8186 &route_ctx(ConnectionId::new(74)).0,
8187 route_open_frame_with_admission_facts(
8188 12,
8189 "other",
8190 unique_project_root("admission-facts"),
8191 Some(subc_control::ConsumerIdentity {
8192 module_id: "fed".to_string(),
8193 launch_nonce: "fed-nonce".to_string(),
8194 }),
8195 Some(json!({"x": 1})),
8196 ),
8197 )
8198 .await
8199 .unwrap();
8200 assert_eq!(
8201 parse_error(&other_target[0])["code"],
8202 "admission_facts_target_not_allowed"
8203 );
8204
8205 let nonexistent = handler
8206 .handle_control_frame(
8207 &route_ctx(ConnectionId::new(75)).0,
8208 route_open_frame_with_admission_facts(
8209 13,
8210 "missing",
8211 unique_project_root("admission-facts"),
8212 None,
8213 Some(json!({"x": 1})),
8214 ),
8215 )
8216 .await
8217 .unwrap();
8218 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
8219
8220 let described = handler
8221 .handle_control_frame(
8222 &route_ctx(ConnectionId::new(76)).0,
8223 Frame::build(
8224 FrameType::Request,
8225 control_flags(),
8226 0,
8227 0,
8228 14,
8229 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
8230 )
8231 .unwrap(),
8232 )
8233 .await
8234 .unwrap();
8235 let ClientControlResponse::ServerDescribe { capabilities, .. } =
8236 serde_json::from_slice(&described[0].body).unwrap()
8237 else {
8238 panic!("expected server.describe response")
8239 };
8240 assert!(capabilities
8241 .iter()
8242 .any(|cap| cap == "admission_facts_relay_v1"));
8243 }
8244
8245 #[tokio::test]
8246 async fn admission_facts_without_configured_carrier_are_rejected() {
8247 let registry = Arc::new(Registry::default());
8248 let forwarding = Arc::new(ForwardingTable::default());
8249 let handler = ControlHandler::with_forwarding(registry, forwarding);
8250 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
8251 hello_via_sink(
8252 &handler,
8253 &target_ctx,
8254 &mut target_rx,
8255 hello_frame("target", PROTOCOL_VERSION, 1),
8256 )
8257 .await;
8258
8259 let responses = handler
8260 .handle_control_frame(
8261 &route_ctx(ConnectionId::new(79)).0,
8262 route_open_frame_with_admission_facts(
8263 16,
8264 "target",
8265 unique_project_root("admission-facts"),
8266 None,
8267 Some(json!({"x": 1})),
8268 ),
8269 )
8270 .await
8271 .unwrap();
8272 assert_eq!(
8273 parse_error(&responses[0])["code"],
8274 "admission_facts_not_permitted"
8275 );
8276 }
8277
8278 #[tokio::test]
8279 async fn route_open_relays_consumer_capabilities_verbatim() {
8280 let registry = Arc::new(Registry::default());
8281 let forwarding = Arc::new(ForwardingTable::default());
8282 let handler =
8283 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8284 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
8285 hello_via_sink(
8286 &handler,
8287 &module_ctx,
8288 &mut module_rx,
8289 hello_frame("aft", PROTOCOL_VERSION, 7),
8290 )
8291 .await;
8292
8293 let expected = vec!["elicitation".to_string(), "roots".to_string()];
8294 let expected_for_request = expected.clone();
8295 let project_root = unique_project_root("consumer-capabilities-present");
8296 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
8297 let route_handler = handler.clone();
8298 let route_task = tokio::spawn(async move {
8299 route_handler
8300 .handle_control_frame(
8301 &client_ctx,
8302 route_open_frame_with_consumer_capabilities(
8303 401,
8304 "aft",
8305 project_root,
8306 Some(expected_for_request),
8307 ),
8308 )
8309 .await
8310 .unwrap()
8311 });
8312 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8313 .await
8314 .unwrap()
8315 .unwrap();
8316 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8317 let ModuleControlRequest::RouteBind {
8318 consumer_capabilities,
8319 ..
8320 } = bind
8321 else {
8322 panic!("expected route.bind request, got {bind:?}");
8323 };
8324 assert_eq!(consumer_capabilities, Some(expected.clone()));
8325
8326 handler
8327 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8328 .await
8329 .unwrap();
8330 let route_response = route_task.await.unwrap();
8331 assert!(route_response.is_empty());
8332 let published = client_rx.recv().await.unwrap();
8333 assert!(matches!(
8334 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8335 ClientControlResponse::RouteOpen { .. }
8336 ));
8337 }
8338
8339 #[tokio::test]
8340 async fn route_open_without_consumer_capabilities_relays_none() {
8341 let registry = Arc::new(Registry::default());
8342 let forwarding = Arc::new(ForwardingTable::default());
8343 let handler =
8344 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8345 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
8346 hello_via_sink(
8347 &handler,
8348 &module_ctx,
8349 &mut module_rx,
8350 hello_frame("aft", PROTOCOL_VERSION, 7),
8351 )
8352 .await;
8353
8354 let project_root = unique_project_root("consumer-capabilities-absent");
8355 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
8356 let route_handler = handler.clone();
8357 let route_task = tokio::spawn(async move {
8358 route_handler
8359 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
8360 .await
8361 .unwrap()
8362 });
8363 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8364 .await
8365 .unwrap()
8366 .unwrap();
8367 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8368 let ModuleControlRequest::RouteBind {
8369 consumer_capabilities,
8370 ..
8371 } = bind
8372 else {
8373 panic!("expected route.bind request, got {bind:?}");
8374 };
8375 assert_eq!(consumer_capabilities, None);
8376
8377 handler
8378 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8379 .await
8380 .unwrap();
8381 let route_response = route_task.await.unwrap();
8382 assert!(route_response.is_empty());
8383 let published = client_rx.recv().await.unwrap();
8384 assert!(matches!(
8385 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8386 ClientControlResponse::RouteOpen { .. }
8387 ));
8388 }
8389
8390 #[tokio::test]
8391 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
8392 let registry = Arc::new(Registry::default());
8393 let forwarding = Arc::new(ForwardingTable::default());
8394 let handler =
8395 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8396 .with_health_probe_timeout(Duration::from_secs(5));
8397 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
8398 hello_via_sink(
8399 &handler,
8400 &module_ctx,
8401 &mut module_rx,
8402 non_routable_hello_frame_with_control_ops(
8403 "mcp",
8404 300,
8405 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
8406 ),
8407 )
8408 .await;
8409 assert!(registry
8410 .get_module("mcp")
8411 .unwrap()
8412 .unwrap()
8413 .manifest
8414 .provides
8415 .is_empty());
8416
8417 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
8418 let route_response = handler
8419 .handle_control_frame(
8420 &route_client_ctx,
8421 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
8422 )
8423 .await
8424 .unwrap();
8425 assert_eq!(route_response[0].header.ty, FrameType::Error);
8426 assert_eq!(
8427 parse_error(&route_response[0])["code"],
8428 "target_unavailable"
8429 );
8430 assert!(parse_error(&route_response[0])["message"]
8431 .as_str()
8432 .unwrap()
8433 .contains("does not provide the requested target"));
8434 assert!(module_rx.try_recv().is_err());
8435
8436 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
8437 let health_handler = handler.clone();
8438 let health_task = tokio::spawn(async move {
8439 health_handler
8440 .handle_control_frame(
8441 &health_client_ctx,
8442 supervisor_health_probe_frame(302, "mcp"),
8443 )
8444 .await
8445 .unwrap()
8446 });
8447 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8448 .await
8449 .unwrap()
8450 .unwrap();
8451 assert_eq!(
8452 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
8453 ModuleControlRequest::HealthCheck {}
8454 );
8455 handler
8456 .handle_control_frame(
8457 &module_ctx,
8458 health_response(health_frame.header.corr, HealthStatus::Ok),
8459 )
8460 .await
8461 .unwrap();
8462 let health_response = health_task.await.unwrap();
8463 assert_eq!(health_response[0].header.ty, FrameType::Response);
8464 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
8465 ClientControlResponse::SupervisorHealthProbe {
8466 module_id, status, ..
8467 } => {
8468 assert_eq!(module_id, "mcp");
8469 assert_eq!(status, HealthStatus::Ok);
8470 }
8471 other => panic!("unexpected health response: {other:?}"),
8472 }
8473
8474 forwarding
8479 .cleanup_connection(module_ctx.connection_id)
8480 .unwrap();
8481 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
8482 let cleanup_response = tokio::time::timeout(
8483 Duration::from_millis(200),
8484 handler.handle_control_frame(
8485 &cleanup_probe_ctx,
8486 supervisor_health_probe_frame(303, "mcp"),
8487 ),
8488 )
8489 .await
8490 .expect("probe should fail immediately when the control lane is gone")
8491 .unwrap();
8492 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
8493 assert_eq!(
8494 parse_error(&cleanup_response[0])["code"],
8495 "target_unavailable"
8496 );
8497 assert!(parse_error(&cleanup_response[0])["message"]
8498 .as_str()
8499 .unwrap()
8500 .contains("no module connection"));
8501
8502 handler
8503 .cleanup_connection(module_ctx.connection_id)
8504 .unwrap();
8505 }
8506
8507 #[tokio::test]
8508 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
8509 let registry = Arc::new(Registry::default());
8510 let supervisor_handle = SupervisorHandle::new();
8511 let supervisor =
8512 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
8513 .with_handle(supervisor_handle.clone())
8514 .with_connection_file_path(
8515 std::env::temp_dir()
8516 .join(format!("subc-route-open-warming-{}", std::process::id())),
8517 );
8518 let module = supervisor
8519 .supervise_configured(
8520 ModuleSpec {
8521 module_id: "warming".to_string(),
8522 program: fake_aft_stub_path(),
8523 args: Vec::new(),
8524 env: Vec::new(),
8525 reserved: false,
8526 reserved_prefixes: Vec::new(),
8527 protocol: ModuleProtocol::Subc,
8528 overlap: Default::default(),
8529 },
8530 true,
8531 )
8532 .unwrap();
8533 assert_eq!(module.state().unwrap(), ModuleState::Running);
8534
8535 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8536 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
8537 let response = handler
8538 .handle_control_frame(
8539 &ctx,
8540 route_open_frame(304, "warming", unique_project_root("warming")),
8541 )
8542 .await
8543 .unwrap();
8544 module.stop().await.unwrap();
8545
8546 assert_eq!(response[0].header.ty, FrameType::Error);
8547 let error = parse_error(&response[0]);
8548 assert_eq!(error["code"], "module_warming");
8549 assert!(error["message"]
8550 .as_str()
8551 .unwrap()
8552 .contains("state=running, enabled=true, live=false"));
8553 }
8554
8555 #[tokio::test]
8560 async fn route_open_refusal_names_the_check_that_refused() {
8561 let handler = ControlHandler::new(Arc::new(Registry::default()));
8562 let capture = EventCapture::default();
8563 let _subscriber =
8564 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8565 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
8566 let response = handler
8567 .handle_control_frame(
8568 &ctx,
8569 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
8570 )
8571 .await
8572 .unwrap();
8573
8574 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
8575 let event = capture
8576 .events()
8577 .into_iter()
8578 .find(|event| {
8579 event.target == "control"
8580 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
8581 })
8582 .expect("route.open refusal event");
8583 assert_eq!(
8584 event.fields.get("reason"),
8585 Some(&"\"not_registered\"".to_string())
8586 );
8587 }
8588
8589 #[tokio::test]
8590 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
8591 let registry = Arc::new(Registry::default());
8592 let supervisor_handle = SupervisorHandle::new();
8593 let supervisor =
8594 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
8595 .with_handle(supervisor_handle.clone())
8596 .with_connection_file_path(std::env::temp_dir().join(format!(
8597 "subc-route-open-refusal-info-{}",
8598 std::process::id()
8599 )));
8600 let module = supervisor
8601 .supervise_configured(
8602 ModuleSpec {
8603 module_id: "warming".to_string(),
8604 program: fake_aft_stub_path(),
8605 args: Vec::new(),
8606 env: Vec::new(),
8607 reserved: false,
8608 reserved_prefixes: Vec::new(),
8609 protocol: ModuleProtocol::Subc,
8610 overlap: Default::default(),
8611 },
8612 true,
8613 )
8614 .unwrap();
8615 assert_eq!(module.state().unwrap(), ModuleState::Running);
8616
8617 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8618 assert!(handler
8619 .counters()
8620 .snapshot()
8621 .get("route_open_refused_by_code")
8622 .is_none());
8623 let capture = EventCapture::default();
8624 let _subscriber =
8625 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8626 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
8627 let response = handler
8628 .handle_control_frame(
8629 &ctx,
8630 route_open_frame(394, "warming", unique_project_root("refusal-info")),
8631 )
8632 .await
8633 .unwrap();
8634 module.stop().await.unwrap();
8635
8636 assert_eq!(parse_error(&response[0])["code"], "module_warming");
8637 let event = capture
8638 .events()
8639 .into_iter()
8640 .find(|event| {
8641 event.target == "control"
8642 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
8643 })
8644 .expect("route.open refusal event");
8645 assert_eq!(
8646 event.fields.get("module_id"),
8647 Some(&"\"warming\"".to_string())
8648 );
8649 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
8650 assert_eq!(
8651 event.fields.get("reason"),
8652 Some(&"\"supervised_not_registered\"".to_string())
8653 );
8654 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
8655 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
8656 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
8657 assert_eq!(
8658 handler.counters().snapshot()["route_open_refused_by_code"],
8659 json!({ "module_warming": 1 })
8660 );
8661 }
8662
8663 #[tokio::test(flavor = "current_thread")]
8664 async fn route_open_unknown_module_escapes_target_module_id() {
8665 let handler = ControlHandler::new(Arc::new(Registry::default()));
8666 let capture = EventCapture::default();
8667 let _subscriber =
8668 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8669 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
8670 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
8671 let response = handler
8672 .handle_control_frame(
8673 &ctx,
8674 route_open_frame(
8675 395,
8676 hostile_module_id,
8677 unique_project_root("hostile-target-module-id"),
8678 ),
8679 )
8680 .await
8681 .unwrap();
8682
8683 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
8684 let event = capture
8685 .events()
8686 .into_iter()
8687 .find(|event| {
8688 event.target == "control"
8689 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
8690 })
8691 .expect("route.open unknown-module refusal event");
8692 let logged = event.fields.get("module_id").expect("module_id field");
8693 assert!(!logged.bytes().any(|byte| byte < 0x20));
8694 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
8695 }
8696
8697 #[tokio::test(flavor = "current_thread")]
8698 async fn route_open_module_rejection_uses_daemon_counter_key() {
8699 let registry = Arc::new(Registry::default());
8700 let forwarding = Arc::new(ForwardingTable::default());
8701 let handler =
8702 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8703 let module_connection = ConnectionId::new(95);
8704 let (module_ctx, mut module_rx) = route_ctx(module_connection);
8705 hello_via_sink(
8706 &handler,
8707 &module_ctx,
8708 &mut module_rx,
8709 hello_frame("aft", PROTOCOL_VERSION, 395),
8710 )
8711 .await;
8712
8713 let client_connection = ConnectionId::new(96);
8714 let (client_ctx, _client_rx) = route_ctx(client_connection);
8715 let capture = EventCapture::default();
8716 let _subscriber =
8717 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8718 let (route_task, bind) = relay_route_open(
8719 &handler,
8720 client_connection,
8721 &client_ctx.egress,
8722 &mut module_rx,
8723 396,
8724 "aft",
8725 "hostile-module-code",
8726 )
8727 .await;
8728 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
8729 let rejection = Frame::build(
8730 FrameType::Error,
8731 control_flags(),
8732 0,
8733 0,
8734 bind.header.corr,
8735 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
8736 )
8737 .unwrap();
8738 handler
8739 .handle_control_frame(&module_ctx, rejection)
8740 .await
8741 .unwrap();
8742
8743 let response = route_task.await.unwrap();
8744 assert_eq!(parse_error(&response[0])["code"], hostile_code);
8745 let counters = handler.counters().snapshot();
8746 assert_eq!(
8747 counters["route_open_refused_by_code"],
8748 json!({ "module_rejected": 1 })
8749 );
8750 assert!(counters["route_open_refused_by_code"]
8751 .get(hostile_code)
8752 .is_none());
8753
8754 let event = capture
8755 .events()
8756 .into_iter()
8757 .find(|event| {
8758 event.target == "control"
8759 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
8760 })
8761 .expect("route.open module-rejection refusal event");
8762 let logged = event.fields.get("module_code").expect("module_code field");
8763 assert!(!logged.bytes().any(|byte| byte < 0x20));
8764 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
8765 }
8766
8767 #[tokio::test]
8768 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
8769 let registry = Arc::new(Registry::default());
8770 let supervisor_handle = SupervisorHandle::new();
8771 let missing_program = std::env::temp_dir().join(format!(
8772 "subc-route-open-missing-program-{}",
8773 std::process::id()
8774 ));
8775 let supervisor =
8776 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
8777 .with_handle(supervisor_handle.clone());
8778 let module = supervisor
8779 .supervise_configured(
8780 ModuleSpec {
8781 module_id: "failed".to_string(),
8782 program: missing_program,
8783 args: Vec::new(),
8784 env: Vec::new(),
8785 reserved: false,
8786 reserved_prefixes: Vec::new(),
8787 protocol: ModuleProtocol::Subc,
8788 overlap: Default::default(),
8789 },
8790 true,
8791 )
8792 .unwrap();
8793 assert_eq!(module.state().unwrap(), ModuleState::Failed);
8794
8795 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8796 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
8797 let response = handler
8798 .handle_control_frame(
8799 &ctx,
8800 route_open_frame(305, "failed", unique_project_root("failed")),
8801 )
8802 .await
8803 .unwrap();
8804
8805 assert_eq!(response[0].header.ty, FrameType::Error);
8806 let error = parse_error(&response[0]);
8807 assert_eq!(error["code"], "target_unavailable");
8808 assert!(error["message"]
8809 .as_str()
8810 .unwrap()
8811 .contains("state=failed, enabled=true, live=false"));
8812 }
8813
8814 #[tokio::test]
8815 async fn route_open_role_mismatch_remains_target_unavailable() {
8816 let registry = Arc::new(Registry::default());
8817 let handler = ControlHandler::new(Arc::clone(®istry));
8818 handler
8819 .handle_control(
8820 ConnectionId::new(41),
8821 non_routable_hello_frame_with_control_ops("health-only", 306, None),
8822 )
8823 .unwrap();
8824
8825 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
8826 let response = handler
8827 .handle_control_frame(
8828 &ctx,
8829 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
8830 )
8831 .await
8832 .unwrap();
8833
8834 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
8835 assert!(parse_error(&response[0])["message"]
8836 .as_str()
8837 .unwrap()
8838 .contains("does not provide the requested target"));
8839 }
8840
8841 #[tokio::test]
8842 async fn route_open_inactive_registration_remains_target_unavailable() {
8843 let registry = Arc::new(Registry::default());
8844 let handler = ControlHandler::new(Arc::clone(®istry));
8845 handler
8846 .handle_control(
8847 ConnectionId::new(43),
8848 hello_frame("inactive", PROTOCOL_VERSION, 308),
8849 )
8850 .unwrap();
8851 assert!(registry
8852 .set_module_state_for_test("inactive", ChannelState::Closed)
8853 .unwrap());
8854
8855 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
8856 let response = handler
8857 .handle_control_frame(
8858 &ctx,
8859 route_open_frame(309, "inactive", unique_project_root("inactive")),
8860 )
8861 .await
8862 .unwrap();
8863
8864 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
8865 assert!(parse_error(&response[0])["message"]
8866 .as_str()
8867 .unwrap()
8868 .contains("is not active"));
8869 }
8870
8871 #[tokio::test]
8872 async fn late_health_reply_is_recorded_through_the_module_response_path() {
8873 let registry = Arc::new(Registry::default());
8874 let forwarding = Arc::new(ForwardingTable::default());
8875 let supervisor_handle = SupervisorHandle::new();
8876 let supervisor = Supervisor::new(Arc::clone(®istry), crate::RestartPolicy::default())
8877 .with_forwarding(Arc::clone(&forwarding))
8878 .with_handle(supervisor_handle.clone());
8879 let module = supervisor
8880 .supervise_configured(
8881 crate::ModuleSpec {
8882 module_id: "late-health-response".to_string(),
8883 program: PathBuf::from("disabled-module"),
8884 args: Vec::new(),
8885 env: Vec::new(),
8886 reserved: false,
8887 reserved_prefixes: Vec::new(),
8888 protocol: ModuleProtocol::Subc,
8889 overlap: Default::default(),
8890 },
8891 false,
8892 )
8893 .unwrap();
8894 let handler =
8895 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8896 .with_supervisor(supervisor_handle);
8897 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
8898 handler
8899 .handle_control_frame(
8900 &module_ctx,
8901 hello_frame_with_control_ops(
8902 "late-health-response",
8903 PROTOCOL_VERSION,
8904 7,
8905 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
8906 ),
8907 )
8908 .await
8909 .unwrap();
8910 let probe_started_at = Instant::now() - Duration::from_millis(80);
8911 let pending = forwarding
8912 .begin_health_probe_rpc_for(
8913 "late-health-response",
8914 MODULE_CONTROL_OP_HEALTH_CHECK,
8915 probe_started_at,
8916 Instant::now() - Duration::from_millis(1),
8917 )
8918 .unwrap();
8919 assert!(forwarding
8920 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
8921 .unwrap());
8922
8923 let responses = handler
8924 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
8925 .await
8926 .unwrap();
8927
8928 assert!(responses.is_empty());
8929 let health = module.status().unwrap().health;
8930 assert_eq!(health.late_answer_count, 1);
8931 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
8932 }
8933
8934 #[tokio::test]
8935 async fn health_probe_timeout_and_module_death_are_typed() {
8936 let registry = Arc::new(Registry::default());
8937 let forwarding = Arc::new(ForwardingTable::default());
8938 let handler =
8939 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8940 .with_health_probe_timeout(Duration::from_millis(50));
8941 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
8942 hello_via_sink(
8943 &handler,
8944 &module_ctx,
8945 &mut module_rx,
8946 hello_frame_with_control_ops(
8947 "aft",
8948 PROTOCOL_VERSION,
8949 7,
8950 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
8951 ),
8952 )
8953 .await;
8954
8955 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
8956 let responses = handler
8957 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
8958 .await
8959 .unwrap();
8960 assert_eq!(responses[0].header.ty, FrameType::Error);
8961 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
8962 let _ = module_rx.try_recv();
8963
8964 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
8965 let health_handler = handler.clone();
8966 let death_task = tokio::spawn(async move {
8967 health_handler
8968 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
8969 .await
8970 .unwrap()
8971 });
8972 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8973 .await
8974 .unwrap()
8975 .unwrap();
8976 handler
8977 .cleanup_connection(module_ctx.connection_id)
8978 .unwrap();
8979 let responses = death_task.await.unwrap();
8980 assert_eq!(responses[0].header.ty, FrameType::Error);
8981 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
8982 }
8983
8984 #[test]
8985 fn hello_requires_exact_protocol_version() {
8986 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
8987 let registry = Arc::new(Registry::default());
8988 let handler = ControlHandler::new(Arc::clone(®istry));
8989 let responses = handler
8990 .handle_control(
8991 ConnectionId::new(connection),
8992 hello_frame("aft", offered, 9),
8993 )
8994 .unwrap();
8995
8996 assert_eq!(responses.len(), 1);
8997 assert_eq!(responses[0].header.ty, FrameType::Error);
8998 let error = parse_error(&responses[0]);
8999 assert_eq!(error["code"], "version_unsupported");
9000 assert!(registry.get_module("aft").unwrap().is_none());
9001 assert_eq!(registry.active_registration_count().unwrap(), 0);
9002 }
9003 }
9004
9005 #[test]
9006 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
9007 let registry = Arc::new(Registry::default());
9008 let forwarding = Arc::new(ForwardingTable::default());
9009 let handler =
9010 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9011 let module_connection = ConnectionId::new(301);
9012 let registration = registry
9013 .register_with_control_ops(
9014 manifest("aft-push", PROTOCOL_VERSION),
9015 PROTOCOL_VERSION,
9016 module_connection,
9017 module_baseline_control_ops(),
9018 )
9019 .unwrap();
9020 let (module_tx, _module_rx) = mpsc::channel(8);
9021 let endpoint = forwarding
9022 .register_module_connection(
9023 module_connection,
9024 "aft-push".to_string(),
9025 PROTOCOL_VERSION,
9026 manifest_concurrency(®istration.manifest),
9027 FrameSink::new(module_tx),
9028 )
9029 .unwrap();
9030
9031 let unknown = Frame::build(
9033 FrameType::Push,
9034 control_flags(),
9035 0,
9036 0,
9037 5,
9038 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
9039 )
9040 .unwrap();
9041 let out = handler.handle_status_update(endpoint, unknown).unwrap();
9042 assert!(
9043 out.is_empty(),
9044 "unknown push op must be ignored, got {out:?}"
9045 );
9046
9047 let malformed = Frame::build(
9049 FrameType::Push,
9050 control_flags(),
9051 0,
9052 0,
9053 6,
9054 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
9055 )
9056 .unwrap();
9057 let out = handler.handle_status_update(endpoint, malformed).unwrap();
9058 assert_eq!(out.len(), 1);
9059 assert_eq!(out[0].header.ty, FrameType::Error);
9060 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
9061 }
9062
9063 #[test]
9064 fn hello_rejected_when_connection_already_owns_client_routes() {
9065 let registry = Arc::new(Registry::default());
9066 let forwarding = Arc::new(ForwardingTable::default());
9067 let handler =
9068 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9069 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
9071 let client_connection = ConnectionId::new(202);
9072
9073 let responses = handler
9076 .handle_control(
9077 client_connection,
9078 hello_frame("aft-second", PROTOCOL_VERSION, 9),
9079 )
9080 .unwrap();
9081 assert_eq!(responses[0].header.ty, FrameType::Error);
9082 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
9083 assert!(registry.get_module("aft-second").unwrap().is_none());
9084 }
9085
9086 #[test]
9087 fn reserved_module_hello_requires_matching_launch_nonce() {
9088 let registry = Arc::new(Registry::default());
9089 let supervisor = SupervisorHandle::new();
9090 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
9093 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9094
9095 let no_nonce = handler
9097 .handle_control(
9098 ConnectionId::new(1),
9099 hello_frame("vault", PROTOCOL_VERSION, 1),
9100 )
9101 .unwrap();
9102 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
9103 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
9104 assert!(registry.get_module("vault").unwrap().is_none());
9105
9106 let wrong = handler
9108 .handle_control(
9109 ConnectionId::new(2),
9110 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
9111 )
9112 .unwrap();
9113 assert_eq!(wrong[0].header.ty, FrameType::Error);
9114 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
9115 assert!(registry.get_module("vault").unwrap().is_none());
9116
9117 let ok = handler
9119 .handle_control(
9120 ConnectionId::new(3),
9121 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
9122 )
9123 .unwrap();
9124 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
9125 assert!(registry.get_module("vault").unwrap().is_some());
9126 }
9127
9128 #[test]
9129 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
9130 let registry = Arc::new(Registry::default());
9131 let supervisor = SupervisorHandle::new();
9132 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
9133 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
9134 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9135
9136 let squat = handler
9137 .handle_control(
9138 ConnectionId::new(1),
9139 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
9140 )
9141 .unwrap();
9142 assert_eq!(squat[0].header.ty, FrameType::Error);
9143 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
9144 assert!(parse_error(&squat[0])["message"]
9145 .as_str()
9146 .unwrap()
9147 .contains("fed:"));
9148
9149 let accepted_peer = handler
9150 .handle_control(
9151 ConnectionId::new(2),
9152 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
9153 )
9154 .unwrap();
9155 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
9156
9157 let accepted_short = handler
9158 .handle_control(
9159 ConnectionId::new(3),
9160 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
9161 )
9162 .unwrap();
9163 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
9164
9165 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
9166 let response = handler
9167 .handle_control(
9168 ConnectionId::new(conn),
9169 hello_frame(module_id, PROTOCOL_VERSION, conn),
9170 )
9171 .unwrap();
9172 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
9173 }
9174 }
9175
9176 #[test]
9177 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
9178 let registry = Arc::new(Registry::default());
9179 let supervisor = SupervisorHandle::new();
9180 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
9181 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
9182 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
9183 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9184
9185 let owner_nonce = handler
9186 .handle_control(
9187 ConnectionId::new(1),
9188 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
9189 )
9190 .unwrap();
9191 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
9192 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
9193 assert!(registry.get_module("fed:special").unwrap().is_none());
9194
9195 let exact_nonce = handler
9196 .handle_control(
9197 ConnectionId::new(2),
9198 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
9199 )
9200 .unwrap();
9201 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
9202 assert!(registry.get_module("fed:special").unwrap().is_some());
9203 }
9204
9205 #[test]
9206 fn non_reserved_module_ignores_launch_nonce() {
9207 let registry = Arc::new(Registry::default());
9208 let handler = ControlHandler::new(Arc::clone(®istry));
9211 let no_nonce = handler
9212 .handle_control(
9213 ConnectionId::new(1),
9214 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
9215 )
9216 .unwrap();
9217 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
9218 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
9219
9220 let echoed_nonce = handler
9221 .handle_control(
9222 ConnectionId::new(2),
9223 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
9224 )
9225 .unwrap();
9226 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
9227 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
9228 }
9229
9230 #[test]
9231 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
9232 let handler = ControlHandler::default();
9233 let conn = ConnectionId::new(1);
9234 let malformed = Frame::build(
9235 FrameType::Hello,
9236 control_flags(),
9237 0,
9238 0,
9239 3,
9240 b"{not json".to_vec(),
9241 )
9242 .unwrap();
9243
9244 let error = handler.handle_control(conn, malformed).unwrap();
9245 assert_eq!(error[0].header.ty, FrameType::Error);
9246 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
9247
9248 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
9249 let pong = handler.handle_control(conn, ping).unwrap();
9250 assert_eq!(pong[0].header.ty, FrameType::Pong);
9251 assert_eq!(pong[0].header.corr, 4);
9252 }
9253
9254 #[test]
9255 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
9256 let registry = Arc::new(Registry::default());
9257 let handler = ControlHandler::new(Arc::clone(®istry));
9258
9259 handler
9260 .handle_control(
9261 ConnectionId::new(1),
9262 hello_frame("aft", PROTOCOL_VERSION, 1),
9263 )
9264 .unwrap();
9265 let duplicate = handler
9266 .handle_control(
9267 ConnectionId::new(2),
9268 hello_frame("aft", PROTOCOL_VERSION, 2),
9269 )
9270 .unwrap();
9271
9272 assert_eq!(duplicate[0].header.ty, FrameType::Error);
9273 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
9274 let registration = registry.get_module("aft").unwrap().unwrap();
9275 assert_eq!(registration.connection_id, ConnectionId::new(1));
9276 }
9277
9278 #[test]
9279 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
9280 let registry = Arc::new(Registry::default());
9281 let forwarding = Arc::new(ForwardingTable::default());
9282 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
9283 let handler =
9284 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9285 .with_process_liveness(process_liveness);
9286 let (ctx, route_channel, route_epoch) =
9287 bind_liveness_route(®istry, &forwarding, "aft-dead");
9288 let responses = handler
9289 .handle_route_poll(
9290 &ctx,
9291 route_poll_frame(41, PollKind::Liveness, route_channel),
9292 route_channel,
9293 route_epoch,
9294 PollKind::Liveness,
9295 )
9296 .unwrap();
9297
9298 assert_eq!(responses.len(), 1);
9299 assert_eq!(responses[0].header.ty, FrameType::Response);
9300 assert_route_poll_liveness(&responses[0], false);
9301 }
9302
9303 #[test]
9304 fn liveness_poll_without_process_source_uses_bound_route() {
9305 let registry = Arc::new(Registry::default());
9306 let forwarding = Arc::new(ForwardingTable::default());
9307 let handler =
9308 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9309 let (ctx, route_channel, route_epoch) =
9310 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
9311 let responses = handler
9312 .handle_route_poll(
9313 &ctx,
9314 route_poll_frame(42, PollKind::Liveness, route_channel),
9315 route_channel,
9316 route_epoch,
9317 PollKind::Liveness,
9318 )
9319 .unwrap();
9320
9321 assert_route_poll_liveness(&responses[0], true);
9322 }
9323
9324 #[test]
9325 fn liveness_poll_untracked_process_source_uses_bound_route() {
9326 let registry = Arc::new(Registry::default());
9327 let forwarding = Arc::new(ForwardingTable::default());
9328 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
9329 let handler =
9330 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9331 .with_process_liveness(process_liveness);
9332 let (ctx, route_channel, route_epoch) =
9333 bind_liveness_route(®istry, &forwarding, "aft-untracked");
9334 let responses = handler
9335 .handle_route_poll(
9336 &ctx,
9337 route_poll_frame(43, PollKind::Liveness, route_channel),
9338 route_channel,
9339 route_epoch,
9340 PollKind::Liveness,
9341 )
9342 .unwrap();
9343
9344 assert_route_poll_liveness(&responses[0], true);
9345 }
9346
9347 #[tokio::test]
9348 async fn unknown_op_returns_unknown_control_op() {
9349 let handler = ControlHandler::default();
9350 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
9351 let request = Frame::build(
9352 FrameType::Request,
9353 control_flags(),
9354 0,
9355 0,
9356 55,
9357 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
9358 )
9359 .unwrap();
9360
9361 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
9362
9363 assert_eq!(response.len(), 1);
9364 assert_eq!(response[0].header.ty, FrameType::Error);
9365 assert_eq!(response[0].header.corr, 55);
9366 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
9367 }
9368
9369 #[tokio::test]
9370 async fn supervisor_provenance_rejects_unknown_exact_module() {
9371 let handler = ControlHandler::default();
9372 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
9373 let request = Frame::build(
9374 FrameType::Request,
9375 control_flags(),
9376 0,
9377 0,
9378 57,
9379 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
9380 )
9381 .unwrap();
9382
9383 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
9384
9385 assert_eq!(response.len(), 1);
9386 assert_eq!(response[0].header.ty, FrameType::Error);
9387 assert_eq!(response[0].header.corr, 57);
9388 let error = parse_error(&response[0]);
9389 assert_eq!(error["code"], "unknown_module");
9390 assert_eq!(error["message"], "module_id 'missing' is not supervised");
9391 }
9392
9393 #[test]
9394 fn provenance_probe_override_keeps_handler_tests_deterministic() {
9395 let expected = subc_control::RunningImageAgreement::Unavailable {
9396 reason: subc_control::RunningImageUnavailableReason::HashFailed,
9397 };
9398 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
9399 assert_eq!(handler.provenance_probe_override, Some(expected));
9400 }
9401
9402 #[test]
9403 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
9404 let verdict = reload_verdict(
9405 std::path::Path::new("/bin/new"),
9406 Some(std::path::Path::new("/bin/old")),
9407 subc_control::RunningImageAgreement::Unavailable {
9408 reason: subc_control::RunningImageUnavailableReason::HashFailed,
9409 },
9410 );
9411 assert!(matches!(
9412 verdict.path,
9413 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
9414 if configured == std::path::Path::new("/bin/new")
9415 && spawned_from == std::path::Path::new("/bin/old")
9416 ));
9417 }
9418
9419 #[test]
9420 fn reload_verdict_detects_replaced_image_at_same_path() {
9421 let image = subc_control::RunningImageAgreement::Mismatch {
9422 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
9423 digest: "old".into(),
9424 },
9425 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
9426 digest: "new".into(),
9427 },
9428 };
9429 let verdict = reload_verdict(
9430 std::path::Path::new("/bin/same"),
9431 Some(std::path::Path::new("/bin/same")),
9432 image.clone(),
9433 );
9434 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
9435 assert_eq!(verdict.image, image);
9436 }
9437
9438 #[test]
9439 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
9440 let image = subc_control::RunningImageAgreement::Unavailable {
9441 reason: subc_control::RunningImageUnavailableReason::NotRunning,
9442 };
9443 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
9444 assert_eq!(
9445 verdict.path,
9446 subc_control::ReloadPathAgreement::Unavailable {
9447 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
9448 }
9449 );
9450 assert_eq!(verdict.image, image);
9451
9452 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
9453 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
9454 };
9455 let verdict = reload_verdict(
9456 std::path::Path::new("/bin/same"),
9457 Some(std::path::Path::new("/bin/same")),
9458 unconfirmed.clone(),
9459 );
9460 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
9461 assert_eq!(verdict.image, unconfirmed);
9462 }
9463
9464 #[test]
9465 fn reload_verdict_preserves_each_image_unavailability_reason() {
9466 use subc_control::RunningImageUnavailableReason as Reason;
9467
9468 for reason in [
9469 Reason::NotRunning,
9470 Reason::UnsupportedPlatform,
9471 Reason::RunningExecutableUnreadable,
9472 Reason::SpawnedPathUnreadable,
9473 Reason::HashFailed,
9474 Reason::ProcessIdentityUnconfirmed,
9475 Reason::Unknown("future_probe_reason".to_string()),
9476 ] {
9477 let image = subc_control::RunningImageAgreement::Unavailable {
9478 reason: reason.clone(),
9479 };
9480 let verdict = reload_verdict(
9481 std::path::Path::new("/bin/same"),
9482 Some(std::path::Path::new("/bin/same")),
9483 image.clone(),
9484 );
9485 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
9486 assert_eq!(verdict.image, image, "{reason:?}");
9487 }
9488 }
9489
9490 #[tokio::test]
9491 async fn malformed_control_bodies_return_invalid_control_body() {
9492 let handler = ControlHandler::default();
9493 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
9494
9495 for (corr, body) in [
9496 (56, br#"{"route_channel":1}"#.as_slice()),
9497 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
9498 (
9499 58,
9500 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
9501 ),
9502 ] {
9503 let request = Frame::build(
9504 FrameType::Request,
9505 control_flags(),
9506 0,
9507 0,
9508 corr,
9509 body.to_vec(),
9510 )
9511 .unwrap();
9512 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
9513
9514 assert_eq!(response.len(), 1);
9515 assert_eq!(response[0].header.ty, FrameType::Error);
9516 assert_eq!(response[0].header.corr, corr);
9517 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
9518 }
9519 }
9520
9521 #[tokio::test]
9522 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
9523 let registry = Arc::new(Registry::default());
9524 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
9525 let router = Router::with_control_handler(Arc::clone(&control));
9526 let connection = router.begin_connection();
9527 let (ctx, mut rx) = route_ctx(connection.id());
9528
9529 router
9530 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
9531 .await
9532 .unwrap();
9533 let response = rx.recv().await.unwrap();
9534 let ack = parse_ack(&response);
9535 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9536 let channel = 1;
9537
9538 let goodbye =
9539 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
9540 router.route_for_connection(&ctx, goodbye).await.unwrap();
9541 assert!(rx.try_recv().is_err());
9542 assert!(registry.get_module("aft").unwrap().is_none());
9543
9544 router
9545 .route_for_connection(&ctx, channel_request(channel, 13))
9546 .await
9547 .unwrap();
9548 let error_frame = rx.recv().await.unwrap();
9549 assert_eq!(error_frame.header.ty, FrameType::Error);
9550 assert_eq!(error_frame.header.channel, channel);
9551 }
9552
9553 #[tokio::test]
9554 async fn dropping_router_connection_releases_registration() {
9555 let registry = Arc::new(Registry::default());
9556 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
9557 let router = Router::with_control_handler(control);
9558 let connection = router.begin_connection();
9559 let (ctx, mut rx) = route_ctx(connection.id());
9560
9561 router
9562 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
9563 .await
9564 .unwrap();
9565 let response = rx.recv().await.unwrap();
9566 let ack = parse_ack(&response);
9567 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9568 assert!(registry.get_module("aft").unwrap().is_some());
9569
9570 drop(connection);
9571
9572 assert!(registry.get_module("aft").unwrap().is_none());
9573 assert_eq!(registry.active_registration_count().unwrap(), 0);
9574 }
9575
9576 fn capability_manifest(
9577 module_id: &str,
9578 provides: &[&str],
9579 must_never_reach: &[&str],
9580 ) -> ModuleManifest {
9581 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
9582 manifest.capabilities = Some(CapabilityDeclarations {
9583 provides: provides
9584 .iter()
9585 .map(|capability| (*capability).to_string())
9586 .collect(),
9587 requires: Vec::new(),
9588 must_never_reach: must_never_reach
9589 .iter()
9590 .map(|capability| (*capability).to_string())
9591 .collect(),
9592 });
9593 manifest
9594 }
9595
9596 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
9597 Frame::build(
9598 FrameType::Hello,
9599 control_flags(),
9600 0,
9601 0,
9602 corr,
9603 serde_json::to_vec(&ModuleHelloBody {
9604 protocol_ver: manifest.protocol_ver,
9605 manifest,
9606 control_ops: None,
9607 launch_nonce: None,
9608 })
9609 .expect("capability test HELLO serializes"),
9610 )
9611 .expect("capability test HELLO frame builds")
9612 }
9613
9614 fn catalog_update_with_capabilities_frame(
9615 corr: u64,
9616 capabilities: CapabilityDeclarations,
9617 ) -> Frame {
9618 Frame::build(
9619 FrameType::Request,
9620 control_flags(),
9621 0,
9622 0,
9623 corr,
9624 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
9625 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
9626 capabilities: Some(capabilities),
9627 ready: None,
9628 })
9629 .expect("capability catalog.update serializes"),
9630 )
9631 .expect("capability catalog.update frame builds")
9632 }
9633
9634 async fn register_capability_manifest(
9635 handler: &ControlHandler,
9636 ctx: &RouteCtx,
9637 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9638 manifest: ModuleManifest,
9639 corr: u64,
9640 ) {
9641 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
9642 }
9643
9644 async fn open_route_for_capability_test(
9645 handler: &ControlHandler,
9646 target_ctx: &RouteCtx,
9647 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9648 client_connection_id: u64,
9649 corr: u64,
9650 target_module_id: &str,
9651 consumer_identity: Option<ConsumerIdentity>,
9652 ) -> (
9653 mpsc::Receiver<crate::router::OutboundFrame>,
9654 ModuleControlRequest,
9655 ) {
9656 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
9657 let route_handler = handler.clone();
9658 let target_module_id = target_module_id.to_string();
9659 let route_task = tokio::spawn(async move {
9660 route_handler
9661 .handle_control_frame(
9662 &client_ctx,
9663 route_open_frame_with_admission_facts(
9664 corr,
9665 &target_module_id,
9666 unique_project_root("admission-facts"),
9667 consumer_identity,
9668 None,
9669 ),
9670 )
9671 .await
9672 .expect("capability test route.open succeeds")
9673 });
9674 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
9675 .await
9676 .expect("capability test route.open must reach route.bind")
9677 .expect("target control receiver stays open");
9678 let bind_request: ModuleControlRequest =
9679 serde_json::from_slice(&bind.body).expect("route.bind decodes");
9680 handler
9681 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
9682 .await
9683 .expect("capability test route.bind ACK succeeds");
9684 assert!(route_task.await.expect("route.open task joins").is_empty());
9685 let opened = client_rx
9686 .recv()
9687 .await
9688 .expect("successful route.open publishes a response");
9689 assert!(matches!(
9690 serde_json::from_slice::<ClientControlResponse>(&opened.body),
9691 Ok(ClientControlResponse::RouteOpen { .. })
9692 ));
9693 (client_rx, bind_request)
9694 }
9695
9696 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
9697 assert_eq!(frame.header.ty, FrameType::Push);
9698 assert_eq!(frame.header.channel, 0);
9699 assert_eq!(
9700 serde_json::from_slice::<ClientControlPush>(&frame.body)
9701 .expect("route.closed control push decodes"),
9702 ClientControlPush::RouteClosed {
9703 module_id: target_module_id.to_string(),
9704 reason: RouteCloseReason::CapabilityDenied,
9705 drained: false,
9706 abandoned: 0,
9707 excluded_subscriptions: 0,
9708 terminal: Some(false),
9709 }
9710 );
9711 }
9712
9713 #[tokio::test]
9714 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
9715 let registry = Arc::new(Registry::default());
9716 let forwarding = Arc::new(ForwardingTable::default());
9717 let supervisor = SupervisorHandle::new();
9718 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9719 let handler =
9720 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9721 .with_supervisor(supervisor);
9722 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
9723 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
9724 register_capability_manifest(
9725 &handler,
9726 &target_ctx,
9727 &mut target_rx,
9728 capability_manifest("target", &["credentials-provider/v1"], &[]),
9729 1,
9730 )
9731 .await;
9732 register_capability_manifest(
9733 &handler,
9734 &opener_ctx,
9735 &mut opener_rx,
9736 capability_manifest("opener", &[], &["credentials-provider/v1"]),
9737 2,
9738 )
9739 .await;
9740
9741 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
9742 let replies = handler
9743 .handle_control_frame(
9744 &client_ctx,
9745 route_open_frame_with_admission_facts(
9746 3,
9747 "target",
9748 unique_project_root("admission-facts"),
9749 Some(ConsumerIdentity {
9750 module_id: "opener".to_string(),
9751 launch_nonce: "opener-nonce".to_string(),
9752 }),
9753 None,
9754 ),
9755 )
9756 .await
9757 .expect("denied route.open returns a typed frame");
9758 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
9759 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
9760 assert!(
9761 target_rx.try_recv().is_err(),
9762 "forbidden route.open must not relay route.bind"
9763 );
9764 }
9765
9766 #[tokio::test]
9767 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
9768 let registry = Arc::new(Registry::default());
9769 let forwarding = Arc::new(ForwardingTable::default());
9770 let supervisor = SupervisorHandle::new();
9771 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9772 let handler =
9773 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9774 .with_supervisor(supervisor);
9775 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
9776 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
9777 register_capability_manifest(
9778 &handler,
9779 &target_ctx,
9780 &mut target_rx,
9781 capability_manifest("target", &["credentials-provider/v1"], &[]),
9782 1,
9783 )
9784 .await;
9785 register_capability_manifest(
9786 &handler,
9787 &old_opener_ctx,
9788 &mut old_opener_rx,
9789 capability_manifest("opener", &[], &[]),
9790 2,
9791 )
9792 .await;
9793 let (mut client_rx, _) = open_route_for_capability_test(
9794 &handler,
9795 &target_ctx,
9796 &mut target_rx,
9797 712,
9798 3,
9799 "target",
9800 Some(ConsumerIdentity {
9801 module_id: "opener".to_string(),
9802 launch_nonce: "opener-nonce".to_string(),
9803 }),
9804 )
9805 .await;
9806 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
9807
9808 handler
9809 .cleanup_connection(old_opener_ctx.connection_id)
9810 .expect("old opener registration cleans up");
9811 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
9812 register_capability_manifest(
9813 &handler,
9814 &new_opener_ctx,
9815 &mut new_opener_rx,
9816 capability_manifest("opener", &[], &["credentials-provider/v1"]),
9817 4,
9818 )
9819 .await;
9820
9821 assert_capability_denied_push(
9822 client_rx
9823 .try_recv()
9824 .expect("HELLO deny addition must emit route.closed")
9825 .frame,
9826 "target",
9827 );
9828 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
9829 assert!(matches!(
9830 target_rx.try_recv(),
9831 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
9832 ));
9833 }
9834
9835 #[tokio::test]
9836 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
9837 let registry = Arc::new(Registry::default());
9838 let forwarding = Arc::new(ForwardingTable::default());
9839 let supervisor = SupervisorHandle::new();
9840 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9841 let handler =
9842 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9843 .with_supervisor(supervisor);
9844 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
9845 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
9846 register_capability_manifest(
9847 &handler,
9848 &target_ctx,
9849 &mut target_rx,
9850 capability_manifest("target", &[], &[]),
9851 1,
9852 )
9853 .await;
9854 register_capability_manifest(
9855 &handler,
9856 &opener_ctx,
9857 &mut opener_rx,
9858 capability_manifest("opener", &[], &["credentials-provider/v1"]),
9859 2,
9860 )
9861 .await;
9862 let (mut client_rx, _) = open_route_for_capability_test(
9863 &handler,
9864 &target_ctx,
9865 &mut target_rx,
9866 722,
9867 3,
9868 "target",
9869 Some(ConsumerIdentity {
9870 module_id: "opener".to_string(),
9871 launch_nonce: "opener-nonce".to_string(),
9872 }),
9873 )
9874 .await;
9875 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
9876
9877 let replies = handler
9878 .handle_control_frame(
9879 &target_ctx,
9880 catalog_update_with_capabilities_frame(
9881 4,
9882 CapabilityDeclarations {
9883 provides: vec!["credentials-provider/v1".to_string()],
9884 requires: Vec::new(),
9885 must_never_reach: Vec::new(),
9886 },
9887 ),
9888 )
9889 .await
9890 .expect("claim catalog.update succeeds");
9891 assert!(matches!(
9892 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
9893 Ok(ModuleControlResponseToModule::CatalogUpdate {})
9894 ));
9895 assert_capability_denied_push(
9896 client_rx
9897 .try_recv()
9898 .expect("claim addition must emit route.closed")
9899 .frame,
9900 "target",
9901 );
9902 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
9903 assert!(matches!(
9904 target_rx.try_recv(),
9905 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
9906 ));
9907 }
9908
9909 #[tokio::test]
9910 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
9911 let registry = Arc::new(Registry::default());
9912 let forwarding = Arc::new(ForwardingTable::default());
9913 let supervisor = SupervisorHandle::new();
9914 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9915 let handler =
9916 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9917 .with_supervisor(supervisor);
9918 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
9919 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
9920 register_capability_manifest(
9921 &handler,
9922 &target_ctx,
9923 &mut target_rx,
9924 capability_manifest("target", &["credentials-provider/v1"], &[]),
9925 1,
9926 )
9927 .await;
9928 register_capability_manifest(
9929 &handler,
9930 &opener_ctx,
9931 &mut opener_rx,
9932 capability_manifest("opener", &[], &[]),
9933 2,
9934 )
9935 .await;
9936 let (mut client_rx, _) = open_route_for_capability_test(
9937 &handler,
9938 &target_ctx,
9939 &mut target_rx,
9940 732,
9941 3,
9942 "target",
9943 Some(ConsumerIdentity {
9944 module_id: "opener".to_string(),
9945 launch_nonce: "opener-nonce".to_string(),
9946 }),
9947 )
9948 .await;
9949 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
9950
9951 handler
9952 .handle_control_frame(
9953 &target_ctx,
9954 catalog_update_with_capabilities_frame(
9955 4,
9956 CapabilityDeclarations {
9957 provides: Vec::new(),
9958 requires: Vec::new(),
9959 must_never_reach: Vec::new(),
9960 },
9961 ),
9962 )
9963 .await
9964 .expect("claim removal catalog.update succeeds");
9965 assert_eq!(
9966 forwarding.active_binding_count().unwrap(),
9967 1,
9968 "removing an attested target claim must leave the route census unchanged"
9969 );
9970 assert!(
9971 client_rx.try_recv().is_err(),
9972 "claim removal must not emit route.closed capability_denied"
9973 );
9974 assert!(
9975 target_rx.try_recv().is_err(),
9976 "claim removal must not send the target a route GOODBYE"
9977 );
9978 }
9979
9980 #[tokio::test]
9983 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
9984 let registry = Arc::new(Registry::default());
9985 let forwarding = Arc::new(ForwardingTable::default());
9986 let supervisor = SupervisorHandle::new();
9987 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9988 let handler =
9989 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9990 .with_supervisor(supervisor);
9991 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
9992 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
9993 register_capability_manifest(
9994 &handler,
9995 &target_ctx,
9996 &mut target_rx,
9997 capability_manifest("target", &["credentials-provider/v1"], &[]),
9998 1,
9999 )
10000 .await;
10001 register_capability_manifest(
10002 &handler,
10003 &opener_ctx,
10004 &mut opener_rx,
10005 capability_manifest("opener", &[], &["credentials-provider/v1"]),
10006 2,
10007 )
10008 .await;
10009
10010 let (_client_rx, bind) = open_route_for_capability_test(
10011 &handler,
10012 &target_ctx,
10013 &mut target_rx,
10014 742,
10015 3,
10016 "target",
10017 None,
10018 )
10019 .await;
10020 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
10021 panic!("direct scope-honesty route must bind");
10022 };
10023 assert_eq!(principal, Some(Principal::Direct));
10024 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
10025 }
10026
10027 #[tokio::test]
10030 async fn must_never_reach_self_route_is_capability_forbidden() {
10031 let registry = Arc::new(Registry::default());
10032 let forwarding = Arc::new(ForwardingTable::default());
10033 let supervisor = SupervisorHandle::new();
10034 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
10035 let handler =
10036 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10037 .with_supervisor(supervisor);
10038 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
10039 register_capability_manifest(
10040 &handler,
10041 &self_ctx,
10042 &mut self_rx,
10043 capability_manifest(
10044 "self-provider",
10045 &["credentials-provider/v1"],
10046 &["credentials-provider/v1"],
10047 ),
10048 1,
10049 )
10050 .await;
10051
10052 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
10053 let replies = handler
10054 .handle_control_frame(
10055 &client_ctx,
10056 route_open_frame_with_admission_facts(
10057 2,
10058 "self-provider",
10059 unique_project_root("admission-facts"),
10060 Some(ConsumerIdentity {
10061 module_id: "self-provider".to_string(),
10062 launch_nonce: "self-nonce".to_string(),
10063 }),
10064 None,
10065 ),
10066 )
10067 .await
10068 .expect("self-route refusal returns a typed frame");
10069 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
10070 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
10071 assert!(
10072 self_rx.try_recv().is_err(),
10073 "self denial must not relay route.bind"
10074 );
10075 }
10076
10077 #[test]
10078 fn unsupported_channel_zero_frame_returns_error() {
10079 let handler = ControlHandler::default();
10080 let request = Frame::build(
10081 FrameType::Request,
10082 control_flags(),
10083 0,
10084 0,
10085 21,
10086 b"opaque".to_vec(),
10087 )
10088 .unwrap();
10089
10090 let response = handler
10091 .handle_control(ConnectionId::new(1), request)
10092 .unwrap();
10093
10094 assert_eq!(response[0].header.ty, FrameType::Error);
10095 assert_eq!(
10096 parse_error(&response[0])["code"],
10097 "unsupported_control_frame"
10098 );
10099 }
10100
10101 mod swap {
10106 use super::*;
10107
10108 const INCUMBENT: ConnectionId = ConnectionId::new(30);
10109 const CANDIDATE: ConnectionId = ConnectionId::new(40);
10110
10111 struct Swap {
10112 registry: Arc<Registry>,
10113 forwarding: Arc<ForwardingTable>,
10114 handler: ControlHandler,
10115 incumbent_ctx: RouteCtx,
10116 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
10117 candidate_ctx: RouteCtx,
10118 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
10119 }
10120
10121 async fn swap_with_incumbent() -> Swap {
10122 let registry = Arc::new(Registry::default());
10123 let forwarding = Arc::new(ForwardingTable::default());
10124 let handler =
10125 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10126 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
10127 hello_via_sink(
10128 &handler,
10129 &incumbent_ctx,
10130 &mut incumbent_rx,
10131 hello_frame("aft", PROTOCOL_VERSION, 7),
10132 )
10133 .await;
10134 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
10135 Swap {
10136 registry,
10137 forwarding,
10138 handler,
10139 incumbent_ctx,
10140 incumbent_rx,
10141 candidate_ctx,
10142 candidate_rx,
10143 }
10144 }
10145
10146 fn register_candidate(swap: &Swap, ready: Option<bool>) {
10147 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
10148 candidate_manifest.ready = ready;
10149 let registration = swap
10150 .registry
10151 .register_candidate_with_control_ops(
10152 candidate_manifest,
10153 PROTOCOL_VERSION,
10154 CANDIDATE,
10155 module_baseline_control_ops(),
10156 )
10157 .unwrap();
10158 swap.forwarding
10159 .register_candidate_module_connection(
10160 CANDIDATE,
10161 "aft".to_string(),
10162 PROTOCOL_VERSION,
10163 manifest_concurrency(®istration.manifest),
10164 swap.candidate_ctx.egress.clone(),
10165 )
10166 .unwrap();
10167 }
10168
10169 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
10170 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
10171 swap.registry.promote_candidate("aft").unwrap().unwrap();
10172 cutover.incumbent.unwrap()
10173 }
10174
10175 fn keyed_total(counters: &Value, key: &str) -> u64 {
10176 counters[key]
10177 .as_object()
10178 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
10179 .unwrap_or(0)
10180 }
10181
10182 #[tokio::test]
10188 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
10189 let mut swap = swap_with_incumbent().await;
10190 let handler = swap.handler.clone();
10191
10192 let cotenant = ConnectionId::new(31);
10194 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
10195 let (cotenant_task, cotenant_bind) = relay_route_open(
10196 &handler,
10197 cotenant,
10198 &cotenant_ctx.egress,
10199 &mut swap.incumbent_rx,
10200 100,
10201 "aft",
10202 "swap-cotenant",
10203 )
10204 .await;
10205 handler
10206 .handle_control_frame(
10207 &swap.incumbent_ctx,
10208 route_bind_ack(cotenant_bind.header.corr),
10209 )
10210 .await
10211 .unwrap();
10212 assert!(cotenant_task.await.unwrap().is_empty());
10213 let (cotenant_channel, cotenant_epoch) =
10214 published_route(&cotenant_rx.recv().await.unwrap());
10215
10216 let caller = ConnectionId::new(32);
10218 let (caller_ctx, mut caller_rx) = route_ctx(caller);
10219 let (caller_task, caller_bind) = relay_route_open(
10220 &handler,
10221 caller,
10222 &caller_ctx.egress,
10223 &mut swap.incumbent_rx,
10224 101,
10225 "aft",
10226 "swap-caller",
10227 )
10228 .await;
10229 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
10230
10231 register_candidate(&swap, None);
10232 cutover(&swap);
10233
10234 let ack = handler
10236 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
10237 .await;
10238 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
10239 if module_loop_error.is_some() {
10240 handler.cleanup_connection(INCUMBENT).unwrap();
10243 }
10244
10245 assert!(
10247 cotenant_rx.try_recv().is_err(),
10248 "the co-tenant route on the incumbent was torn down by one late ack: \
10249 {module_loop_error:?}"
10250 );
10251 assert!(matches!(
10252 swap.forwarding
10253 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
10254 .unwrap(),
10255 DataRoute::Client(DataRouteState::Bound(_))
10256 ));
10257 assert_eq!(module_loop_error, None);
10258 assert!(swap
10259 .registry
10260 .get_module_by_connection(INCUMBENT)
10261 .unwrap()
10262 .is_some());
10263
10264 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
10266 .await
10267 .expect("the incumbent is told to drop the abandoned binding")
10268 .unwrap()
10269 .frame;
10270 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10271 assert_eq!(goodbye.header.channel, abandoned_channel);
10272 assert_eq!(goodbye.header.epoch, abandoned_epoch);
10273 assert!(swap.incumbent_rx.try_recv().is_err());
10274
10275 let response = caller_task.await.unwrap();
10277 assert_eq!(response.len(), 1);
10278 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
10279 assert!(caller_rx.try_recv().is_err());
10280
10281 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
10285 let counters = handler.counters().snapshot();
10286 assert_eq!(
10287 keyed_total(&counters, "route_open_accepted_by_principal"),
10288 1
10289 );
10290 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
10291 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
10292 }
10293
10294 #[tokio::test]
10298 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
10299 let mut swap = swap_with_incumbent().await;
10300 register_candidate(&swap, None);
10301 let incumbent = cutover(&swap);
10302 swap.forwarding
10303 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
10304 .unwrap()
10305 .expect("the incumbent is still registered");
10306
10307 let client = ConnectionId::new(33);
10308 let (client_ctx, mut client_rx) = route_ctx(client);
10309 let route_handler = swap.handler.clone();
10310 let open_ctx = RouteCtx {
10311 connection_id: client,
10312 egress: client_ctx.egress.clone(),
10313 };
10314 let mut route_task = tokio::spawn(async move {
10315 route_handler
10316 .handle_control_frame(
10317 &open_ctx,
10318 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
10319 )
10320 .await
10321 .unwrap()
10322 });
10323 let bind = tokio::select! {
10324 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
10325 response = &mut route_task => {
10326 let response = response.unwrap();
10327 panic!(
10328 "post-cutover route.open was refused instead of relayed to the candidate: {}",
10329 parse_error(&response[0])["code"]
10330 );
10331 }
10332 };
10333 swap.handler
10334 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
10335 .await
10336 .unwrap();
10337 assert!(route_task.await.unwrap().is_empty());
10338 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
10339 match swap
10340 .forwarding
10341 .lookup_data_route(client, channel, epoch)
10342 .unwrap()
10343 {
10344 DataRoute::Client(DataRouteState::Bound(route)) => {
10345 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
10346 }
10347 other => panic!("expected a bound route on the candidate, got {other:?}"),
10348 }
10349 assert!(swap.incumbent_rx.try_recv().is_err());
10350 }
10351
10352 #[tokio::test]
10357 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
10358 let swap = swap_with_incumbent().await;
10359 register_candidate(&swap, Some(false));
10360 let update = Frame::build(
10361 FrameType::Request,
10362 control_flags(),
10363 0,
10364 0,
10365 55,
10366 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
10367 provides: manifest("aft", PROTOCOL_VERSION).provides,
10368 capabilities: None,
10369 ready: Some(true),
10370 })
10371 .unwrap(),
10372 )
10373 .unwrap();
10374
10375 let replies = swap
10376 .handler
10377 .handle_control_frame(&swap.candidate_ctx, update)
10378 .await
10379 .unwrap();
10380
10381 assert_eq!(replies.len(), 1);
10382 assert_eq!(
10383 replies[0].header.ty,
10384 FrameType::Response,
10385 "candidate catalog.update was refused: {:?}",
10386 serde_json::from_slice::<Value>(&replies[0].body).ok()
10387 );
10388 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
10389 assert_eq!(
10390 swap.registry
10391 .get_module("aft")
10392 .unwrap()
10393 .unwrap()
10394 .connection_id,
10395 INCUMBENT
10396 );
10397 }
10398 }
10399
10400 mod swap_admission {
10404 use super::*;
10405
10406 const INCUMBENT_NONCE: &str = "incumbent-nonce";
10407 const CANDIDATE_NONCE: &str = "candidate-nonce";
10408
10409 fn handler_with_incumbent(
10410 module_id: &str,
10411 reserved: bool,
10412 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
10413 let registry = Arc::new(Registry::default());
10414 let supervisor = SupervisorHandle::new();
10415 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
10416 if reserved {
10417 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
10418 }
10419 let handler =
10420 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
10421 let incumbent = handler
10422 .handle_control(
10423 ConnectionId::new(1),
10424 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
10425 )
10426 .unwrap();
10427 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
10428 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
10429 (registry, supervisor, handler)
10430 }
10431
10432 #[test]
10438 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
10439 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
10440
10441 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
10442 let replies = handler
10443 .handle_control(
10444 ConnectionId::new(connection),
10445 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
10446 )
10447 .unwrap();
10448 assert_eq!(replies[0].header.ty, FrameType::Error);
10449 assert_eq!(
10450 parse_error(&replies[0])["code"],
10451 "swap_token_invalid",
10452 "nonce {nonce:?}"
10453 );
10454 }
10455 assert!(registry.get_candidate("aft").unwrap().is_none());
10456 assert_eq!(
10457 registry.get_module("aft").unwrap().unwrap().connection_id,
10458 ConnectionId::new(1)
10459 );
10460
10461 let admitted = handler
10464 .handle_control(
10465 ConnectionId::new(4),
10466 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
10467 )
10468 .unwrap();
10469 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
10470 assert_eq!(
10471 registry
10472 .get_candidate("aft")
10473 .unwrap()
10474 .unwrap()
10475 .connection_id,
10476 ConnectionId::new(4)
10477 );
10478 assert_eq!(
10479 registry.get_module("aft").unwrap().unwrap().connection_id,
10480 ConnectionId::new(1),
10481 "the candidate must not take the active slot"
10482 );
10483 let replayed = handler
10484 .handle_control(
10485 ConnectionId::new(5),
10486 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
10487 )
10488 .unwrap();
10489 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
10490
10491 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
10495 let squatter = handler
10496 .handle_control(
10497 ConnectionId::new(6),
10498 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
10499 )
10500 .unwrap();
10501 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
10502 assert!(
10503 registry.get_module("aft").unwrap().is_none(),
10504 "a squatter took the active slot of an id being swapped"
10505 );
10506 }
10507
10508 #[test]
10513 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
10514 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
10515
10516 let replies = handler
10517 .handle_control(
10518 ConnectionId::new(2),
10519 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
10520 )
10521 .unwrap();
10522
10523 assert_eq!(
10524 replies[0].header.ty,
10525 FrameType::HelloAck,
10526 "reserved candidate refused: {:?}",
10527 serde_json::from_slice::<Value>(&replies[0].body).ok()
10528 );
10529 assert_eq!(
10530 registry
10531 .get_candidate("vault")
10532 .unwrap()
10533 .unwrap()
10534 .connection_id,
10535 ConnectionId::new(2)
10536 );
10537 }
10538
10539 #[test]
10542 fn without_an_open_swap_the_ordinary_gates_decide() {
10543 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
10544 supervisor.close_swap("vault");
10545
10546 let candidate = handler
10547 .handle_control(
10548 ConnectionId::new(2),
10549 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
10550 )
10551 .unwrap();
10552 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
10553 let duplicate = handler
10554 .handle_control(
10555 ConnectionId::new(3),
10556 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
10557 )
10558 .unwrap();
10559 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
10560 assert!(registry.get_candidate("vault").unwrap().is_none());
10561 }
10562 }
10563}
10564
10565#[cfg(test)]
10566mod concurrency_default_exposure_tests {
10567 use super::*;
10568
10569 fn hello_body(role_json: &str) -> Vec<u8> {
10570 format!(
10571 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
10572 )
10573 .into_bytes()
10574 }
10575
10576 fn manifest_from(body: &[u8]) -> ModuleManifest {
10577 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
10578 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
10579 .expect("manifest parses")
10580 }
10581
10582 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
10583
10584 #[test]
10585 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
10586 let body = hello_body(&format!(
10587 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
10588 ));
10589 let manifest = manifest_from(&body);
10590 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
10593 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
10594 }
10595
10596 #[test]
10597 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
10598 let body = hello_body(&format!(
10599 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
10600 ));
10601 let manifest = manifest_from(&body);
10602 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
10603 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
10604 }
10605
10606 #[test]
10607 fn non_management_roles_are_never_reported() {
10608 let body = hello_body(
10609 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
10610 );
10611 let manifest = manifest_from(&body);
10612 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
10613 }
10614}