1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 path::{Path, PathBuf},
5 sync::{Arc, Mutex},
6 time::{Duration, Instant as StdInstant},
7};
8
9use serde::{Deserialize, Serialize};
10use subc_control::{
11 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
12 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
13 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
14 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
15 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
16 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
17 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
18};
19use subc_protocol::{
20 error_codes,
21 manifest::{
22 validate_hello_capability_grammar, validate_hello_self_signal_declarations,
23 CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
24 ProviderRole,
25 },
26 session::{
27 HealthReport, ModuleControlPush, ModuleControlRequest, ModuleControlRequestFromModule,
28 ModuleControlResponse, ModuleControlResponseToModule, MODULE_CONTROL_OP_HEALTH_CHECK,
29 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
30 },
31 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
32 Priority, RouteTarget, PROTOCOL_VERSION,
33};
34use tokio::time::{timeout_at, Instant};
35use tracing::{debug, info, warn};
36
37use crate::{
38 capability_requirements::{
39 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
40 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
41 RuntimeModule,
42 },
43 daemon_config::RestartRequiredSection,
44 forwarding::{
45 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
46 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
47 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
48 },
49 observability::{
50 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
51 },
52 provenance::{
53 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
54 },
55 registry::{ChannelState, ConnectionId, Registry, RegistryError},
56 router::{RouteCtx, RouterError},
57 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
58 stderr_tail::{CaptureState, TailEntry},
59 supervise::{
60 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
61 SupervisorHandle, SwapHelloAdmission,
62 },
63 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
64};
65
66pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
72
73const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
74const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
75const CAP_PING_PONG: &str = "ping_pong_v1";
76const CAP_SESSION_ATTACH: &str = "session_attach_v1";
77const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
78
79const SUBC_CONTROL_OPS: &[&str] = &[
80 ops::SERVER_DESCRIBE,
81 ops::CATALOG_LIST,
82 ops::ROUTE_OPEN,
83 ops::ROUTE_POLL,
84 ops::ROUTE_CLOSING,
85 ops::ROUTE_CLOSED,
86 ops::SUPERVISOR_LIST,
87 ops::SUPERVISOR_RESTART,
88 ops::SUPERVISOR_SWAP,
89 ops::SUPERVISOR_RELOAD,
90 ops::SUPERVISOR_RESCAN,
91 ops::SUPERVISOR_RELEASE_RESERVED,
92 ops::SUPERVISOR_SET_ENABLED,
93 ops::SUPERVISOR_HEALTH_PROBE,
94 ops::SUPERVISOR_HEALTH,
95 ops::SUPERVISOR_STDERR_TAIL,
96 ops::SUPERVISOR_TERMINALS,
97 ops::SUPERVISOR_ROUTES,
98 ops::SUPERVISOR_PROVENANCE,
99 ops::SUPERVISOR_SPAWN_SNAPSHOT,
100 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
101];
102
103const MODULE_TO_SUBC_CONTROL_OPS: &[&str] =
104 &[MODULE_TO_SUBC_OP_CATALOG_UPDATE, "supervisor.live_roots"];
105
106const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
107
108pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
116
117pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
132
133pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
151
152const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
153const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
154
155fn reload_verdict(
156 configured: &Path,
157 spawned_from: Option<&Path>,
158 image: subc_control::RunningImageAgreement,
159) -> PendingReloadVerdict {
160 let path = match spawned_from {
161 Some(spawned_from) if configured == spawned_from => ReloadPathAgreement::Match,
162 Some(spawned_from) => ReloadPathAgreement::Mismatch {
163 configured: configured.to_path_buf(),
164 spawned_from: spawned_from.to_path_buf(),
165 },
166 None => ReloadPathAgreement::Unavailable {
167 reason: if matches!(
168 image,
169 subc_control::RunningImageAgreement::Unavailable {
170 reason: subc_control::RunningImageUnavailableReason::NotRunning
171 }
172 ) {
173 ReloadPathUnavailableReason::NotRunning
174 } else {
175 ReloadPathUnavailableReason::SpawnedPathUnavailable
176 },
177 },
178 };
179 PendingReloadVerdict { path, image }
180}
181
182#[derive(Clone)]
183struct DaemonProvenanceFacts {
184 build: DaemonBuildProvenance,
185 pid: Option<u32>,
186 started_at_ms: Option<u64>,
187 start_clock: Option<crate::clock::StartClock>,
188 executable_path: Option<PathBuf>,
189 executable_identity: Option<SpawnedFileIdentity>,
190 process_start_time: Option<u64>,
191 probe: ExecutableIdentityProbe,
192}
193
194impl Default for DaemonProvenanceFacts {
195 fn default() -> Self {
196 Self {
197 build: DaemonBuildProvenance {
198 build_git_sha: None,
199 build_lock_digest: None,
200 },
201 pid: None,
202 started_at_ms: None,
203 start_clock: None,
204 executable_path: None,
205 executable_identity: None,
206 process_start_time: None,
207 probe: ExecutableIdentityProbe::default(),
208 }
209 }
210}
211
212#[derive(Debug, Clone)]
213struct SupervisorRescanContext {
214 supervisor: Supervisor,
215 config_path: PathBuf,
216 configured_port: Option<u16>,
217 storage_config: Option<crate::daemon_config::StorageConfig>,
218 admission_facts_carrier_module_id: Option<String>,
219 admission_facts_targets: Option<Vec<String>>,
220}
221
222#[derive(Clone)]
224pub struct ControlHandler {
225 registry: Arc<Registry>,
226 forwarding: Arc<ForwardingTable>,
227 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
228 supervisor: SupervisorHandle,
229 subc_capabilities: Arc<[String]>,
230 route_bind_relay_timeout: Duration,
234 route_bind_relay_timeouts: BTreeMap<String, Duration>,
238 route_bind_breakers: RouteBindBreakers,
241 route_bind_concurrency: RouteBindConcurrency,
244 route_bind_breaker_threshold: u32,
246 route_bind_breaker_cooldown: Duration,
248 health_probe_timeout: Duration,
249 storage_config: Option<crate::daemon_config::StorageConfig>,
252 machine_id: Option<crate::machine_id::MachineId>,
256 admission_facts_carrier_module_id: Option<String>,
257 admission_facts_targets: Option<Vec<String>>,
258 rescan: Option<SupervisorRescanContext>,
259 connected_clients: ConnectedClients,
260 counters: DaemonCounters,
261 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
262 daemon_provenance: DaemonProvenanceFacts,
263 #[cfg(test)]
264 control_dispatch_delay: Option<Duration>,
265 #[cfg(test)]
266 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
267}
268
269impl fmt::Debug for ControlHandler {
270 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
271 f.debug_struct("ControlHandler")
272 .field("registry", &self.registry)
273 .field("forwarding", &self.forwarding)
274 .field("process_liveness", &self.process_liveness.is_some())
275 .field("supervisor", &self.supervisor)
276 .field("subc_capabilities", &self.subc_capabilities)
277 .finish()
278 }
279}
280
281struct RouteOpenRequest {
282 target: RouteTarget,
283 identity: BindIdentity,
284 consumer_identity: Option<ConsumerIdentity>,
285 consumer_capabilities: Option<Vec<String>>,
286 admission_facts: Option<serde_json::Value>,
287}
288
289struct RouteBindReservationGuard {
290 forwarding: Arc<ForwardingTable>,
291 endpoint: ModuleEndpointId,
292 relay_corr: u64,
293 armed: bool,
294}
295
296struct ModuleControlRpcGuard {
297 forwarding: Arc<ForwardingTable>,
298 endpoint: ModuleEndpointId,
299 corr: u64,
300 armed: bool,
301}
302
303impl ModuleControlRpcGuard {
304 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
305 Self {
306 forwarding,
307 endpoint,
308 corr,
309 armed: true,
310 }
311 }
312
313 fn disarm(&mut self) {
314 self.armed = false;
315 }
316}
317
318impl Drop for ModuleControlRpcGuard {
319 fn drop(&mut self) {
320 if self.armed {
321 let _ = self
322 .forwarding
323 .cancel_module_control_rpc(self.endpoint, self.corr);
324 }
325 }
326}
327
328impl RouteBindReservationGuard {
329 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
330 Self {
331 forwarding,
332 endpoint,
333 relay_corr,
334 armed: true,
335 }
336 }
337
338 fn release_and_disarm(&mut self) {
339 if !self.armed {
340 return;
341 }
342 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
343 self.endpoint,
344 self.relay_corr,
345 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
346 ) {
347 send_goodbye_target_best_effort(
348 &self.forwarding.counters(),
349 &target,
350 "canceled route.bind",
351 );
352 }
353 self.armed = false;
354 }
355
356 fn disarm(&mut self) {
357 self.armed = false;
358 }
359}
360
361impl Drop for RouteBindReservationGuard {
362 fn drop(&mut self) {
363 self.release_and_disarm();
364 }
365}
366
367#[derive(Debug, Clone, Default)]
397pub(crate) struct RouteBindBreakers {
398 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
399}
400
401#[derive(Debug, Clone, Default)]
402pub(crate) struct RouteBindConcurrency {
403 modules: Arc<Mutex<HashMap<String, usize>>>,
404}
405
406struct RouteBindConcurrencyGuard {
407 concurrency: RouteBindConcurrency,
408 module_id: String,
409}
410
411impl RouteBindConcurrency {
412 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
415 let mut modules = self
416 .modules
417 .lock()
418 .expect("route.bind concurrency mutex poisoned");
419 let in_flight = modules.entry(module_id.to_string()).or_default();
420 if *in_flight >= limit {
421 return Err(*in_flight);
422 }
423 *in_flight += 1;
424 Ok(RouteBindConcurrencyGuard {
425 concurrency: self.clone(),
426 module_id: module_id.to_string(),
427 })
428 }
429}
430
431impl Drop for RouteBindConcurrencyGuard {
432 fn drop(&mut self) {
433 let mut modules = self
434 .concurrency
435 .modules
436 .lock()
437 .expect("route.bind concurrency mutex poisoned");
438 let remove = {
439 let in_flight = modules
440 .get_mut(&self.module_id)
441 .expect("admitted route.bind has a concurrency entry");
442 *in_flight -= 1;
443 *in_flight == 0
444 };
445 if remove {
446 modules.remove(&self.module_id);
447 }
448 }
449}
450
451#[derive(Debug, Default)]
452struct ModuleBreakerState {
453 consecutive_timeouts: u32,
455 cooldown_until: Option<Instant>,
458 probe_in_flight: bool,
463}
464
465enum RouteBindAdmission<'a> {
467 Admitted {
468 guard: RouteBindBreakerGuard<'a>,
469 probe: bool,
472 },
473 Refused {
474 consecutive_timeouts: u32,
475 retry_in: Duration,
479 probe_in_flight: bool,
480 },
481}
482
483struct RouteBindBreakerGuard<'a> {
490 breakers: RouteBindBreakers,
491 module_id: &'a str,
492 settled: bool,
493}
494
495impl RouteBindBreakerGuard<'_> {
496 fn record_accepted(&mut self) -> bool {
500 self.settled = true;
501 self.breakers.record_accepted(self.module_id)
502 }
503
504 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
507 self.settled = true;
508 self.breakers
509 .record_timeout(self.module_id, threshold, cooldown)
510 }
511
512 fn record_inconclusive(&mut self) {
521 self.settled = true;
522 self.breakers.record_inconclusive(self.module_id);
523 }
524}
525
526impl Drop for RouteBindBreakerGuard<'_> {
527 fn drop(&mut self) {
528 if !self.settled {
529 self.breakers.record_inconclusive(self.module_id);
530 }
531 }
532}
533
534struct BreakerOpened {
538 consecutive_timeouts: u32,
539 reopened_after_probe: bool,
542}
543
544impl RouteBindBreakers {
545 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
546 self.modules
547 .lock()
548 .expect("route.bind breaker mutex poisoned")
549 }
550
551 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
554 let admitted = |probe| RouteBindAdmission::Admitted {
555 guard: RouteBindBreakerGuard {
556 breakers: self.clone(),
557 module_id,
558 settled: false,
559 },
560 probe,
561 };
562
563 let mut modules = self.lock();
564 let Some(state) = modules.get_mut(module_id) else {
565 return admitted(false);
566 };
567 let Some(cooldown_until) = state.cooldown_until else {
568 return admitted(false);
569 };
570 if state.probe_in_flight {
571 return RouteBindAdmission::Refused {
572 consecutive_timeouts: state.consecutive_timeouts,
573 retry_in: Duration::ZERO,
574 probe_in_flight: true,
575 };
576 }
577 let now = Instant::now();
578 if now < cooldown_until {
579 return RouteBindAdmission::Refused {
580 consecutive_timeouts: state.consecutive_timeouts,
581 retry_in: cooldown_until - now,
582 probe_in_flight: false,
583 };
584 }
585 state.probe_in_flight = true;
586 admitted(true)
587 }
588
589 fn record_accepted(&self, module_id: &str) -> bool {
590 self.lock()
591 .remove(module_id)
592 .is_some_and(|state| state.cooldown_until.is_some())
593 }
594
595 fn record_timeout(
596 &self,
597 module_id: &str,
598 threshold: u32,
599 cooldown: Duration,
600 ) -> Option<BreakerOpened> {
601 let mut modules = self.lock();
602 let state = modules.entry(module_id.to_string()).or_default();
603 let was_open = state.cooldown_until.is_some();
604 let was_probe = state.probe_in_flight;
605 state.probe_in_flight = false;
606 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
607 if state.consecutive_timeouts < threshold {
608 return None;
609 }
610 state.cooldown_until = Some(Instant::now() + cooldown);
611 Some(BreakerOpened {
612 consecutive_timeouts: state.consecutive_timeouts,
613 reopened_after_probe: was_open && was_probe,
614 })
615 }
616
617 fn record_inconclusive(&self, module_id: &str) {
618 if let Some(state) = self.lock().get_mut(module_id) {
619 state.probe_in_flight = false;
620 }
621 }
622
623 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
638 self.lock()
639 .remove(module_id)
640 .map(|state| state.consecutive_timeouts)
641 .filter(|discarded| *discarded > 0)
642 }
643
644 fn open_snapshot(&self) -> Option<serde_json::Value> {
651 let now = Instant::now();
652 let modules = self.lock();
653 let open = modules
654 .iter()
655 .filter_map(|(module_id, state)| {
656 let cooldown_until = state.cooldown_until?;
657 Some((
658 module_id.clone(),
659 serde_json::json!({
660 "consecutive_timeouts": state.consecutive_timeouts,
661 "cooldown_remaining_ms":
662 cooldown_until.saturating_duration_since(now).as_millis() as u64,
663 "probe_in_flight": state.probe_in_flight,
664 }),
665 ))
666 })
667 .collect::<serde_json::Map<String, serde_json::Value>>();
668 (!open.is_empty()).then_some(serde_json::Value::Object(open))
669 }
670}
671
672impl ControlHandler {
673 pub fn new(registry: Arc<Registry>) -> Self {
674 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
675 }
676
677 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
678 let counters = forwarding.counters();
679 let route_bind_breakers = forwarding.route_bind_breakers();
683 let route_bind_concurrency = forwarding.route_bind_concurrency();
684 Self {
685 registry,
686 forwarding,
687 process_liveness: None,
688 supervisor: SupervisorHandle::new(),
689 subc_capabilities: Arc::from([
690 CAP_MANIFEST_REGISTRATION.to_string(),
691 CAP_CHANNEL_LIFECYCLE.to_string(),
692 CAP_PING_PONG.to_string(),
693 CAP_SESSION_ATTACH.to_string(),
694 CAP_ADMISSION_FACTS_RELAY.to_string(),
695 ]),
696 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
697 route_bind_relay_timeouts: BTreeMap::new(),
698 route_bind_breakers,
699 route_bind_concurrency,
700 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
701 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
702 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
703 storage_config: None,
704 machine_id: None,
705 admission_facts_carrier_module_id: None,
706 admission_facts_targets: None,
707 rescan: None,
708 connected_clients: ConnectedClients::new(),
709 counters,
710 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
711 daemon_provenance: DaemonProvenanceFacts::default(),
712 #[cfg(test)]
713 control_dispatch_delay: None,
714 #[cfg(test)]
715 provenance_probe_override: None,
716 }
717 }
718
719 pub fn with_storage_config(
722 mut self,
723 storage_config: Option<crate::daemon_config::StorageConfig>,
724 ) -> Self {
725 self.storage_config = storage_config;
726 self
727 }
728
729 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
732 self.machine_id = machine_id;
733 self
734 }
735
736 pub fn with_admission_facts_config(
740 mut self,
741 carrier_module_id: Option<String>,
742 targets: Option<Vec<String>>,
743 ) -> Self {
744 self.admission_facts_carrier_module_id = carrier_module_id;
745 self.admission_facts_targets = targets;
746 self
747 }
748
749 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
752 self.route_bind_relay_timeout = timeout;
753 self
754 }
755
756 pub fn with_route_bind_relay_timeouts(
762 mut self,
763 timeouts: impl IntoIterator<Item = (String, Duration)>,
764 ) -> Self {
765 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
766 self
767 }
768
769 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
775 self.route_bind_relay_timeouts
776 .get(module_id)
777 .copied()
778 .unwrap_or(self.route_bind_relay_timeout)
779 }
780
781 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
789 self.route_bind_breaker_threshold = threshold.max(1);
790 self.route_bind_breaker_cooldown = cooldown;
791 self
792 }
793
794 #[cfg(test)]
795 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
796 self.health_probe_timeout = timeout;
797 self
798 }
799
800 #[cfg(test)]
801 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
802 self.control_dispatch_delay = Some(delay);
803 self
804 }
805
806 pub fn with_process_liveness(
807 mut self,
808 process_liveness: Arc<dyn ModuleProcessLiveness>,
809 ) -> Self {
810 self.process_liveness = Some(process_liveness);
811 self
812 }
813
814 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
815 self.supervisor = supervisor;
816 self
817 }
818
819 pub fn with_daemon_provenance(
820 mut self,
821 pid: u32,
822 started_at_ms: u64,
823 executable_path: Option<PathBuf>,
824 build_git_sha: Option<String>,
825 build_lock_digest: Option<String>,
826 ) -> Self {
827 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
828 let process_start_time = process_start_time(pid);
829 self.daemon_provenance = DaemonProvenanceFacts {
830 build: DaemonBuildProvenance {
831 build_git_sha,
832 build_lock_digest,
833 },
834 pid: Some(pid),
835 started_at_ms: Some(started_at_ms),
836 start_clock: None,
837 executable_path,
838 executable_identity,
839 process_start_time,
840 probe: ExecutableIdentityProbe::default(),
841 };
842 self
843 }
844
845 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
846 self.daemon_provenance.start_clock = Some(clock);
847 self
848 }
849
850 #[cfg(test)]
851 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
852 self.provenance_probe_override = Some(result);
853 self
854 }
855
856 pub fn with_capability_config(
860 self,
861 modules: impl IntoIterator<Item = (String, bool)>,
862 reserved_capabilities: BTreeMap<String, String>,
863 ) -> Self {
864 self.capability_evaluator
865 .configure(modules, reserved_capabilities);
866 self
867 }
868
869 pub fn with_supervisor_rescan(
870 mut self,
871 supervisor: Supervisor,
872 config_path: impl Into<PathBuf>,
873 configured_port: Option<u16>,
874 ) -> Self {
875 self.rescan = Some(SupervisorRescanContext {
876 supervisor,
877 config_path: config_path.into(),
878 configured_port,
879 storage_config: self.storage_config.clone(),
880 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
881 admission_facts_targets: self.admission_facts_targets.clone(),
882 });
883 self
884 }
885
886 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
887 self.connected_clients = connected_clients;
888 self
889 }
890
891 pub fn forwarding(&self) -> Arc<ForwardingTable> {
892 Arc::clone(&self.forwarding)
893 }
894
895 pub(crate) fn counters(&self) -> DaemonCounters {
896 self.counters.clone()
897 }
898
899 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
902 tokio::spawn(async move {
903 loop {
904 self.capability_evaluator
905 .wait_for_change_or_deadline()
906 .await;
907 self.refresh_capability_requirements();
908 }
909 });
910 }
911
912 fn runtime_capability_snapshot(
913 &self,
914 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
915 let runtime = self
916 .supervisor
917 .list()
918 .into_iter()
919 .map(|module| {
920 let status = module.status().map_err(|err| {
921 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
922 })?;
923 Ok(RuntimeModule {
924 module_id: status.module_id,
925 state: status.state,
926 enabled: status.enabled,
927 })
928 })
929 .collect::<Result<Vec<_>, RouterError>>()?;
930 let (_, registrations) = self.registry.list_modules().map_err(|err| {
931 RouterError::backend(
932 0,
933 0,
934 format!("failed to list capability registrations: {err}"),
935 )
936 })?;
937 let registrations = registrations
938 .into_iter()
939 .map(|registration| RegisteredModule {
940 module_id: registration.manifest.module_id,
941 module_version: registration.manifest.module_version,
942 capabilities: registration.manifest.capabilities,
943 })
944 .collect();
945 Ok((runtime, registrations))
946 }
947
948 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
955 let cached_registration = RegisteredModule {
956 module_id: registration.manifest.module_id.clone(),
957 module_version: registration.manifest.module_version.clone(),
958 capabilities: registration.manifest.capabilities.clone(),
959 };
960 if self.capability_evaluator.record_hello(&cached_registration) {
961 warn!(
962 module_id = %cached_registration.module_id,
963 "capability claims drifted from the cached manifest"
964 );
965 }
966 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
967 self.enforce_capability_denies();
968 }
969 self.refresh_capability_requirements();
970 }
971
972 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
976 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
977 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
978 self.supervisor.set_swap_promotion_observer(observer);
979 }
980
981 pub fn refresh_capability_requirements(&self) {
982 match self.runtime_capability_snapshot() {
983 Ok((runtime, registrations)) => {
984 log_requirement_events(
985 self.capability_evaluator
986 .evaluate_now(&runtime, ®istrations),
987 );
988 }
989 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
990 }
991 }
992
993 fn enforce_capability_denies(&self) {
997 let (_, registrations) = match self.registry.list_modules() {
998 Ok(snapshot) => snapshot,
999 Err(err) => {
1000 warn!(error = %err, "failed to read registrations for capability deny census");
1001 return;
1002 }
1003 };
1004 let manifests = registrations
1005 .into_iter()
1006 .map(|registration| {
1007 (
1008 registration.manifest.module_id.clone(),
1009 registration.manifest,
1010 )
1011 })
1012 .collect::<BTreeMap<_, _>>();
1013 let census = match self.forwarding.route_census(None) {
1014 Ok(census) => census,
1015 Err(err) => {
1016 warn!(error = %err, "failed to read route census for capability deny enforcement");
1017 return;
1018 }
1019 };
1020
1021 for (target_module_id, routes) in census {
1022 let Some(target_manifest) = manifests.get(&target_module_id) else {
1023 continue;
1024 };
1025 let mut closed_routes = Vec::new();
1026 let mut module_goodbyes = Vec::new();
1027 for route in routes {
1028 let Principal::Reserved {
1029 module_id: opening_module_id,
1030 } = &route.principal
1031 else {
1032 continue;
1033 };
1034 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1035 continue;
1036 };
1037 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1038 continue;
1039 };
1040
1041 match self.forwarding.release_client_route(
1042 route.goodbye_target.connection_id,
1043 route.goodbye_target.channel,
1044 route.goodbye_target.epoch,
1045 ) {
1046 Ok(RouteRelease::Removed(module_goodbye)) => {
1047 warn!(
1048 opening_module_id,
1049 target_module_id,
1050 capability,
1051 "force-closing route because an attested capability deny edge now matches"
1052 );
1053 closed_routes.push(route);
1054 module_goodbyes.push(module_goodbye);
1055 }
1056 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1057 Err(err) => warn!(
1058 opening_module_id,
1059 target_module_id,
1060 capability,
1061 error = %err,
1062 "failed to force-close capability-denied route"
1063 ),
1064 }
1065 }
1066
1067 if closed_routes.is_empty() {
1068 continue;
1069 }
1070 send_route_control_pushes(
1071 &self.forwarding,
1072 closed_routes,
1073 ClientControlPush::RouteClosed {
1074 module_id: target_module_id,
1075 reason: RouteCloseReason::CapabilityDenied,
1076 drained: false,
1077 abandoned: 0,
1078 excluded_subscriptions: 0,
1079 terminal: Some(false),
1080 },
1081 );
1082 self.emit_route_goodbyes(module_goodbyes);
1083 }
1084 }
1085
1086 fn not_ready_reason(
1092 &self,
1093 registration: &crate::registry::ModuleRegistration,
1094 ) -> Option<NotReadyReason> {
1095 if !registration.ready {
1096 return Some(NotReadyReason {
1097 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1098 capability: None,
1099 });
1100 }
1101 self.first_unprovided_required_capability(registration)
1102 .map(|capability| NotReadyReason {
1103 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1104 capability: Some(capability),
1105 })
1106 }
1107
1108 fn first_unprovided_required_capability(
1126 &self,
1127 registration: &crate::registry::ModuleRegistration,
1128 ) -> Option<String> {
1129 let required = registration
1130 .manifest
1131 .capabilities
1132 .iter()
1133 .flat_map(|declarations| declarations.requires.iter())
1134 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1135 .map(|requirement| requirement.capability.as_str())
1136 .collect::<BTreeSet<_>>();
1137 if required.is_empty() {
1138 return None;
1139 }
1140 let module_id = registration.manifest.module_id.as_str();
1141 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1142 if required
1143 .iter()
1144 .any(|capability| verdict(capability).is_none())
1145 {
1146 self.refresh_capability_requirements();
1147 }
1148 required
1149 .into_iter()
1150 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1151 .map(str::to_string)
1152 }
1153
1154 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1155 self.capability_evaluator
1156 .statuses()
1157 .into_iter()
1158 .map(capability_requirement_status)
1159 .collect()
1160 }
1161
1162 fn deregister_connection(
1169 &self,
1170 connection_id: ConnectionId,
1171 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1172 self.registry.deregister_connection(connection_id)
1173 }
1174
1175 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1176 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1177 return None;
1178 }
1179 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1180 parse_client_control_request(&frame.body)
1181 else {
1182 return None;
1183 };
1184 Some(target_module_id(&target).to_string())
1185 }
1186
1187 pub(crate) fn route_open_capacity_refusal(
1188 &self,
1189 ctx: &RouteCtx,
1190 frame: &Frame,
1191 target_module_id: &str,
1192 limit: usize,
1193 ) -> Result<Frame, RouterError> {
1194 self.route_open_admission_refusal_frame(
1195 ctx,
1196 frame,
1197 target_module_id,
1198 format!(
1199 "connection already has {limit} route.open binds in flight; retry after one settles"
1200 ),
1201 )
1202 }
1203
1204 fn route_open_admission_refusal_frame(
1212 &self,
1213 ctx: &RouteCtx,
1214 frame: &Frame,
1215 target_module_id: &str,
1216 message: impl Into<String>,
1217 ) -> Result<Frame, RouterError> {
1218 self.route_open_refusal_frame(
1219 ctx,
1220 frame,
1221 target_module_id,
1222 "open_admission_full",
1223 error_codes::TARGET_UNAVAILABLE,
1224 message,
1225 )
1226 }
1227
1228 #[cfg(test)]
1234 pub fn handle_control(
1235 &self,
1236 connection_id: ConnectionId,
1237 frame: Frame,
1238 ) -> Result<Vec<Frame>, RouterError> {
1239 match frame.header.ty {
1240 FrameType::Ping => Ok(vec![pong(&frame)?]),
1241 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1242 FrameType::Goodbye => self.handle_goodbye(connection_id),
1243 ty => Ok(vec![control_error_frame(
1244 &frame,
1245 "unsupported_control_frame",
1246 format!("unsupported channel-0 frame {ty:?}"),
1247 )?]),
1248 }
1249 }
1250
1251 pub async fn handle_control_frame(
1252 &self,
1253 ctx: &RouteCtx,
1254 frame: Frame,
1255 ) -> Result<Vec<Frame>, RouterError> {
1256 self.handle_control_frame_timed(ctx, frame, None).await
1257 }
1258
1259 pub(crate) async fn handle_control_frame_timed(
1260 &self,
1261 ctx: &RouteCtx,
1262 frame: Frame,
1263 dispatch_started_at: Option<StdInstant>,
1264 ) -> Result<Vec<Frame>, RouterError> {
1265 match frame.header.ty {
1266 FrameType::Ping => Ok(vec![pong(&frame)?]),
1267 FrameType::Hello => {
1268 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1269 }
1270 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1271 FrameType::Cancel => {
1272 if self
1273 .supervisor
1274 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1275 {
1276 Ok(Vec::new())
1277 } else {
1278 Ok(vec![control_error_frame(
1279 &frame,
1280 "unknown_subscription",
1281 "no supervisor spawn subscription has this correlation id",
1282 )?])
1283 }
1284 }
1285 FrameType::Request => {
1286 if self
1287 .forwarding
1288 .module_endpoint_for_connection(ctx.connection_id)
1289 .map_err(RouterError::Forwarding)?
1290 .is_some()
1291 {
1292 if !is_known_module_request_op(&frame.body) {
1293 return Ok(vec![control_error_frame(
1294 &frame,
1295 "unsupported_control_frame",
1296 "module-originated channel-0 REQUEST is not supported",
1297 )?]);
1298 }
1299 let request = match parse_module_control_request_from_module(&frame.body) {
1300 Ok(request) => request,
1301 Err((err, ControlRequestBodyError::UnknownOp)) => {
1302 return Ok(vec![control_error_frame(
1303 &frame,
1304 "unsupported_control_frame",
1305 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1306 )?])
1307 }
1308 Err((err, ControlRequestBodyError::InvalidBody)) => {
1309 return Ok(vec![control_error_frame(
1310 &frame,
1311 "invalid_control_body",
1312 format!("malformed module control body: {err}"),
1313 )?])
1314 }
1315 };
1316 let op = module_control_request_op(&request);
1317 let corr = frame.header.corr;
1318 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1319 let result =
1320 self.handle_module_control_request(ctx.connection_id, frame, request);
1321 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1322 return result;
1323 }
1324
1325 if is_known_module_request_op(&frame.body) {
1326 return Ok(vec![control_error_frame(
1327 &frame,
1328 "not_registered",
1329 "catalog.update requires an active module registration owned by this connection",
1330 )?]);
1331 }
1332
1333 let request = match parse_client_control_request(&frame.body) {
1334 Ok(request) => request,
1335 Err((err, ControlRequestBodyError::UnknownOp)) => {
1336 return Ok(vec![control_error_frame(
1337 &frame,
1338 "unknown_control_op",
1339 format!("unknown client control op: {err}"),
1340 )?])
1341 }
1342 Err((err, ControlRequestBodyError::InvalidBody)) => {
1343 return Ok(vec![control_error_frame(
1344 &frame,
1345 "invalid_control_body",
1346 format!("malformed client control body: {err}"),
1347 )?])
1348 }
1349 };
1350 let op = client_control_request_op(&request);
1351 let corr = frame.header.corr;
1352 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1353 #[cfg(test)]
1354 if let Some(delay) = self.control_dispatch_delay {
1355 tokio::time::sleep(delay).await;
1356 }
1357 let result = self
1358 .handle_client_control_request(ctx, frame, request)
1359 .await;
1360 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1361 result
1362 }
1363 FrameType::Push => {
1364 let Some(endpoint) = self
1365 .forwarding
1366 .module_endpoint_for_connection(ctx.connection_id)
1367 .map_err(RouterError::Forwarding)?
1368 else {
1369 return Ok(vec![control_error_frame(
1370 &frame,
1371 "unsupported_control_frame",
1372 "client-originated channel-0 PUSH is not supported",
1373 )?]);
1374 };
1375 self.handle_status_update(endpoint, frame)
1376 }
1377 FrameType::Response | FrameType::Error
1378 if self
1379 .forwarding
1380 .module_endpoint_for_connection(ctx.connection_id)
1381 .map_err(RouterError::Forwarding)?
1382 .is_some() =>
1383 {
1384 self.handle_module_relay_response(ctx.connection_id, frame)
1385 }
1386 ty => Ok(vec![control_error_frame(
1387 &frame,
1388 "unsupported_control_frame",
1389 format!("unsupported channel-0 frame {ty:?}"),
1390 )?]),
1391 }
1392 }
1393
1394 pub fn cleanup_connection(
1395 &self,
1396 connection_id: ConnectionId,
1397 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1398 let crash_closed = self
1399 .registry
1400 .get_module_by_connection(connection_id)?
1401 .and_then(|registration| {
1402 self.forwarding
1403 .module_endpoint_for_connection(connection_id)
1404 .ok()
1405 .flatten()
1406 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1407 .map(|routes| (registration.manifest.module_id, routes))
1408 });
1409 let crash_closed = crash_closed.map(|(module_id, routes)| {
1410 let terminal = match self.supervisor.get(&module_id) {
1411 None => false,
1412 Some(module) => match module.will_recover_after_connection_loss() {
1413 Ok(will_recover) => !will_recover,
1414 Err(err) => {
1415 warn!(
1416 %module_id,
1417 error = %err,
1418 "failed to read crash recovery verdict; reporting non-terminal conservatively"
1419 );
1420 false
1421 }
1422 },
1423 };
1424 let reason = match self.forwarding.is_daemon_draining() {
1428 Ok(true) => RouteCloseReason::Restart,
1429 Ok(false) => RouteCloseReason::Crash,
1430 Err(err) => {
1431 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
1432 RouteCloseReason::Crash
1433 }
1434 };
1435 (module_id, routes, reason, terminal)
1436 });
1437 let registrations = self.deregister_connection(connection_id);
1438 let cleanup = self.forwarding.cleanup_connection_counted(connection_id);
1439 if let Some((module_id, routes, reason, terminal)) = crash_closed {
1444 let abandoned = cleanup
1445 .as_ref()
1446 .map_or(0, |cleanup| cleanup.abandoned_relays);
1447 send_route_control_pushes(
1448 &self.forwarding,
1449 routes,
1450 ClientControlPush::RouteClosed {
1451 module_id,
1452 reason,
1453 drained: false,
1454 abandoned,
1455 excluded_subscriptions: 0,
1456 terminal: Some(terminal),
1457 },
1458 );
1459 }
1460 if let Ok(cleanup) = cleanup {
1461 self.emit_route_goodbyes(cleanup.released);
1462 }
1463 if matches!(®istrations, Ok(r) if !r.is_empty()) {
1467 crate::supervise::notify_registration_release();
1468 self.capability_evaluator.wake_deadline_loop();
1469 self.refresh_capability_requirements();
1470 }
1471 self.supervisor.remove_spawn_subscribers(connection_id);
1472 registrations
1473 }
1474
1475 pub(crate) fn handle_route_goodbye(
1476 &self,
1477 connection_id: ConnectionId,
1478 route_channel: u16,
1479 route_epoch: u32,
1480 ) -> Result<bool, RouterError> {
1481 debug!(
1482 connection_id = connection_id.get(),
1483 route_channel, route_epoch, "handling route GOODBYE"
1484 );
1485 let RouteRelease::Removed(released_route) = self
1486 .forwarding
1487 .release_client_route(connection_id, route_channel, route_epoch)
1488 .map_err(RouterError::Forwarding)?
1489 else {
1490 return Ok(false);
1491 };
1492 self.emit_route_goodbyes(vec![released_route]);
1493 Ok(true)
1494 }
1495
1496 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
1497 for released in released_routes {
1498 let frame = match Frame::build_with_version(
1499 released.negotiated_ver,
1500 FrameType::Goodbye,
1501 control_flags(),
1502 released.channel,
1503 released.epoch,
1504 0,
1505 Vec::new(),
1506 ) {
1507 Ok(frame) => frame,
1508 Err(err) => {
1509 warn!(
1510 route_channel = released.channel,
1511 error = %err,
1512 "failed to build route GOODBYE frame"
1513 );
1514 continue;
1515 }
1516 };
1517 if !released.close_on_delivery_failure() {
1518 crate::forwarding::send_module_route_goodbye(
1519 &self.counters,
1520 &released.sink,
1521 frame,
1522 released.module_id.as_deref(),
1523 "client route released",
1524 );
1525 continue;
1526 }
1527 if let Err(err) = released.sink.try_send(frame) {
1528 warn!(
1529 target_connection_id = released.connection_id.get(),
1530 route_channel = released.channel,
1531 error = %err,
1532 "route GOODBYE was not delivered to client; closing target connection"
1533 );
1534 if self
1535 .forwarding
1536 .escalate_client_delivery_failure(
1537 released.connection_id,
1538 released.channel,
1539 released.epoch,
1540 CloseReason::new(
1541 "route_goodbye_delivery_failed",
1542 format!(
1543 "failed to enqueue route GOODBYE for channel {}: {err}",
1544 released.channel
1545 ),
1546 ),
1547 crate::forwarding::UndeliveredFrame {
1548 module_id: released.module_id.as_deref(),
1549 sink: &released.sink,
1550 },
1551 )
1552 .unwrap_or(false)
1553 {
1554 self.counters.increment_goodbye_relay_client_failed();
1555 }
1556 }
1557 }
1558 }
1559
1560 fn send_abandoned_route_bind_goodbye(
1572 &self,
1573 module_sink: &crate::FrameSink,
1574 negotiated_ver: u8,
1575 module_channel: u16,
1576 module_epoch: u32,
1577 ) {
1578 let frame = match Frame::build_with_version(
1579 negotiated_ver,
1580 FrameType::Goodbye,
1581 control_flags(),
1582 module_channel,
1583 module_epoch,
1584 0,
1585 Vec::new(),
1586 ) {
1587 Ok(frame) => frame,
1588 Err(err) => {
1589 warn!(
1590 route_channel = module_channel,
1591 error = %err,
1592 "failed to build GOODBYE for abandoned route.bind"
1593 );
1594 return;
1595 }
1596 };
1597 crate::forwarding::send_module_route_goodbye(
1598 &self.counters,
1599 module_sink,
1600 frame,
1601 None,
1602 "abandoned route.bind",
1603 );
1604 }
1605
1606 fn handle_hello(
1607 &self,
1608 connection_id: ConnectionId,
1609 sink: Option<crate::FrameSink>,
1610 frame: Frame,
1611 ) -> Result<Vec<Frame>, RouterError> {
1612 debug!(
1613 connection_id = connection_id.get(),
1614 corr = frame.header.corr,
1615 "handling HELLO"
1616 );
1617 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
1618 Ok(value) => value,
1619 Err(err) => {
1620 return Ok(vec![control_error_frame(
1621 &frame,
1622 "invalid_hello",
1623 format!("malformed HELLO body: {err}"),
1624 )?])
1625 }
1626 };
1627 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
1628 return Ok(vec![control_error_frame(
1629 &frame,
1630 "invalid_capability_grammar",
1631 err.to_string(),
1632 )?]);
1633 }
1634 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
1635 return Ok(vec![control_error_frame(
1636 &frame,
1637 "invalid_manifest",
1638 err.to_string(),
1639 )?]);
1640 }
1641 if let Some(provenance) = hello_value
1642 .get("manifest")
1643 .and_then(|manifest| manifest.get("provenance"))
1644 {
1645 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
1646 return Ok(vec![control_error_frame(
1647 &frame,
1648 "invalid_manifest",
1649 format!("malformed manifest provenance: {err}"),
1650 )?]);
1651 }
1652 }
1653 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
1654 Ok(hello) => hello,
1655 Err(err) => {
1656 return Ok(vec![control_error_frame(
1657 &frame,
1658 "invalid_hello",
1659 format!("malformed HELLO body: {err}"),
1660 )?])
1661 }
1662 };
1663
1664 if hello.protocol_ver != hello.manifest.protocol_ver {
1665 return Ok(vec![control_error_frame(
1666 &frame,
1667 "invalid_manifest",
1668 format!(
1669 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
1670 hello.protocol_ver, hello.manifest.protocol_ver
1671 ),
1672 )?]);
1673 }
1674
1675 if hello.manifest.module_id.trim().is_empty() {
1676 return Ok(vec![control_error_frame(
1677 &frame,
1678 "invalid_manifest",
1679 "manifest module_id must not be empty",
1680 )?]);
1681 }
1682
1683 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
1684 Ok(negotiated_ver) => negotiated_ver,
1685 Err(message) => {
1686 return Ok(vec![control_error_frame(
1687 &frame,
1688 "version_unsupported",
1689 message,
1690 )?])
1691 }
1692 };
1693
1694 let swap_admission = self
1702 .supervisor
1703 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
1704 if swap_admission == SwapHelloAdmission::Refused {
1705 warn!(
1706 module_id = %hello.manifest.module_id,
1707 connection_id = connection_id.get(),
1708 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
1709 );
1710 return Ok(vec![control_error_frame(
1711 &frame,
1712 "swap_token_invalid",
1713 format!(
1714 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
1715 hello.manifest.module_id
1716 ),
1717 )?]);
1718 }
1719 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
1720
1721 if let Some(rejection) = (!swap_candidate)
1729 .then(|| {
1730 self.supervisor.reserved_hello_rejection(
1731 &hello.manifest.module_id,
1732 hello.launch_nonce.as_deref(),
1733 )
1734 })
1735 .flatten()
1736 {
1737 let message = match rejection {
1738 ReservedHelloRejection::Exact { module_id } => format!(
1739 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
1740 ),
1741 ReservedHelloRejection::Prefix {
1742 prefix,
1743 owner_module_id,
1744 } => format!(
1745 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
1746 hello.manifest.module_id
1747 ),
1748 };
1749 return Ok(vec![control_error_frame(
1750 &frame,
1751 "reserved_module",
1752 message,
1753 )?]);
1754 }
1755
1756 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
1757 &hello.manifest.module_id,
1758 hello.manifest.capabilities.as_ref(),
1759 );
1760 if let Some(refusal) = reserved_capability_refusals.first() {
1761 let capability = refusal.capability.clone();
1762 let bound_module = refusal.claimants[0].clone();
1763 log_duplicate_claim_events(reserved_capability_refusals);
1764 return Ok(vec![control_error_frame(
1765 &frame,
1766 "reserved_capability",
1767 format!(
1768 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
1769 capability, bound_module, hello.manifest.module_id
1770 ),
1771 )?]);
1772 }
1773
1774 if self
1777 .forwarding
1778 .connection_has_client_routes(connection_id)
1779 .map_err(RouterError::Forwarding)?
1780 {
1781 return Ok(vec![control_error_frame(
1782 &frame,
1783 "invalid_hello",
1784 "connection has open client routes and cannot also register as a module",
1785 )?]);
1786 }
1787
1788 let control_ops = effective_module_control_ops(hello.control_ops);
1789 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
1792 if swap_candidate {
1793 return self.register_swap_candidate(
1794 connection_id,
1795 sink,
1796 &frame,
1797 hello.manifest,
1798 negotiated_ver,
1799 control_ops,
1800 hello_ack,
1801 );
1802 }
1803 let registration = match self.registry.register_with_control_ops(
1804 hello.manifest,
1805 negotiated_ver,
1806 connection_id,
1807 control_ops,
1808 ) {
1809 Ok(registration) => registration,
1810 Err(RegistryError::DuplicateModuleId { module_id }) => {
1811 return Ok(vec![control_error_frame(
1812 &frame,
1813 "duplicate_module_id",
1814 format!(
1815 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
1816 ),
1817 )?])
1818 }
1819 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
1820 return Ok(vec![control_error_frame(
1821 &frame,
1822 "invalid_module_id",
1823 err.to_string(),
1824 )?])
1825 }
1826 Err(err) => {
1827 return Ok(vec![control_error_frame(
1828 &frame,
1829 "registry_error",
1830 err.to_string(),
1831 )?])
1832 }
1833 };
1834
1835 let reply = if let Some(sink) = sink {
1836 let concurrency = manifest_concurrency(®istration.manifest);
1852 if let Err(err) = self.forwarding.register_module_connection_acked(
1853 connection_id,
1854 registration.manifest.module_id.clone(),
1855 negotiated_ver,
1856 concurrency,
1857 sink,
1858 hello_ack,
1859 ) {
1860 if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
1864 crate::supervise::notify_registration_release();
1865 }
1866 return Ok(vec![control_error_frame(
1867 &frame,
1868 forwarding_error_code(&err),
1869 err.to_string(),
1870 )?]);
1871 }
1872 Vec::new()
1873 } else {
1874 vec![hello_ack]
1877 };
1878
1879 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
1888 info!(
1889 module_id = %registration.manifest.module_id,
1890 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
1891 );
1892 }
1893
1894 self.apply_registration_capabilities(®istration);
1895
1896 info!(
1897 module_id = %registration.manifest.module_id,
1898 module_version = %registration.manifest.module_version,
1899 negotiated_ver,
1900 routable_provider = manifest_provides_routable_role(®istration.manifest),
1901 connection_id = connection_id.get(),
1902 "module registered"
1903 );
1904
1905 Ok(reply)
1906 }
1907
1908 #[allow(clippy::too_many_arguments)]
1918 fn register_swap_candidate(
1919 &self,
1920 connection_id: ConnectionId,
1921 sink: Option<crate::FrameSink>,
1922 frame: &Frame,
1923 manifest: ModuleManifest,
1924 negotiated_ver: u8,
1925 control_ops: Vec<String>,
1926 hello_ack: Frame,
1927 ) -> Result<Vec<Frame>, RouterError> {
1928 let module_id = manifest.module_id.clone();
1929 let registration = match self.registry.register_candidate_with_control_ops(
1930 manifest,
1931 negotiated_ver,
1932 connection_id,
1933 control_ops,
1934 ) {
1935 Ok(registration) => registration,
1936 Err(RegistryError::DuplicateModuleId { module_id }) => {
1937 return Ok(vec![control_error_frame(
1938 frame,
1939 "duplicate_module_id",
1940 format!(
1941 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
1942 ),
1943 )?])
1944 }
1945 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
1946 return Ok(vec![control_error_frame(
1947 frame,
1948 "invalid_module_id",
1949 err.to_string(),
1950 )?])
1951 }
1952 Err(err) => {
1953 return Ok(vec![control_error_frame(
1954 frame,
1955 "registry_error",
1956 err.to_string(),
1957 )?])
1958 }
1959 };
1960 let reply = if let Some(sink) = sink {
1961 let concurrency = manifest_concurrency(®istration.manifest);
1965 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
1966 connection_id,
1967 module_id.clone(),
1968 negotiated_ver,
1969 concurrency,
1970 sink,
1971 hello_ack,
1972 ) {
1973 if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
1974 crate::supervise::notify_registration_release();
1975 }
1976 return Ok(vec![control_error_frame(
1977 frame,
1978 forwarding_error_code(&err),
1979 err.to_string(),
1980 )?]);
1981 }
1982 Vec::new()
1983 } else {
1984 vec![hello_ack]
1985 };
1986 self.supervisor.mark_swap_candidate_admitted(&module_id);
1987 info!(
1988 module_id = %module_id,
1989 module_version = %registration.manifest.module_version,
1990 negotiated_ver,
1991 ready = registration.ready,
1992 connection_id = connection_id.get(),
1993 "swap candidate registered; not routable until cutover"
1994 );
1995 Ok(reply)
1996 }
1997
1998 fn build_hello_ack(
1999 &self,
2000 frame: &Frame,
2001 negotiated_ver: u8,
2002 module_id: &str,
2003 ) -> Result<Frame, RouterError> {
2004 let ack = ModuleHelloAckBody {
2005 negotiated_ver,
2006 subc_ops: module_subc_ops(),
2007 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2008 storage: self
2009 .storage_config
2010 .as_ref()
2011 .map(|cfg| cfg.descriptor_for(module_id)),
2012 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2013 };
2014 let body = serde_json::to_vec(&ack).map_err(|err| {
2015 RouterError::backend(
2016 0,
2017 frame.header.corr,
2018 format!("failed to encode HELLO_ACK: {err}"),
2019 )
2020 })?;
2021
2022 Frame::build_with_version(
2023 negotiated_ver,
2024 FrameType::HelloAck,
2025 control_flags(),
2026 0,
2027 0,
2028 frame.header.corr,
2029 body,
2030 )
2031 .map_err(RouterError::FrameBuild)
2032 }
2033
2034 async fn handle_client_control_request(
2035 &self,
2036 ctx: &RouteCtx,
2037 frame: Frame,
2038 request: ClientControlRequest,
2039 ) -> Result<Vec<Frame>, RouterError> {
2040 match request {
2041 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2042 ClientControlRequest::CatalogList { module_id } => {
2043 self.handle_catalog_list(frame, module_id)
2044 }
2045 ClientControlRequest::RouteOpen {
2046 target,
2047 identity,
2048 consumer_identity,
2049 consumer_capabilities,
2050 admission_facts,
2051 } => {
2052 self.handle_route_open(
2053 ctx,
2054 frame,
2055 RouteOpenRequest {
2056 target,
2057 identity,
2058 consumer_identity,
2059 consumer_capabilities,
2060 admission_facts,
2061 },
2062 )
2063 .await
2064 }
2065 ClientControlRequest::RoutePoll {
2066 route_channel,
2067 route_epoch,
2068 kind,
2069 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2070 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2071 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2072 self.handle_supervisor_spawn_snapshot(frame)
2073 }
2074 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2075 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2076 }
2077 ClientControlRequest::SupervisorRestart {
2078 module_id,
2079 drain_timeout_ms,
2080 } => {
2081 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms)
2082 .await
2083 }
2084 ClientControlRequest::SupervisorSwap {
2085 module_id,
2086 ready_timeout_ms,
2087 } => {
2088 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms)
2089 .await
2090 }
2091 ClientControlRequest::SupervisorReload { module_id } => {
2092 self.handle_supervisor_reload(frame, module_id).await
2093 }
2094 ClientControlRequest::SupervisorRescan { preview } => {
2095 self.handle_supervisor_rescan(frame, preview).await
2096 }
2097 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2098 self.handle_supervisor_release_reserved(frame, module_id)
2099 .await
2100 }
2101 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2102 self.handle_supervisor_set_enabled(frame, module_id, enabled)
2103 .await
2104 }
2105 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2106 self.handle_supervisor_health_probe(frame, module_id).await
2107 }
2108 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2109 ClientControlRequest::SupervisorRoutes { module_id } => {
2110 self.handle_supervisor_routes(frame, module_id)
2111 }
2112 ClientControlRequest::SupervisorProvenance { module_id } => {
2113 self.handle_supervisor_provenance(frame, module_id).await
2114 }
2115 ClientControlRequest::SupervisorStderrTail {
2116 module_id,
2117 max_lines,
2118 max_bytes,
2119 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2120 ClientControlRequest::SupervisorTerminals { module_id } => {
2121 self.handle_supervisor_terminals(frame, module_id).await
2122 }
2123 }
2124 }
2125
2126 fn handle_module_control_request(
2127 &self,
2128 connection_id: ConnectionId,
2129 frame: Frame,
2130 request: ModuleControlRequestFromModule,
2131 ) -> Result<Vec<Frame>, RouterError> {
2132 match request {
2133 ModuleControlRequestFromModule::CatalogUpdate {
2134 provides,
2135 capabilities,
2136 ready,
2137 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2138 ModuleControlRequestFromModule::LiveRoots {} => {
2139 let registered = self
2140 .registry
2141 .get_module_by_connection(connection_id)
2142 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2143 let Some(registration) = registered else {
2144 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2145 };
2146 let response = self
2147 .forwarding
2148 .live_roots(®istration.manifest.module_id)
2149 .map_err(RouterError::Forwarding)?;
2150 Ok(vec![control_response_body_frame(
2151 &frame,
2152 &response,
2153 "ModuleControlResponseToModule::LiveRoots",
2154 )?])
2155 }
2156 }
2157 }
2158
2159 fn handle_catalog_update(
2160 &self,
2161 connection_id: ConnectionId,
2162 frame: Frame,
2163 provides: Vec<ProviderRole>,
2164 capabilities: Option<CapabilityDeclarations>,
2165 ready: Option<bool>,
2166 ) -> Result<Vec<Frame>, RouterError> {
2167 self.refresh_capability_requirements();
2168 let Some(registration) = self
2169 .registry
2170 .get_module_by_connection(connection_id)
2171 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2172 else {
2173 return Ok(vec![control_error_frame(
2174 &frame,
2175 "not_registered",
2176 "catalog.update requires an active module registration owned by this connection",
2177 )?]);
2178 };
2179
2180 if let Some(message) =
2181 catalog_update_frozen_field_message(®istration.manifest, &provides)
2182 {
2183 return Ok(vec![control_error_frame(
2184 &frame,
2185 "catalog_update_frozen_field",
2186 message,
2187 )?]);
2188 }
2189
2190 let mut candidate = registration.manifest.clone();
2191 candidate.provides = provides.clone();
2192 candidate.capabilities = capabilities
2193 .clone()
2194 .or_else(|| registration.manifest.capabilities.clone());
2195 if let Err(err) = candidate.validate_capability_grammar() {
2196 return Ok(vec![control_error_frame(
2197 &frame,
2198 "invalid_capability_grammar",
2199 err.to_string(),
2200 )?]);
2201 }
2202
2203 let updated = self
2204 .registry
2205 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
2206 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2207 if updated.is_none() {
2208 return Ok(vec![control_error_frame(
2209 &frame,
2210 "not_registered",
2211 "catalog.update requires an active module registration owned by this connection",
2212 )?]);
2213 }
2214 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
2215 log_duplicate_claim_events(
2216 self.capability_evaluator
2217 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
2218 );
2219 }
2220 if capability_census_trigger(
2221 registration.manifest.capabilities.as_ref(),
2222 updated
2223 .as_ref()
2224 .and_then(|entry| entry.manifest.capabilities.as_ref()),
2225 ) {
2226 self.enforce_capability_denies();
2227 }
2228 self.refresh_capability_requirements();
2229
2230 let response = ModuleControlResponseToModule::CatalogUpdate {};
2231 control_response_body_frame(
2232 &frame,
2233 &response,
2234 "ModuleControlResponseToModule::CatalogUpdate",
2235 )
2236 .map(|frame| vec![frame])
2237 }
2238
2239 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
2240 self.refresh_capability_requirements();
2241 let mut counters = self.counters.snapshot();
2247 if let (Ok((connections_with_routes, max)), Some(obj)) = (
2248 self.forwarding.client_route_concentration(),
2249 counters.as_object_mut(),
2250 ) {
2251 obj.insert(
2252 "client_connections_with_routes".into(),
2253 connections_with_routes.into(),
2254 );
2255 obj.insert("max_routes_on_one_connection".into(), max.into());
2256 }
2257 if let (Some(open_breakers), Some(obj)) = (
2263 self.route_bind_breakers.open_snapshot(),
2264 counters.as_object_mut(),
2265 ) {
2266 obj.insert("route_bind_breakers_open".into(), open_breakers);
2267 }
2268 let response = ClientControlResponse::ServerDescribe {
2269 protocol_ver: PROTOCOL_VERSION,
2270 subc_ops: subc_ops(),
2271 capabilities: self.subc_capabilities.as_ref().to_vec(),
2272 connected_clients: self.connected_clients.count(),
2273 counters: Some(counters),
2274 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
2275 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
2276 capability_requirements: self.capability_requirement_statuses(),
2277 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2278 };
2279 Ok(vec![control_response_body_frame(
2280 &frame,
2281 &response,
2282 "ClientControlResponse::ServerDescribe",
2283 )?])
2284 }
2285
2286 fn handle_catalog_list(
2287 &self,
2288 frame: Frame,
2289 module_id: Option<String>,
2290 ) -> Result<Vec<Frame>, RouterError> {
2291 let (generation, modules) = self.registry.list_modules().map_err(|err| {
2292 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
2293 })?;
2294 let entries = modules
2295 .into_iter()
2296 .filter(|registration| {
2297 module_id
2298 .as_deref()
2299 .map(|wanted| registration.manifest.module_id == wanted)
2300 .unwrap_or(true)
2301 })
2302 .map(|registration| {
2303 let not_ready = self.not_ready_reason(®istration);
2304 let roles = registration.manifest.provides;
2305 CatalogEntry {
2306 module_id: registration.manifest.module_id,
2307 ready: not_ready.is_none(),
2308 not_ready,
2309 module_version: Some(registration.manifest.module_version),
2310 roles,
2311 control_ops: registration.control_ops,
2312 capabilities: registration.manifest.capabilities,
2313 self_signals: registration.manifest.self_signals,
2314 }
2315 })
2316 .collect();
2317 let response = ClientControlResponse::CatalogList {
2318 generation,
2319 modules: entries,
2320 subc_ops: subc_ops(),
2321 };
2322 Ok(vec![control_response_body_frame(
2323 &frame,
2324 &response,
2325 "ClientControlResponse::CatalogList",
2326 )?])
2327 }
2328
2329 fn route_open_principal(
2330 &self,
2331 frame: &Frame,
2332 consumer_identity: Option<ConsumerIdentity>,
2333 ) -> Result<Result<Principal, Frame>, RouterError> {
2334 let Some(consumer_identity) = consumer_identity else {
2335 return Ok(Ok(Principal::Direct));
2336 };
2337
2338 if self.supervisor.spawned_consumer_authorized(
2339 &consumer_identity.module_id,
2340 &consumer_identity.launch_nonce,
2341 ) {
2342 return Ok(Ok(Principal::Reserved {
2343 module_id: consumer_identity.module_id,
2344 }));
2345 }
2346
2347 Ok(Err(control_error_frame(
2348 frame,
2349 "bad_consumer_identity",
2350 format!(
2351 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
2352 consumer_identity.module_id
2353 ),
2354 )?))
2355 }
2356
2357 fn route_open_refusal_frame(
2363 &self,
2364 ctx: &RouteCtx,
2365 frame: &Frame,
2366 module_id: &str,
2367 reason: &'static str,
2368 code: &'static str,
2369 message: impl Into<String>,
2370 ) -> Result<Frame, RouterError> {
2371 self.observe_route_open_refusal(ctx, module_id, reason, code);
2372 control_error_frame(frame, code, message.into())
2373 }
2374
2375 fn route_open_breaker_refusal_frame(
2393 &self,
2394 ctx: &RouteCtx,
2395 frame: &Frame,
2396 module_id: &str,
2397 consecutive_timeouts: u32,
2398 retry_in: Duration,
2399 probe_in_flight: bool,
2400 ) -> Result<Frame, RouterError> {
2401 self.counters
2402 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
2403 debug!(
2404 target: "control",
2405 code = "module_timeout",
2406 module_id = ?module_id,
2407 connection_id = ctx.connection_id.get(),
2408 consecutive_timeouts,
2409 retry_in_ms = retry_in.as_millis() as u64,
2410 probe_in_flight,
2411 "route.open refused by open bind-relay breaker"
2412 );
2413 let detail = if probe_in_flight {
2414 "one probe bind is already in flight; retry once it settles".to_string()
2415 } else {
2416 format!("not relaying for another {retry_in:?}")
2417 };
2418 control_error_frame(
2419 frame,
2420 "module_timeout",
2421 format!(
2422 "module_id '{module_id}' failed {consecutive_timeouts} consecutive route.bind \
2423 relays; {detail}"
2424 ),
2425 )
2426 }
2427
2428 fn observe_route_open_refusal(
2441 &self,
2442 ctx: &RouteCtx,
2443 module_id: &str,
2444 reason: &'static str,
2445 code: &'static str,
2446 ) {
2447 self.counters.increment_route_open_refused(code);
2448 info!(
2449 target: "control",
2450 code,
2451 reason,
2452 module_id = ?module_id,
2453 connection_id = ctx.connection_id.get(),
2454 "route.open refused"
2455 );
2456 }
2457
2458 fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
2503 self.counters.increment_route_open_accepted(principal);
2504 info!(
2505 target: "control",
2506 principal,
2507 module_id,
2508 connection_id = ctx.connection_id.get(),
2509 "route.open accepted"
2510 );
2511 }
2512
2513 fn supervised_absent_route_open_refusal_frame(
2514 &self,
2515 ctx: &RouteCtx,
2516 frame: &Frame,
2517 module_id: &str,
2518 code: &'static str,
2519 status: &crate::supervise::ModuleStatus,
2520 ) -> Result<Frame, RouterError> {
2521 self.counters.increment_route_open_refused(code);
2522 info!(
2523 target: "control",
2524 code,
2525 reason = "supervised_not_registered",
2526 module_id = ?module_id,
2527 connection_id = ctx.connection_id.get(),
2528 state = %status.state,
2529 enabled = status.enabled,
2530 live = status.live,
2531 "route.open refused"
2532 );
2533 control_error_frame(
2534 frame,
2535 code,
2536 format!(
2537 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
2538 status.state, status.enabled, status.live
2539 ),
2540 )
2541 }
2542
2543 async fn handle_route_open(
2544 &self,
2545 ctx: &RouteCtx,
2546 frame: Frame,
2547 request: RouteOpenRequest,
2548 ) -> Result<Vec<Frame>, RouterError> {
2549 let RouteOpenRequest {
2550 target,
2551 mut identity,
2552 consumer_identity,
2553 consumer_capabilities,
2554 admission_facts,
2555 } = request;
2556 let target_module_id = target_module_id(&target).to_string();
2557 debug!(
2558 connection_id = ctx.connection_id.get(),
2559 corr = frame.header.corr,
2560 module_id = %target_module_id,
2561 "handling route.open"
2562 );
2563
2564 let Some(registration) = self
2583 .registry
2584 .get_module(&target_module_id)
2585 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2586 else {
2587 if let Some((status, warming)) =
2588 self.supervisor_status(&target_module_id, frame.header.corr)?
2589 {
2590 if status.protocol == ModuleProtocol::None {
2597 return Ok(vec![self.route_open_refusal_frame(
2598 ctx,
2599 &frame,
2600 &target_module_id,
2601 "protocol_none",
2602 error_codes::MODULE_NO_PROTOCOL,
2603 format!(
2604 "module_id '{target_module_id}' is declared protocol: none; \
2605 it speaks no subc wire and serves no routes"
2606 ),
2607 )?]);
2608 }
2609 let code = if warming {
2610 "module_warming"
2611 } else {
2612 "target_unavailable"
2613 };
2614 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
2615 ctx,
2616 &frame,
2617 &target_module_id,
2618 code,
2619 &status,
2620 )?]);
2621 }
2622 if let Some(removed_ago_ms) =
2623 self.supervisor.removal_tombstone_age_ms(&target_module_id)
2624 {
2625 return Ok(vec![self.route_open_refusal_frame(
2626 ctx,
2627 &frame,
2628 &target_module_id,
2629 "removed",
2630 error_codes::MODULE_REMOVED,
2631 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
2632 )?]);
2633 }
2634 return Ok(vec![self.route_open_refusal_frame(
2635 ctx,
2636 &frame,
2637 &target_module_id,
2638 "not_registered",
2639 error_codes::UNKNOWN_MODULE,
2640 format!("module_id '{target_module_id}' is not registered"),
2641 )?]);
2642 };
2643
2644 if !registration.ready {
2648 self.counters
2649 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
2650 info!(
2651 target: "control",
2652 code = error_codes::MODULE_WARMING,
2653 module_id = ?target_module_id,
2654 connection_id = ctx.connection_id.get(),
2655 reason = "declared_not_ready",
2656 "route.open refused"
2657 );
2658 return Ok(vec![control_error_body_frame(
2659 &frame,
2660 ErrorBody {
2661 code: error_codes::MODULE_WARMING.to_string(),
2662 message: format!(
2663 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
2664 ),
2665 detail: Some(serde_json::json!({
2666 "reason": "declared_not_ready"
2667 })),
2668 },
2669 )?]);
2670 }
2671
2672 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
2691 self.counters
2692 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
2693 info!(
2694 target: "control",
2695 code = error_codes::MODULE_WARMING,
2696 module_id = ?target_module_id,
2697 connection_id = ctx.connection_id.get(),
2698 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
2699 capability = %capability,
2700 "route.open refused"
2701 );
2702 return Ok(vec![control_error_body_frame(
2703 &frame,
2704 ErrorBody {
2705 code: error_codes::MODULE_WARMING.to_string(),
2706 message: format!(
2707 "module_id '{target_module_id}' requires capability '{capability}', \
2708 which no registered module provides; retry"
2709 ),
2710 detail: Some(serde_json::json!({
2711 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
2712 "capability": capability,
2713 })),
2714 },
2715 )?]);
2716 }
2717
2718 if !target_has_required_role(&target, ®istration.manifest.provides) {
2719 return Ok(vec![self.route_open_refusal_frame(
2720 ctx,
2721 &frame,
2722 &target_module_id,
2723 "role_not_provided",
2724 "target_unavailable",
2725 format!("module_id '{target_module_id}' does not provide the requested target"),
2726 )?]);
2727 }
2728
2729 if registration.state != ChannelState::Active {
2730 return Ok(vec![self.route_open_refusal_frame(
2731 ctx,
2732 &frame,
2733 &target_module_id,
2734 "registration_not_active",
2735 "target_unavailable",
2736 format!("module_id '{target_module_id}' is not active"),
2737 )?]);
2738 }
2739
2740 if self
2741 .forwarding
2742 .module_is_draining(&target_module_id)
2743 .map_err(RouterError::Forwarding)?
2744 {
2745 return Ok(vec![self.route_open_refusal_frame(
2746 ctx,
2747 &frame,
2748 &target_module_id,
2749 "reloading",
2750 "module_reloading",
2751 format!("module_id '{target_module_id}' is reloading"),
2752 )?]);
2753 }
2754
2755 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
2756 process_liveness.process_live(&target_module_id) == Some(false)
2757 }) {
2758 if process_liveness.process_replacing(&target_module_id) {
2765 return Ok(vec![self.route_open_refusal_frame(
2766 ctx,
2767 &frame,
2768 &target_module_id,
2769 "reloading",
2770 "module_reloading",
2771 format!("module_id '{target_module_id}' is reloading"),
2772 )?]);
2773 }
2774 return Ok(vec![self.route_open_refusal_frame(
2775 ctx,
2776 &frame,
2777 &target_module_id,
2778 "supervisor_not_live",
2779 "target_unavailable",
2780 format!("module_id '{target_module_id}' is not live"),
2781 )?]);
2782 }
2783
2784 if !self
2785 .forwarding
2786 .has_live_module_connection(&target_module_id)
2787 .map_err(RouterError::Forwarding)?
2788 {
2789 return Ok(vec![self.route_open_refusal_frame(
2790 ctx,
2791 &frame,
2792 &target_module_id,
2793 "no_forwarding_connection",
2794 "target_unavailable",
2795 format!("module_id '{target_module_id}' has no live forwarding connection"),
2796 )?]);
2797 }
2798
2799 if let Some(error) =
2800 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
2801 {
2802 self.observe_route_open_refusal(
2803 ctx,
2804 &target_module_id,
2805 "op_not_allowed",
2806 "op_not_allowed",
2807 );
2808 return Ok(vec![error]);
2809 }
2810
2811 let principal = match self.route_open_principal(&frame, consumer_identity)? {
2812 Ok(principal) => principal,
2813 Err(error) => {
2814 self.observe_route_open_refusal(
2815 ctx,
2816 &target_module_id,
2817 "bad_consumer_identity",
2818 "bad_consumer_identity",
2819 );
2820 return Ok(vec![error]);
2821 }
2822 };
2823
2824 if let Principal::Reserved {
2828 module_id: opening_module_id,
2829 } = &principal
2830 {
2831 if let Some(opening_registration) = self
2832 .registry
2833 .get_module(opening_module_id)
2834 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2835 {
2836 if let Some(capability) =
2837 denied_capability(&opening_registration.manifest, ®istration.manifest)
2838 {
2839 warn!(
2840 opening_module_id,
2841 target_module_id,
2842 capability,
2843 "refusing route.open because an attested capability deny edge matches"
2844 );
2845 return Ok(vec![self.route_open_refusal_frame(
2846 ctx,
2847 &frame,
2848 &target_module_id,
2849 "capability_deny_edge",
2850 "capability_forbidden",
2851 format!(
2852 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
2853 ),
2854 )?]);
2855 }
2856 }
2857 }
2858
2859 if admission_facts.is_some() {
2860 let carrier_matches = matches!(
2861 &principal,
2862 Principal::Reserved { module_id }
2863 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
2864 );
2865 if !carrier_matches {
2866 return Ok(vec![self.route_open_refusal_frame(
2867 ctx,
2868 &frame,
2869 &target_module_id,
2870 "admission_facts_carrier_not_permitted",
2871 "admission_facts_not_permitted",
2872 "admission facts may only be carried by the configured reserved module",
2873 )?]);
2874 }
2875
2876 let target_allowed = self
2877 .admission_facts_targets
2878 .as_ref()
2879 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
2880 if !target_allowed {
2881 return Ok(vec![self.route_open_refusal_frame(
2882 ctx,
2883 &frame,
2884 &target_module_id,
2885 "admission_facts_target_not_listed",
2886 "admission_facts_target_not_allowed",
2887 format!(
2888 "admission facts are not permitted for target module_id '{target_module_id}'"
2889 ),
2890 )?]);
2891 }
2892
2893 }
2897
2898 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
2920 Ok(project_root) => project_root,
2921 Err(err) => {
2922 return Ok(vec![control_error_frame(
2923 &frame,
2924 "invalid_project_root",
2925 err.to_string(),
2926 )?])
2927 }
2928 };
2929 identity.project_root = project_root.as_path().to_path_buf();
2930
2931 let _concurrency_guard = match self
2942 .route_bind_concurrency
2943 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
2944 {
2945 Ok(guard) => guard,
2946 Err(in_flight) => {
2947 return Ok(vec![self.route_open_admission_refusal_frame(
2948 ctx,
2949 &frame,
2950 &target_module_id,
2951 format!(
2952 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
2953 ),
2954 )?]);
2955 }
2956 };
2957
2958 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
2961 RouteBindAdmission::Admitted { guard, probe } => {
2962 if probe {
2963 info!(
2964 module_id = %target_module_id,
2965 connection_id = ctx.connection_id.get(),
2966 "route.bind breaker half-open: admitting one probe"
2967 );
2968 }
2969 guard
2970 }
2971 RouteBindAdmission::Refused {
2972 consecutive_timeouts,
2973 retry_in,
2974 probe_in_flight,
2975 } => {
2976 return Ok(vec![self.route_open_breaker_refusal_frame(
2977 ctx,
2978 &frame,
2979 &target_module_id,
2980 consecutive_timeouts,
2981 retry_in,
2982 probe_in_flight,
2983 )?]);
2984 }
2985 };
2986
2987 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
2992 let relay_deadline = Instant::now() + route_bind_relay_timeout;
2993 let pending = match self
2994 .forwarding
2995 .begin_route_bind_relay_for(
2996 ctx.connection_id,
2997 ctx.egress.clone(),
2998 response_version(&frame),
2999 frame.header.corr,
3000 &target_module_id,
3001 principal.clone(),
3002 Some(project_root),
3003 relay_deadline,
3004 )
3005 .await
3006 {
3007 Ok(pending) => pending,
3008 Err(err) => {
3009 return Ok(vec![self.route_open_refusal_frame(
3010 ctx,
3011 &frame,
3012 &target_module_id,
3013 "relay_reservation_failed",
3014 forwarding_error_code(&err),
3015 err.to_string(),
3016 )?])
3017 }
3018 };
3019 let crate::forwarding::PendingRouteBindRelay {
3020 endpoint,
3021 module_sink,
3022 negotiated_ver,
3023 client_channel,
3024 client_epoch,
3025 module_channel,
3026 module_epoch,
3027 corr: relay_corr,
3028 receiver,
3029 } = pending;
3030 let mut reservation =
3031 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
3032
3033 debug!(
3034 connection_id = ctx.connection_id.get(),
3035 client_channel,
3036 client_epoch,
3037 module_channel,
3038 module_epoch,
3039 "reserved route handle pair"
3040 );
3041 let principal_label = match &principal {
3044 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
3045 Principal::Direct => "direct".to_string(),
3046 other => format!("{other:?}"),
3047 };
3048 let relay = ModuleControlRequest::RouteBind {
3049 route_channel: module_channel,
3050 epoch: module_epoch,
3051 target,
3052 identity,
3053 principal: Some(principal),
3054 consumer_capabilities,
3055 admission_facts,
3056 };
3057 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
3058 RouterError::backend(
3059 0,
3060 frame.header.corr,
3061 format!("failed to encode route.bind request: {err}"),
3062 )
3063 })?;
3064 let relay_frame = Frame::build_with_version(
3065 negotiated_ver,
3066 FrameType::Request,
3067 control_flags(),
3068 0,
3069 0,
3070 relay_corr,
3071 relay_body,
3072 )
3073 .map_err(RouterError::FrameBuild)?;
3074
3075 if let Err(err) = module_sink.send(relay_frame).await {
3076 reservation.release_and_disarm();
3077 return Ok(vec![self.route_open_refusal_frame(
3078 ctx,
3079 &frame,
3080 &target_module_id,
3081 "relay_send_failed",
3082 "target_unavailable",
3083 err.to_string(),
3084 )?]);
3085 }
3086
3087 if !self
3088 .forwarding
3089 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
3090 .map_err(RouterError::Forwarding)?
3091 {
3092 self.send_abandoned_route_bind_goodbye(
3093 &module_sink,
3094 negotiated_ver,
3095 module_channel,
3096 module_epoch,
3097 );
3098 }
3099
3100 match timeout_at(relay_deadline, receiver).await {
3101 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
3102 reservation.disarm();
3103 if breaker.record_accepted() {
3104 info!(
3105 module_id = %target_module_id,
3106 "route.bind breaker closed: the probe was accepted"
3107 );
3108 }
3109 self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
3110 Ok(Vec::new())
3111 }
3112 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
3113 reservation.release_and_disarm();
3114 breaker.record_inconclusive();
3118 self.counters
3119 .increment_route_open_refused("module_rejected");
3120 info!(
3121 target: "control",
3122 code = "module_rejected",
3123 module_code = ?body.code,
3124 module_id = ?target_module_id,
3125 connection_id = ctx.connection_id.get(),
3126 "route.open refused"
3127 );
3128 Ok(vec![control_error_body_frame(&frame, body)?])
3129 }
3130 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
3131 reservation.release_and_disarm();
3132 breaker.record_inconclusive();
3133 tracing::warn!(
3140 module_id = %target_module_id,
3141 "route.bind relay abandoned: {message}"
3142 );
3143 Ok(vec![self.route_open_refusal_frame(
3144 ctx,
3145 &frame,
3146 &target_module_id,
3147 "relay_abandoned",
3148 "target_unavailable",
3149 message,
3150 )?])
3151 }
3152 Ok(Err(_)) => {
3153 reservation.release_and_disarm();
3154 breaker.record_inconclusive();
3155 Ok(vec![self.route_open_refusal_frame(
3156 ctx,
3157 &frame,
3158 &target_module_id,
3159 "relay_waiter_canceled",
3160 "target_unavailable",
3161 "route.bind relay waiter was canceled before the module responded",
3162 )?])
3163 }
3164 Err(_) => {
3165 reservation.release_and_disarm();
3166 if let Some(opened) = breaker.record_timeout(
3170 self.route_bind_breaker_threshold,
3171 self.route_bind_breaker_cooldown,
3172 ) {
3173 warn!(
3174 module_id = %target_module_id,
3175 consecutive_timeouts = opened.consecutive_timeouts,
3176 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
3177 reopened_after_probe = opened.reopened_after_probe,
3178 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
3179 );
3180 }
3181 tracing::warn!(
3189 module_id = %target_module_id,
3190 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
3191 "route.bind relay timed out: module did not ack within budget"
3192 );
3193 Ok(vec![self.route_open_refusal_frame(
3194 ctx,
3195 &frame,
3196 &target_module_id,
3197 "relay_timed_out",
3198 "module_timeout",
3199 format!(
3200 "module_id '{target_module_id}' did not answer route.bind within {:?}",
3201 route_bind_relay_timeout
3202 ),
3203 )?])
3204 }
3205 }
3206 }
3207
3208 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3209 let response = ClientControlResponse::SupervisorSpawnSnapshot {
3210 snapshot: self.supervisor.spawn_snapshot(),
3211 };
3212 Ok(vec![control_response_body_frame(
3213 &frame,
3214 &response,
3215 "ClientControlResponse::SupervisorSpawnSnapshot",
3216 )?])
3217 }
3218
3219 fn handle_supervisor_spawn_subscribe(
3220 &self,
3221 ctx: &RouteCtx,
3222 frame: Frame,
3223 since: Option<SpawnCursor>,
3224 ) -> Result<Vec<Frame>, RouterError> {
3225 match self.supervisor.subscribe_spawns(
3226 ctx.connection_id,
3227 frame.header.corr,
3228 response_version(&frame),
3229 since,
3230 ctx.egress.clone(),
3231 ) {
3232 Ok(()) => Ok(Vec::new()),
3233 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
3234 Ok(vec![control_error_body_frame(
3235 &frame,
3236 ErrorBody {
3237 code: "spawn_cursor_incarnation_mismatch".to_string(),
3238 message: "spawn cursor belongs to a different daemon incarnation"
3239 .to_string(),
3240 detail: Some(serde_json::json!({
3241 "current_daemon_incarnation": current
3242 })),
3243 },
3244 )?])
3245 }
3246 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
3247 &frame,
3248 ErrorBody {
3249 code: "spawn_cursor_too_old".to_string(),
3250 message: "spawn cursor predates the retained event ring".to_string(),
3251 detail: Some(serde_json::json!({
3252 "oldest_retained_cursor": oldest
3253 })),
3254 },
3255 )?]),
3256 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
3257 0,
3258 frame.header.corr,
3259 format!("failed to open supervisor spawn subscription: {error}"),
3260 )),
3261 }
3262 }
3263
3264 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3265 let generation = self
3266 .registry
3267 .generation()
3268 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3269 let mut modules = Vec::new();
3270 for module in self.supervisor.list() {
3271 let status = module.status_for_control("list").map_err(|err| {
3272 RouterError::backend(
3273 0,
3274 frame.header.corr,
3275 format!("failed to read supervisor status: {err}"),
3276 )
3277 })?;
3278 let (configured, _) = module.configuration().map_err(|err| {
3279 RouterError::backend(
3280 0,
3281 frame.header.corr,
3282 format!("failed to read module configuration: {err}"),
3283 )
3284 })?;
3285 let image = module.running_image_agreement().await;
3287 let pending_reload = Some(reload_verdict(
3288 &configured.program,
3289 status.spawned_from.as_deref(),
3290 image,
3291 ));
3292 modules.push(SupervisorEntry {
3293 module_id: status.module_id,
3294 state: status.state.to_string(),
3295 enabled: status.enabled,
3296 live: status.live,
3297 protocol: status.protocol,
3298 health: status.health.status,
3299 pending_reload,
3300 last_probe_ms: status.health.last_probe_ms,
3301 last_exit_code: status.last_exit.as_ref().and_then(|e| e.code),
3302 last_exit_signal: status.last_exit.as_ref().and_then(|e| e.signal),
3303 last_exit_ms: status.last_exit.as_ref().map(|e| e.at_ms),
3304 last_exit_kind: status.last_exit.as_ref().map(|e| e.kind.into()),
3305 restart_count: Some(status.restart_count),
3306 max_restarts: Some(status.max_restarts),
3307 lifetime_restarts: Some(status.lifetime_restarts),
3308 spawn_generation: Some(status.spawn_generation),
3309 restart_window_secs: Some(status.restart_window.as_secs()),
3310 drain_timeout_ms: Some(status.drain_timeout.as_millis() as u64),
3311 restart_backoff_ms: Some(status.restart_backoff.as_millis() as u64),
3312 restart_max_backoff_ms: Some(status.restart_max_backoff.as_millis() as u64),
3313 });
3314 }
3315 let response = ClientControlResponse::SupervisorList {
3316 generation,
3317 modules,
3318 };
3319 Ok(vec![control_response_body_frame(
3320 &frame,
3321 &response,
3322 "ClientControlResponse::SupervisorList",
3323 )?])
3324 }
3325
3326 fn handle_supervisor_stderr_tail(
3327 &self,
3328 frame: Frame,
3329 module_id: String,
3330 max_lines: Option<u32>,
3331 max_bytes: Option<u32>,
3332 ) -> Result<Vec<Frame>, RouterError> {
3333 let Some(module) = self.supervisor.get(&module_id) else {
3334 return Ok(vec![control_error_frame(
3335 &frame,
3336 "unknown_module",
3337 format!("module_id '{module_id}' is not supervised"),
3338 )?]);
3339 };
3340
3341 let snapshot = module.stderr_tail(
3342 max_lines.map(|value| value as usize),
3343 max_bytes.map(|value| value as usize),
3344 );
3345
3346 let response = ClientControlResponse::SupervisorStderrTail {
3347 module_id,
3348 tail: StderrTail {
3349 capture: match snapshot.capture {
3350 CaptureState::Captured => StderrCaptureState::Captured,
3351 CaptureState::Incomplete { reason } => {
3352 StderrCaptureState::Incomplete { reason }
3353 }
3354 CaptureState::NotCaptured { reason } => {
3355 StderrCaptureState::NotCaptured { reason }
3356 }
3357 },
3358 entries: snapshot
3359 .entries
3360 .into_iter()
3361 .map(|entry| match entry {
3362 TailEntry::Line { text, truncated } => {
3363 StderrTailEntry::Line { text, truncated }
3364 }
3365 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
3366 })
3367 .collect(),
3368 dropped_lines: snapshot.dropped_lines,
3369 },
3370 };
3371 Ok(vec![control_response_body_frame(
3372 &frame,
3373 &response,
3374 "ClientControlResponse::SupervisorStderrTail",
3375 )?])
3376 }
3377
3378 async fn handle_supervisor_terminals(
3379 &self,
3380 frame: Frame,
3381 module_id: String,
3382 ) -> Result<Vec<Frame>, RouterError> {
3383 let Some(module) = self.supervisor.get(&module_id) else {
3384 return Ok(vec![control_error_frame(
3385 &frame,
3386 "unknown_module",
3387 format!("module_id '{module_id}' is not supervised"),
3388 )?]);
3389 };
3390
3391 let terminals = module
3394 .read_durable_terminal_history()
3395 .await
3396 .map_err(|error| {
3397 RouterError::backend(
3398 0,
3399 frame.header.corr,
3400 format!("failed to read terminal history: {error}"),
3401 )
3402 })?;
3403 let response = ClientControlResponse::SupervisorTerminals {
3404 module_id,
3405 terminals,
3406 };
3407 Ok(vec![control_response_body_frame(
3408 &frame,
3409 &response,
3410 "ClientControlResponse::SupervisorTerminals",
3411 )?])
3412 }
3413
3414 fn handle_supervisor_routes(
3415 &self,
3416 frame: Frame,
3417 module_id: Option<String>,
3418 ) -> Result<Vec<Frame>, RouterError> {
3419 let modules = self
3420 .forwarding
3421 .route_census(module_id.as_deref())
3422 .map_err(RouterError::Forwarding)?
3423 .into_iter()
3424 .map(|(module_id, routes)| SupervisorRouteModule {
3425 module_id,
3426 routes: routes
3427 .into_iter()
3428 .map(|route| SupervisorRoute {
3429 consumer: match route.principal {
3430 Principal::Reserved { module_id } => {
3431 SupervisorRouteConsumer::Reserved { module_id }
3432 }
3433 Principal::Direct | Principal::Unverified => {
3434 SupervisorRouteConsumer::Direct {
3435 connection_id: route.goodbye_target.connection_id.get(),
3436 }
3437 }
3438 },
3439 age_ms: Instant::now()
3440 .saturating_duration_since(route.bound_at)
3441 .as_millis()
3442 .try_into()
3443 .unwrap_or(u64::MAX),
3444 draining: route.draining,
3445 drain_reason: route.drain_reason,
3446 })
3447 .collect(),
3448 })
3449 .collect();
3450 let response = ClientControlResponse::SupervisorRoutes { modules };
3451 Ok(vec![control_response_body_frame(
3452 &frame,
3453 &response,
3454 "ClientControlResponse::SupervisorRoutes",
3455 )?])
3456 }
3457
3458 async fn handle_supervisor_provenance(
3459 &self,
3460 frame: Frame,
3461 module_id: Option<String>,
3462 ) -> Result<Vec<Frame>, RouterError> {
3463 let mut selected = if let Some(module_id) = module_id {
3464 let Some(module) = self.supervisor.get(&module_id) else {
3465 return Ok(vec![control_error_frame(
3466 &frame,
3467 "unknown_module",
3468 format!("module_id '{module_id}' is not supervised"),
3469 )?]);
3470 };
3471 vec![module]
3472 } else {
3473 self.supervisor.list()
3474 };
3475
3476 let mut modules = Vec::with_capacity(selected.len());
3477 for module in selected.drain(..) {
3478 let status = module.status().map_err(|err| {
3479 RouterError::backend(
3480 0,
3481 frame.header.corr,
3482 format!("failed to read supervisor status: {err}"),
3483 )
3484 })?;
3485 let module_declared = self
3486 .registry
3487 .get_module(&status.module_id)
3488 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3489 .and_then(|registration| registration.manifest.provenance)
3490 .map(|build| ModuleDeclaredProvenance::Reported { build })
3491 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
3492 #[cfg(test)]
3493 let running_image = match &self.provenance_probe_override {
3494 Some(result) => result.clone(),
3495 None => module.running_image_agreement().await,
3496 };
3497 #[cfg(not(test))]
3498 let running_image = module.running_image_agreement().await;
3499 modules.push(SupervisorModuleProvenance {
3500 module_id: status.module_id,
3501 module_declared,
3502 daemon_observed: SupervisorObservedProcess {
3503 pid: status.pid,
3504 spawned_at_ms: status.spawned_at_ms,
3505 spawned_from: status.spawned_from,
3506 running_image,
3507 },
3508 });
3509 }
3510 let daemon = SupervisorDaemonProvenance {
3511 daemon_build: self.daemon_provenance.build.clone(),
3512 daemon_observed: DaemonObservedProcess {
3513 pid: self.daemon_provenance.pid,
3514 started_at_ms: self
3515 .daemon_provenance
3516 .start_clock
3517 .map(|clock| clock.started_at_ms())
3518 .or(self.daemon_provenance.started_at_ms),
3519 running_image: self
3520 .daemon_provenance
3521 .probe
3522 .observe(
3523 self.daemon_provenance.pid,
3524 self.daemon_provenance.executable_path.as_deref(),
3525 self.daemon_provenance.executable_identity,
3526 self.daemon_provenance.process_start_time,
3527 )
3528 .await,
3529 },
3530 };
3531 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
3532 Ok(vec![control_response_body_frame(
3533 &frame,
3534 &response,
3535 "ClientControlResponse::SupervisorProvenance",
3536 )?])
3537 }
3538
3539 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3540 self.refresh_capability_requirements();
3541 let generation = self
3542 .registry
3543 .generation()
3544 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3545 let modules = self
3546 .supervisor
3547 .list()
3548 .into_iter()
3549 .map(|module| {
3550 let status = module.status_for_control("health").map_err(|err| {
3551 RouterError::backend(
3552 0,
3553 frame.header.corr,
3554 format!("failed to read supervisor health: {err}"),
3555 )
3556 })?;
3557 let module_id = status.module_id;
3558 let capability_detail = self
3559 .capability_evaluator
3560 .required_problem_detail(&module_id);
3561 Ok(SupervisorHealthEntry {
3562 module_id,
3563 status: status.health.status,
3564 detail: append_capability_problem_detail(
3565 status.health.detail,
3566 capability_detail,
3567 ),
3568 metrics: status.health.metrics,
3569 consecutive_failures: status.health.consecutive_failures,
3570 late_answer_count: status.health.late_answer_count,
3571 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
3572 last_action: status.health.last_action,
3573 last_action_ms: status.health.last_action_ms,
3574 last_probe_ms: status.health.last_probe_ms,
3575 })
3576 })
3577 .collect::<Result<Vec<_>, RouterError>>()?;
3578 let response = ClientControlResponse::SupervisorHealth {
3579 generation,
3580 modules,
3581 };
3582 Ok(vec![control_response_body_frame(
3583 &frame,
3584 &response,
3585 "ClientControlResponse::SupervisorHealth",
3586 )?])
3587 }
3588
3589 async fn handle_supervisor_restart(
3590 &self,
3591 frame: Frame,
3592 module_id: String,
3593 drain_timeout_ms: Option<u64>,
3594 ) -> Result<Vec<Frame>, RouterError> {
3595 let operation_lock = self.supervisor.operation_lock();
3596 let _operation_guard = operation_lock.lock().await;
3597 let Some(module) = self.supervisor.get(&module_id) else {
3598 return Ok(vec![control_error_frame(
3599 &frame,
3600 "unknown_module",
3601 format!("module_id '{module_id}' is not supervised"),
3602 )?]);
3603 };
3604
3605 if let Err(err) = module.restart(drain_timeout_ms).await {
3606 let (code, message) = match err {
3607 crate::supervise::SuperviseError::Disabled { .. } => {
3608 ("module_disabled", err.to_string())
3609 }
3610 crate::supervise::SuperviseError::SwapInProgress { .. } => {
3611 ("swap_in_progress", err.to_string())
3612 }
3613 _ => (
3614 "target_unavailable",
3615 format!("failed to restart module_id '{module_id}': {err}"),
3616 ),
3617 };
3618 return Ok(vec![control_error_frame(&frame, code, message)?]);
3619 }
3620
3621 let response = ClientControlResponse::SupervisorAck {
3622 module_id,
3623 applied: true,
3624 };
3625 Ok(vec![control_response_body_frame(
3626 &frame,
3627 &response,
3628 "ClientControlResponse::SupervisorAck",
3629 )?])
3630 }
3631
3632 async fn handle_supervisor_swap(
3636 &self,
3637 frame: Frame,
3638 module_id: String,
3639 ready_timeout_ms: Option<u64>,
3640 ) -> Result<Vec<Frame>, RouterError> {
3641 let module = {
3648 let operation_lock = self.supervisor.operation_lock();
3649 let _operation_guard = operation_lock.lock().await;
3650 self.supervisor.get(&module_id)
3651 };
3652 let Some(module) = module else {
3653 return Ok(vec![control_error_frame(
3654 &frame,
3655 "unknown_module",
3656 format!("module_id '{module_id}' is not supervised"),
3657 )?]);
3658 };
3659
3660 if let Err(err) = module
3661 .swap(ready_timeout_ms.map(Duration::from_millis))
3662 .await
3663 {
3664 use crate::supervise::SuperviseError;
3665 let message = err.to_string();
3666 let error = match err {
3667 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
3668 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
3669 code: "swap_refused".to_string(),
3670 message,
3671 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
3672 },
3673 SuperviseError::SwapFailed {
3674 arm,
3675 candidate_exit,
3676 ..
3677 } => ErrorBody {
3678 code: "swap_failed".to_string(),
3679 message,
3680 detail: Some(serde_json::json!({
3681 "arm": arm.as_str(),
3682 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
3683 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
3684 })),
3685 },
3686 _ => ErrorBody::new(
3687 "target_unavailable",
3688 format!("failed to swap module_id '{module_id}': {message}"),
3689 ),
3690 };
3691 return Ok(vec![control_error_body_frame(&frame, error)?]);
3692 }
3693
3694 let response = ClientControlResponse::SupervisorAck {
3695 module_id,
3696 applied: true,
3697 };
3698 Ok(vec![control_response_body_frame(
3699 &frame,
3700 &response,
3701 "ClientControlResponse::SupervisorAck",
3702 )?])
3703 }
3704
3705 async fn handle_supervisor_reload(
3706 &self,
3707 frame: Frame,
3708 module_id: String,
3709 ) -> Result<Vec<Frame>, RouterError> {
3710 let operation_lock = self.supervisor.operation_lock();
3711 let _operation_guard = operation_lock.lock().await;
3712 let Some(module) = self.supervisor.get(&module_id) else {
3713 return Ok(vec![control_error_frame(
3714 &frame,
3715 "unknown_module",
3716 format!("module_id '{module_id}' is not supervised"),
3717 )?]);
3718 };
3719
3720 if let Err(err) = module.reload().await {
3721 let (code, message) = match err {
3722 crate::supervise::SuperviseError::Disabled { .. } => {
3723 ("module_disabled", err.to_string())
3724 }
3725 crate::supervise::SuperviseError::SwapInProgress { .. } => {
3726 ("swap_in_progress", err.to_string())
3727 }
3728 _ => (
3729 "reload_failed",
3730 format!("failed to reload module_id '{module_id}': {err}"),
3731 ),
3732 };
3733 return Ok(vec![control_error_frame(&frame, code, message)?]);
3734 }
3735
3736 let response = ClientControlResponse::SupervisorAck {
3737 module_id,
3738 applied: true,
3739 };
3740 Ok(vec![control_response_body_frame(
3741 &frame,
3742 &response,
3743 "ClientControlResponse::SupervisorAck",
3744 )?])
3745 }
3746
3747 async fn handle_supervisor_rescan(
3748 &self,
3749 frame: Frame,
3750 preview: bool,
3751 ) -> Result<Vec<Frame>, RouterError> {
3752 let Some(context) = self.rescan.clone() else {
3753 return Ok(vec![control_error_frame(
3754 &frame,
3755 "rescan_unavailable",
3756 "the daemon was not started with a reloadable config path".to_string(),
3757 )?]);
3758 };
3759
3760 let operation_lock = self.supervisor.operation_lock();
3761 let _operation_guard = operation_lock.lock().await;
3762 let loaded = match crate::daemon_config::load(&context.config_path) {
3763 Ok(config) => config,
3764 Err(err) => {
3765 return Ok(vec![control_error_frame(
3766 &frame,
3767 "invalid_daemon_config",
3768 format!("supervisor rescan rejected daemon config: {err}"),
3769 )?])
3770 }
3771 };
3772 let Some(config) = loaded else {
3780 return Ok(vec![control_error_frame(
3781 &frame,
3782 "invalid_daemon_config",
3783 format!(
3784 "daemon config not found at {}; refusing to rescan (an absent config would \
3785 retire every supervised module)",
3786 context.config_path.display()
3787 ),
3788 )?]);
3789 };
3790 let (
3791 configured_port,
3792 storage_config,
3793 admission_facts_carrier_module_id,
3794 admission_facts_targets,
3795 modules,
3796 reserved_capabilities,
3797 ) = (
3798 config.port,
3799 config.storage,
3800 config.admission_facts_carrier_module_id,
3801 config.admission_facts_targets,
3802 config.modules,
3803 config.reserved_capabilities,
3804 );
3805
3806 let mut restart_required = Vec::new();
3815 for section in RestartRequiredSection::ALL {
3816 let changed = match section {
3817 RestartRequiredSection::Port => configured_port != context.configured_port,
3818 RestartRequiredSection::Storage => storage_config != context.storage_config,
3819 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
3820 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
3821 }
3822 RestartRequiredSection::AdmissionFactsTargets => {
3823 admission_facts_targets != context.admission_facts_targets
3824 }
3825 };
3826 if changed {
3827 restart_required.push(section.label().to_string());
3828 }
3829 }
3830 if !restart_required.is_empty() {
3831 warn!(
3832 config_path = %context.config_path.display(),
3833 sections = %restart_required.join(", "),
3834 "daemon config changed outside the modules section; restart the daemon to apply those changes"
3835 );
3836 }
3837
3838 for configured in &modules {
3839 if let Err(err) = validate_spec(&configured.module_spec()) {
3840 return Ok(vec![control_error_frame(
3841 &frame,
3842 "invalid_daemon_config",
3843 format!("supervisor rescan rejected daemon config: {err}"),
3844 )?]);
3845 }
3846 }
3847
3848 let configured_capabilities = modules
3849 .iter()
3850 .map(|module| (module.module_id.clone(), module.enabled))
3851 .collect::<Vec<_>>();
3852 let preview_capability_warnings = if preview {
3853 let (_, registrations) = self.runtime_capability_snapshot()?;
3854 let current_modules = self
3855 .supervisor
3856 .list()
3857 .into_iter()
3858 .map(|module| module.module_id().to_string())
3859 .collect::<BTreeSet<_>>();
3860 let resulting_modules = configured_capabilities.clone();
3861 let removed = current_modules
3862 .into_iter()
3863 .filter(|module_id| {
3864 !resulting_modules
3865 .iter()
3866 .any(|(configured_id, _)| configured_id == module_id)
3867 })
3868 .collect::<Vec<_>>();
3869 self.capability_evaluator.preview_removal_warnings(
3870 resulting_modules,
3871 &removed,
3872 ®istrations,
3873 )
3874 } else {
3875 Vec::new()
3876 };
3877 let result = match self
3878 .reconcile_supervised_modules(&context.supervisor, modules, preview)
3879 .await
3880 {
3881 Ok(result) => result,
3882 Err(message) => {
3883 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
3884 }
3885 };
3886 if !preview {
3887 self.capability_evaluator
3888 .configure(configured_capabilities, reserved_capabilities);
3889 self.capability_evaluator.wake_deadline_loop();
3890 self.refresh_capability_requirements();
3891 }
3892 let mut result = result;
3893 result.restart_required = restart_required;
3894 result.capability_warnings = preview_capability_warnings;
3895 let response = ClientControlResponse::SupervisorRescan { result };
3896 Ok(vec![control_response_body_frame(
3897 &frame,
3898 &response,
3899 "ClientControlResponse::SupervisorRescan",
3900 )?])
3901 }
3902
3903 async fn handle_supervisor_release_reserved(
3904 &self,
3905 frame: Frame,
3906 module_id: String,
3907 ) -> Result<Vec<Frame>, RouterError> {
3908 let Some(context) = self.rescan.clone() else {
3909 return Ok(vec![control_error_frame(
3910 &frame,
3911 "release_unavailable",
3912 "reserved-id release requires a daemon started with a reloadable config path",
3913 )?]);
3914 };
3915 let operation_lock = self.supervisor.operation_lock();
3916 let _operation_guard = operation_lock.lock().await;
3917 let loaded = match crate::daemon_config::load(&context.config_path) {
3918 Ok(Some(config)) => config,
3919 Ok(None) => {
3920 return Ok(vec![control_error_frame(
3921 &frame,
3922 "invalid_daemon_config",
3923 format!(
3924 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
3925 context.config_path.display()
3926 ),
3927 )?])
3928 }
3929 Err(err) => {
3930 return Ok(vec![control_error_frame(
3931 &frame,
3932 "invalid_daemon_config",
3933 format!("unable to verify reserved-id release against daemon config: {err}"),
3934 )?])
3935 }
3936 };
3937 if loaded
3938 .modules
3939 .iter()
3940 .any(|configured| configured.module_id == module_id)
3941 {
3942 return Ok(vec![control_error_frame(
3943 &frame,
3944 "reserved_module_configured",
3945 format!(
3946 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
3947 ),
3948 )?]);
3949 }
3950 if !self.supervisor.release_retained_reserved_gate(&module_id) {
3951 return Ok(vec![control_error_frame(
3952 &frame,
3953 "reserved_gate_not_retained",
3954 format!(
3955 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
3956 ),
3957 )?]);
3958 }
3959
3960 let response = ClientControlResponse::SupervisorAck {
3961 module_id,
3962 applied: true,
3963 };
3964 Ok(vec![control_response_body_frame(
3965 &frame,
3966 &response,
3967 "ClientControlResponse::SupervisorAck",
3968 )?])
3969 }
3970
3971 async fn reconcile_supervised_modules(
3980 &self,
3981 supervisor: &Supervisor,
3982 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
3983 preview: bool,
3984 ) -> Result<SupervisorRescanResult, String> {
3985 let mut current = BTreeMap::new();
3986 for module in self.supervisor.list() {
3987 let (spec, health) = module.configuration().map_err(|err| {
3988 format!(
3989 "failed to read configuration for module_id '{}': {err}",
3990 module.module_id()
3991 )
3992 })?;
3993 let enabled = module
3994 .status()
3995 .map_err(|err| {
3996 format!(
3997 "failed to read status for module_id '{}': {err}",
3998 module.module_id()
3999 )
4000 })?
4001 .enabled;
4002 current.insert(
4003 module.module_id().to_string(),
4004 (module, spec, health, enabled),
4005 );
4006 }
4007 let configured = configured_modules
4008 .into_iter()
4009 .map(|module| (module.module_id.clone(), module))
4010 .collect::<BTreeMap<_, _>>();
4011
4012 let added = configured
4013 .keys()
4014 .filter(|module_id| !current.contains_key(*module_id))
4015 .cloned()
4016 .collect::<Vec<_>>();
4017 let removed = current
4018 .keys()
4019 .filter(|module_id| !configured.contains_key(*module_id))
4020 .cloned()
4021 .collect::<Vec<_>>();
4022 let mut changed_pending_reload = Vec::new();
4023 let mut configuration_changes = BTreeSet::new();
4024 let mut enabled_changes = BTreeSet::new();
4025 let mut unchanged = 0_u32;
4026
4027 for (module_id, configured_module) in &configured {
4028 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
4029 else {
4030 continue;
4031 };
4032 let configuration_changed = *current_spec != configured_module.module_spec()
4033 || *current_health != configured_module.health;
4034 let enabled_changed = *current_enabled != configured_module.enabled;
4035 if configuration_changed {
4036 configuration_changes.insert(module_id.clone());
4037 changed_pending_reload.push(module_id.clone());
4038 }
4039 if enabled_changed {
4040 enabled_changes.insert(module_id.clone());
4041 }
4042 if !configuration_changed && !enabled_changed {
4043 unchanged = unchanged.saturating_add(1);
4044 }
4045 }
4046
4047 if preview {
4053 return Ok(SupervisorRescanResult {
4054 added,
4055 removed,
4056 changed_pending_reload,
4057 enabled_changes: enabled_changes.iter().cloned().collect(),
4058 unchanged,
4059 preview: true,
4060 restart_required: Vec::new(),
4064 capability_warnings: Vec::new(),
4065 });
4066 }
4067
4068 for module_id in &removed {
4069 let module = ¤t
4070 .get(module_id)
4071 .expect("removed module came from current supervisor state")
4072 .0;
4073 module.retire().await.map_err(|err| {
4074 format!("failed to retire module_id '{module_id}' during rescan: {err}")
4075 })?;
4076 self.supervisor.record_rescan_removal(module_id);
4110 self.supervisor.retire(module_id);
4111 }
4112
4113 for module_id in configured.keys() {
4114 let Some((module, _, _, _)) = current.get(module_id) else {
4115 continue;
4116 };
4117 let configured_module = configured
4118 .get(module_id)
4119 .expect("configured module id came from configured map");
4120 if configuration_changes.contains(module_id) {
4121 module
4122 .update_configuration(
4123 configured_module.module_spec(),
4124 configured_module.health,
4125 configured_module.drain_timeout_ms,
4126 )
4127 .await
4128 .map_err(|err| {
4129 format!(
4130 "failed to update module_id '{module_id}' configuration during rescan: {err}"
4131 )
4132 })?;
4133 }
4134 if enabled_changes.contains(module_id) {
4135 module
4136 .set_enabled(configured_module.enabled)
4137 .await
4138 .map_err(|err| {
4139 format!(
4140 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
4141 configured_module.enabled
4142 )
4143 })?;
4144 }
4145 }
4146
4147 for module_id in &added {
4148 let configured_module = configured
4149 .get(module_id)
4150 .expect("added module id came from configured map");
4151 supervisor
4152 .supervise_configured_with_health(
4153 configured_module.module_spec(),
4154 configured_module.enabled,
4155 configured_module.health,
4156 configured_module.drain_timeout_ms,
4157 configured_module.restart,
4158 )
4159 .map_err(|err| {
4160 format!("failed to add module_id '{module_id}' during rescan: {err}")
4161 })?;
4162 }
4163
4164 Ok(SupervisorRescanResult {
4165 added,
4166 removed,
4167 changed_pending_reload,
4168 enabled_changes: enabled_changes.iter().cloned().collect(),
4169 unchanged,
4170 preview: false,
4171 restart_required: Vec::new(),
4174 capability_warnings: Vec::new(),
4175 })
4176 }
4177
4178 async fn handle_supervisor_set_enabled(
4179 &self,
4180 frame: Frame,
4181 module_id: String,
4182 enabled: bool,
4183 ) -> Result<Vec<Frame>, RouterError> {
4184 let operation_lock = self.supervisor.operation_lock();
4185 let _operation_guard = operation_lock.lock().await;
4186 let Some(module) = self.supervisor.get(&module_id) else {
4187 return Ok(vec![control_error_frame(
4188 &frame,
4189 "unknown_module",
4190 format!("module_id '{module_id}' is not supervised"),
4191 )?]);
4192 };
4193
4194 let applied = match module.set_enabled(enabled).await {
4195 Ok(applied) => applied,
4196 Err(err) => {
4197 return Ok(vec![control_error_frame(
4198 &frame,
4199 "target_unavailable",
4200 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
4201 )?])
4202 }
4203 };
4204
4205 self.capability_evaluator.wake_deadline_loop();
4206 self.refresh_capability_requirements();
4207 let response = ClientControlResponse::SupervisorAck { module_id, applied };
4208 Ok(vec![control_response_body_frame(
4209 &frame,
4210 &response,
4211 "ClientControlResponse::SupervisorAck",
4212 )?])
4213 }
4214
4215 async fn handle_supervisor_health_probe(
4216 &self,
4217 frame: Frame,
4218 module_id: String,
4219 ) -> Result<Vec<Frame>, RouterError> {
4220 self.refresh_capability_requirements();
4221 let Some(registration) = self
4222 .registry
4223 .get_module(&module_id)
4224 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4225 else {
4226 return Ok(vec![control_error_frame(
4227 &frame,
4228 "unknown_module",
4229 format!("module_id '{module_id}' is not registered"),
4230 )?]);
4231 };
4232
4233 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
4246 {
4247 return Ok(vec![control_error_frame(
4248 &frame,
4249 "health_not_advertised",
4250 format!("module_id '{module_id}' did not advertise health.check"),
4251 )?]);
4252 }
4253
4254 let deadline = Instant::now() + self.health_probe_timeout;
4255 let pending = match self.forwarding.begin_module_control_rpc_for(
4256 &module_id,
4257 MODULE_CONTROL_OP_HEALTH_CHECK,
4258 deadline,
4259 ) {
4260 Ok(pending) => pending,
4261 Err(err) => {
4262 return Ok(vec![control_error_frame(
4263 &frame,
4264 forwarding_error_code(&err),
4265 err.to_string(),
4266 )?])
4267 }
4268 };
4269
4270 let PendingModuleControlRpc {
4271 endpoint,
4272 module_sink,
4273 negotiated_ver,
4274 corr: probe_corr,
4275 receiver,
4276 } = pending;
4277 let mut guard =
4278 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
4279 let probe_body =
4280 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
4281 RouterError::backend(
4282 0,
4283 frame.header.corr,
4284 format!("failed to encode health.check request: {err}"),
4285 )
4286 })?;
4287 let probe_frame = Frame::build_with_version(
4288 negotiated_ver,
4289 FrameType::Request,
4290 control_flags(),
4291 0,
4292 0,
4293 probe_corr,
4294 probe_body,
4295 )
4296 .map_err(RouterError::FrameBuild)?;
4297
4298 if let Err(err) = module_sink.send(probe_frame).await {
4299 return Ok(vec![control_error_frame(
4300 &frame,
4301 "target_unavailable",
4302 err.to_string(),
4303 )?]);
4304 }
4305
4306 match timeout_at(deadline, receiver).await {
4307 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
4308 guard.disarm();
4309 let Some(report) = response.health_report() else {
4310 return Ok(vec![control_error_frame(
4311 &frame,
4312 "invalid_control_body",
4313 "health.check RPC returned a non-health response",
4314 )?]);
4315 };
4316 let HealthReport {
4321 status,
4322 detail,
4323 metrics,
4324 } = report;
4325 let capability_detail = self
4326 .capability_evaluator
4327 .required_problem_detail(&module_id);
4328 let response = ClientControlResponse::SupervisorHealthProbe {
4329 module_id,
4330 status,
4331 detail: append_capability_problem_detail(detail, capability_detail),
4332 metrics,
4333 };
4334 Ok(vec![control_response_body_frame(
4335 &frame,
4336 &response,
4337 "ClientControlResponse::SupervisorHealthProbe",
4338 )?])
4339 }
4340 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
4341 guard.disarm();
4342 Ok(vec![control_error_body_frame(&frame, body)?])
4343 }
4344 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
4345 guard.disarm();
4346 Ok(vec![control_error_frame(
4347 &frame,
4348 "target_unavailable",
4349 message,
4350 )?])
4351 }
4352 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
4353 guard.disarm();
4354 Ok(vec![control_error_frame(
4355 &frame,
4356 "invalid_control_body",
4357 message,
4358 )?])
4359 }
4360 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
4361 guard.disarm();
4362 Ok(vec![control_error_frame(
4363 &frame,
4364 "invalid_control_body",
4365 format!("expected module-control op '{expected}', got '{actual}'"),
4366 )?])
4367 }
4368 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
4369 guard.disarm();
4370 Ok(vec![control_error_frame(
4371 &frame,
4372 "module_timeout",
4373 format!(
4374 "module_id '{module_id}' answered health.check after {:?}",
4375 self.health_probe_timeout
4376 ),
4377 )?])
4378 }
4379 Ok(Err(_)) => Ok(vec![control_error_frame(
4380 &frame,
4381 "target_unavailable",
4382 "health.check waiter was canceled before the module responded",
4383 )?]),
4384 Err(_) => Ok(vec![control_error_frame(
4385 &frame,
4386 "module_timeout",
4387 format!(
4388 "module_id '{module_id}' did not answer health.check within {:?}",
4389 self.health_probe_timeout
4390 ),
4391 )?]),
4392 }
4393 }
4394
4395 fn supervisor_status(
4396 &self,
4397 module_id: &str,
4398 corr: u64,
4399 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
4400 self.supervisor
4401 .get(module_id)
4402 .map(|module| {
4403 let warming = module.is_warming_for_control("status").map_err(|err| {
4404 RouterError::backend(
4405 0,
4406 corr,
4407 format!(
4408 "failed to read supervisor warming state for module_id '{module_id}': {err}"
4409 ),
4410 )
4411 })?;
4412 module.status_for_control("status").map_err(|err| {
4413 RouterError::backend(
4414 0,
4415 corr,
4416 format!(
4417 "failed to read supervisor status for module_id '{module_id}': {err}"
4418 ),
4419 )
4420 }).map(|status| (status, warming))
4421 })
4422 .transpose()
4423 }
4424
4425 fn guard_module_control_op(
4426 &self,
4427 frame: &Frame,
4428 module_id: &str,
4429 op: &str,
4430 ) -> Result<Option<Frame>, RouterError> {
4431 if self.module_grants_op(module_id, op, frame.header.corr)? {
4432 return Ok(None);
4433 }
4434
4435 Ok(Some(control_error_frame(
4436 frame,
4437 "op_not_allowed",
4438 format!("module_id '{module_id}' did not grant control op '{op}'"),
4439 )?))
4440 }
4441
4442 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
4443 let Some(registration) = self
4444 .registry
4445 .get_module(module_id)
4446 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
4447 else {
4448 return Ok(false);
4449 };
4450 Ok(module_registration_grants_op(®istration.control_ops, op))
4451 }
4452
4453 fn handle_status_update(
4454 &self,
4455 endpoint: ModuleEndpointId,
4456 frame: Frame,
4457 ) -> Result<Vec<Frame>, RouterError> {
4458 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
4459 Ok(update) => update,
4460 Err(err) => {
4461 if is_known_module_push_op(&frame.body) {
4466 return Ok(vec![control_error_frame(
4467 &frame,
4468 "invalid_control_body",
4469 format!("malformed module control push body: {err}"),
4470 )?]);
4471 }
4472 return Ok(Vec::new());
4473 }
4474 };
4475
4476 match update {
4477 ModuleControlPush::RouteStatus {
4478 route_channel,
4479 route_epoch,
4480 status,
4481 } => {
4482 self.forwarding
4483 .cache_status(endpoint, route_channel, route_epoch, status)
4484 .map_err(RouterError::Forwarding)?;
4485 }
4486 }
4487 Ok(Vec::new())
4488 }
4489
4490 fn handle_route_poll(
4491 &self,
4492 ctx: &RouteCtx,
4493 frame: Frame,
4494 route_channel: u16,
4495 route_epoch: u32,
4496 kind: PollKind,
4497 ) -> Result<Vec<Frame>, RouterError> {
4498 let snapshot = self
4499 .forwarding
4500 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
4501 .map_err(RouterError::Forwarding)?;
4502 let response = match (kind, snapshot) {
4503 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
4504 ClientControlResponse::RoutePoll {
4505 route_channel,
4506 route_epoch,
4507 status,
4508 live: None,
4509 }
4510 }
4511 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
4512 route_channel,
4513 route_epoch,
4514 status: None,
4515 live: None,
4516 },
4517 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
4518 let live = self
4544 .process_liveness
4545 .as_ref()
4546 .and_then(|source| source.process_live(&module_id))
4547 .unwrap_or(true);
4548 ClientControlResponse::RoutePoll {
4549 route_channel,
4550 route_epoch,
4551 status: None,
4552 live: Some(live),
4553 }
4554 }
4555 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
4556 route_channel,
4557 route_epoch,
4558 status: None,
4559 live: Some(false),
4560 },
4561 };
4562
4563 Ok(vec![control_response_body_frame(
4564 &frame,
4565 &response,
4566 "ClientControlResponse::RoutePoll",
4567 )?])
4568 }
4569
4570 pub(crate) fn observe_module_control_completion(
4571 &self,
4572 completion: ModuleControlRpcCompletion,
4573 ) -> bool {
4574 match completion {
4575 ModuleControlRpcCompletion::Unknown => false,
4576 ModuleControlRpcCompletion::Settled => true,
4577 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
4578 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
4579 info!(
4580 module_id = %module_id,
4581 latency_ms,
4582 "late health.check answer proves the module is alive"
4583 );
4584 match self
4585 .supervisor
4586 .record_late_health_answer(&module_id, latency_ms)
4587 {
4588 Ok(true) => {}
4589 Ok(false) => debug!(
4590 module_id = %module_id,
4591 latency_ms,
4592 "late health.check answer has no active supervisor snapshot"
4593 ),
4594 Err(err) => warn!(
4595 module_id = %module_id,
4596 latency_ms,
4597 error = %err,
4598 "failed to record late health.check answer"
4599 ),
4600 }
4601 true
4602 }
4603 }
4604 }
4605
4606 fn refuse_to_end_module_connection_for_a_client(
4627 &self,
4628 module_connection_id: ConnectionId,
4629 corr: u64,
4630 err: ForwardingError,
4631 ) -> Result<(), RouterError> {
4632 if let ForwardingError::ConnectionClosing { connection_id } = err {
4633 if connection_id != module_connection_id {
4634 warn!(
4635 module_connection_id = module_connection_id.get(),
4636 client_connection_id = connection_id.get(),
4637 corr,
4638 "dropping a route.bind response for a closing client; the module connection keeps serving"
4639 );
4640 return Ok(());
4641 }
4642 }
4643 Err(RouterError::Forwarding(err))
4644 }
4645
4646 fn handle_module_relay_response(
4647 &self,
4648 connection_id: ConnectionId,
4649 frame: Frame,
4650 ) -> Result<Vec<Frame>, RouterError> {
4651 let mut secondary_error = None;
4652 let outcome = match frame.header.ty {
4653 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
4654 Ok(probe) if probe.op == "route.bind" => {
4655 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
4656 Ok(ModuleControlResponse::RouteBindAck {}) => {
4657 RouteBindRelayOutcome::Accepted
4658 }
4659 Ok(other) => {
4660 let message =
4661 format!("route.bind response carried unexpected body: {other:?}");
4662 secondary_error = Some(control_error_frame(
4663 &frame,
4664 "invalid_control_body",
4665 message.clone(),
4666 )?);
4667 RouteBindRelayOutcome::ModuleGone(message)
4668 }
4669 Err(err) => {
4670 let message = format!("malformed route.bind response body: {err}");
4671 secondary_error = Some(control_error_frame(
4672 &frame,
4673 "invalid_control_body",
4674 message.clone(),
4675 )?);
4676 RouteBindRelayOutcome::ModuleGone(message)
4677 }
4678 }
4679 }
4680 Ok(probe) => {
4681 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
4682 {
4683 Ok(response) => ModuleControlRpcOutcome::Response(response),
4684 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
4685 "malformed {} response body: {err}",
4686 probe.op
4687 )),
4688 };
4689 let completion = self
4690 .forwarding
4691 .complete_module_control_rpc(
4692 connection_id,
4693 frame.header.corr,
4694 Some(&probe.op),
4695 outcome,
4696 )
4697 .map_err(RouterError::Forwarding)?;
4698 if !self.observe_module_control_completion(completion) {
4699 debug!(
4700 connection_id = connection_id.get(),
4701 corr = frame.header.corr,
4702 op = %probe.op,
4703 "dropping late or unknown module-control RPC response"
4704 );
4705 }
4706 return Ok(Vec::new());
4707 }
4708 Err(err) => {
4709 if let Some(expected_op) = self
4710 .forwarding
4711 .pending_module_control_op(connection_id, frame.header.corr)
4712 .map_err(RouterError::Forwarding)?
4713 {
4714 let completion = self
4715 .forwarding
4716 .complete_module_control_rpc(
4717 connection_id,
4718 frame.header.corr,
4719 None,
4720 ModuleControlRpcOutcome::MalformedResponse(format!(
4721 "malformed {expected_op} response body: {err}"
4722 )),
4723 )
4724 .map_err(RouterError::Forwarding)?;
4725 if !self.observe_module_control_completion(completion) {
4726 debug!(
4727 connection_id = connection_id.get(),
4728 corr = frame.header.corr,
4729 "dropping late malformed module-control RPC response"
4730 );
4731 }
4732 return Ok(Vec::new());
4733 }
4734 let message = format!("malformed route.bind response body: {err}");
4735 secondary_error = Some(control_error_frame(
4736 &frame,
4737 "invalid_control_body",
4738 message.clone(),
4739 )?);
4740 RouteBindRelayOutcome::ModuleGone(message)
4741 }
4742 },
4743 FrameType::Error => {
4744 if self
4745 .forwarding
4746 .pending_module_control_op(connection_id, frame.header.corr)
4747 .map_err(RouterError::Forwarding)?
4748 .is_some()
4749 {
4750 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
4751 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
4752 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
4753 "malformed module-control ERROR body: {err}"
4754 )),
4755 };
4756 let completion = self
4757 .forwarding
4758 .complete_module_control_rpc(
4759 connection_id,
4760 frame.header.corr,
4761 None,
4762 outcome,
4763 )
4764 .map_err(RouterError::Forwarding)?;
4765 if !self.observe_module_control_completion(completion) {
4766 debug!(
4767 connection_id = connection_id.get(),
4768 corr = frame.header.corr,
4769 "dropping late or unknown module-control RPC error"
4770 );
4771 }
4772 return Ok(Vec::new());
4773 }
4774 match serde_json::from_slice::<ErrorBody>(&frame.body) {
4775 Ok(body) => RouteBindRelayOutcome::Rejected(body),
4776 Err(err) => {
4777 let message = format!("malformed route.bind ERROR body: {err}");
4778 secondary_error = Some(control_error_frame(
4779 &frame,
4780 "invalid_control_body",
4781 message.clone(),
4782 )?);
4783 RouteBindRelayOutcome::ModuleGone(message)
4784 }
4785 }
4786 }
4787 ty => {
4788 return Ok(vec![control_error_frame(
4789 &frame,
4790 "unsupported_control_frame",
4791 format!("unsupported module channel-0 frame {ty:?}"),
4792 )?])
4793 }
4794 };
4795
4796 let settled =
4797 self.forwarding
4798 .complete_pending_relay(connection_id, frame.header.corr, outcome);
4799 let completion = match settled {
4800 Ok(completion) => completion,
4801 Err(err) => {
4802 self.refuse_to_end_module_connection_for_a_client(
4803 connection_id,
4804 frame.header.corr,
4805 err,
4806 )?;
4807 return Ok(secondary_error.into_iter().collect());
4808 }
4809 };
4810 if let Some(target) = completion.abandoned.as_ref() {
4811 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
4812 }
4813 if !completion.settled {
4814 debug!(
4815 connection_id = connection_id.get(),
4816 corr = frame.header.corr,
4817 frame_type = ?frame.header.ty,
4818 "dropping late or unknown route.bind relay response"
4819 );
4820 }
4821 Ok(secondary_error.into_iter().collect())
4822 }
4823
4824 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
4825 debug!(connection_id = connection_id.get(), "handling GOODBYE");
4826 let registrations = self
4827 .deregister_connection(connection_id)
4828 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
4829 let released_routes = self
4830 .forwarding
4831 .cleanup_connection(connection_id)
4832 .map_err(RouterError::Forwarding)?;
4833 self.emit_route_goodbyes(released_routes);
4834 if !registrations.is_empty() {
4836 crate::supervise::notify_registration_release();
4837 }
4838 Ok(Vec::new())
4839 }
4840}
4841
4842impl Default for ControlHandler {
4843 fn default() -> Self {
4844 Self::new(Arc::new(Registry::default()))
4845 }
4846}
4847
4848impl crate::supervise::SwapPromotionObserver for ControlHandler {
4849 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
4850 self.apply_registration_capabilities(registration);
4851 }
4852}
4853
4854fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
4855 CapabilityRequirementStatus {
4856 consumer: status.consumer,
4857 capability: status.capability,
4858 need: match status.need {
4859 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
4860 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
4861 },
4862 verdict: status.verdict.as_str().to_string(),
4863 episode_seq: status.episode_seq,
4864 config_satisfiable: status.config_satisfiable,
4865 runtime_available: status.runtime_available,
4866 detail: status.detail,
4867 }
4868}
4869
4870fn append_capability_problem_detail(
4871 detail: Option<String>,
4872 capability_detail: Option<String>,
4873) -> Option<String> {
4874 match (detail, capability_detail) {
4875 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
4876 (Some(detail), None) => Some(detail),
4877 (None, Some(capability_detail)) => Some(capability_detail),
4878 (None, None) => None,
4879 }
4880}
4881
4882fn subc_ops() -> Vec<String> {
4883 SUBC_CONTROL_OPS
4884 .iter()
4885 .map(|op| (*op).to_string())
4886 .collect()
4887}
4888
4889fn module_subc_ops() -> Vec<String> {
4890 SUBC_CONTROL_OPS
4891 .iter()
4892 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
4893 .map(|op| (*op).to_string())
4894 .collect()
4895}
4896
4897#[cfg(test)]
4898fn module_baseline_control_ops() -> Vec<String> {
4899 MODULE_BASELINE_CONTROL_OPS
4900 .iter()
4901 .map(|op| (*op).to_string())
4902 .collect()
4903}
4904
4905fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
4906 let mut seen = HashSet::new();
4907 let mut effective = Vec::new();
4908 for op in MODULE_BASELINE_CONTROL_OPS {
4909 if seen.insert((*op).to_string()) {
4910 effective.push((*op).to_string());
4911 }
4912 }
4913 for op in declared.unwrap_or_default() {
4914 if seen.insert(op.clone()) {
4915 effective.push(op);
4916 }
4917 }
4918 effective
4919}
4920
4921fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
4922 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
4923}
4924
4925fn target_module_id(target: &RouteTarget) -> &str {
4926 match target {
4927 RouteTarget::ToolProvider { module_id }
4928 | RouteTarget::ManagementSurface { module_id }
4929 | RouteTarget::InternalService { module_id, .. } => module_id,
4930 }
4931}
4932
4933fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
4934 roles.iter().any(|role| match (target, role) {
4935 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
4936 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
4937 (
4938 RouteTarget::InternalService { service_id, .. },
4939 ProviderRole::InternalService {
4940 service_id: provided,
4941 ..
4942 },
4943 ) => service_id == provided,
4944 _ => false,
4945 })
4946}
4947
4948fn is_routable_role(role: &ProviderRole) -> bool {
4949 matches!(
4950 role,
4951 ProviderRole::ToolProvider { .. }
4952 | ProviderRole::ManagementSurface { .. }
4953 | ProviderRole::InternalService { .. }
4954 )
4955}
4956
4957#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4958enum ControlRequestBodyError {
4959 UnknownOp,
4960 InvalidBody,
4961}
4962
4963#[derive(Debug, Deserialize)]
4964struct ControlOpProbe {
4965 op: String,
4966}
4967
4968const MODULE_PUSH_OPS: &[&str] = &["route.status"];
4971
4972fn is_known_module_push_op(body: &[u8]) -> bool {
4973 serde_json::from_slice::<ControlOpProbe>(body)
4974 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
4975 .unwrap_or(false)
4976}
4977
4978fn is_known_module_request_op(body: &[u8]) -> bool {
4979 serde_json::from_slice::<ControlOpProbe>(body)
4980 .map(|probe| MODULE_TO_SUBC_CONTROL_OPS.contains(&probe.op.as_str()))
4981 .unwrap_or(false)
4982}
4983
4984fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
4985 debug!(
4986 op = %op,
4987 connection_id = connection_id.get(),
4988 corr,
4989 "control dispatch"
4990 );
4991}
4992
4993fn log_slow_control_dispatch(
4994 dispatch_started_at: Option<StdInstant>,
4995 op: &'static str,
4996 connection_id: ConnectionId,
4997 corr: u64,
4998) {
4999 let Some(dispatch_started_at) = dispatch_started_at else {
5000 return;
5001 };
5002 let elapsed = dispatch_started_at.elapsed();
5003 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
5004 warn!(
5005 op = %op,
5006 connection_id = connection_id.get(),
5007 corr,
5008 elapsed_ms = elapsed.as_millis() as u64,
5009 "slow control dispatch"
5010 );
5011 }
5012}
5013
5014fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
5015 match request {
5016 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
5017 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
5018 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
5019 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
5020 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
5021 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
5022 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
5023 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
5024 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
5025 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
5026 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
5027 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
5028 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
5029 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
5030 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
5031 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
5032 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
5033 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
5034 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
5035 }
5036}
5037
5038fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
5039 match request {
5040 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
5041 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
5042 }
5043}
5044
5045fn parse_client_control_request(
5046 body: &[u8],
5047) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
5048 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
5049 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5050 Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
5051 ControlRequestBodyError::InvalidBody
5052 }
5053 Ok(_) => ControlRequestBodyError::UnknownOp,
5054 Err(_) => ControlRequestBodyError::InvalidBody,
5055 };
5056 (err, classification)
5057 })
5058}
5059
5060fn parse_module_control_request_from_module(
5061 body: &[u8],
5062) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
5063 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
5064 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5065 Ok(probe) if MODULE_TO_SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
5066 ControlRequestBodyError::InvalidBody
5067 }
5068 Ok(_) => ControlRequestBodyError::UnknownOp,
5069 Err(_) => ControlRequestBodyError::InvalidBody,
5070 };
5071 (err, classification)
5072 })
5073}
5074
5075#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
5076enum ProviderRoleKind {
5077 ToolProvider,
5078 PipelineStage,
5079 ManagementSurface,
5080 InternalService,
5081}
5082
5083fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
5084 match role {
5085 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
5086 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
5087 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
5088 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
5089 }
5090}
5091
5092fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
5093 roles.iter().map(provider_role_kind).collect()
5094}
5095
5096fn capability_census_trigger(
5100 old: Option<&CapabilityDeclarations>,
5101 new: Option<&CapabilityDeclarations>,
5102) -> bool {
5103 let old_provides = old
5104 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
5105 .unwrap_or_default();
5106 let old_denies = old
5107 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
5108 .unwrap_or_default();
5109 let new = new.cloned().unwrap_or(CapabilityDeclarations {
5110 provides: Vec::new(),
5111 requires: Vec::new(),
5112 must_never_reach: Vec::new(),
5113 });
5114
5115 new.provides
5116 .iter()
5117 .any(|capability| !old_provides.contains(capability))
5118 || new
5119 .must_never_reach
5120 .iter()
5121 .any(|capability| !old_denies.contains(capability))
5122}
5123
5124fn denied_capability<'a>(
5127 opening_manifest: &'a ModuleManifest,
5128 target_manifest: &ModuleManifest,
5129) -> Option<&'a str> {
5130 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
5131 let target_capabilities = target_manifest.capabilities.as_ref()?;
5132 opening_capabilities
5133 .must_never_reach
5134 .iter()
5135 .find(|denied| {
5136 target_capabilities
5137 .provides
5138 .iter()
5139 .any(|provided| provided == *denied)
5140 })
5141 .map(String::as_str)
5142}
5143
5144fn catalog_update_frozen_field_message(
5145 registered: &ModuleManifest,
5146 provides: &[ProviderRole],
5147) -> Option<String> {
5148 let old_has_provides = !registered.provides.is_empty();
5149 let new_has_provides = !provides.is_empty();
5150 if old_has_provides != new_has_provides {
5151 return Some(format!(
5152 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
5153 registered.module_id
5154 ));
5155 }
5156
5157 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
5158 return Some(format!(
5159 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
5160 registered.module_id
5161 ));
5162 }
5163
5164 let registered_concurrency = manifest_concurrency(registered);
5165 let mut candidate = registered.clone();
5166 candidate.provides = provides.to_vec();
5167 let candidate_concurrency = manifest_concurrency(&candidate);
5168 if candidate_concurrency != registered_concurrency {
5169 return Some(format!(
5170 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
5171 registered.module_id, registered_concurrency, candidate_concurrency
5172 ));
5173 }
5174
5175 None
5178}
5179
5180fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
5181 manifest.provides.iter().any(is_routable_role)
5182}
5183
5184fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
5190 manifest
5191 .provides
5192 .iter()
5193 .find_map(|provider| match provider {
5194 ProviderRole::ToolProvider { concurrency, .. }
5195 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
5196 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
5197 })
5198 .unwrap_or(Concurrency::ModuleManaged)
5199}
5200
5201fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
5208 let has_management_surface = manifest
5209 .provides
5210 .iter()
5211 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
5212 if !has_management_surface {
5213 return false;
5214 }
5215 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
5216 return false;
5217 };
5218 let Some(provides) = raw
5219 .get("manifest")
5220 .and_then(|manifest| manifest.get("provides"))
5221 .and_then(serde_json::Value::as_array)
5222 else {
5223 return false;
5224 };
5225 provides.iter().any(|role| {
5230 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
5231 && role.get("concurrency").is_none()
5232 })
5233}
5234
5235fn negotiate_version(peer_version: u8) -> Result<u8, String> {
5236 if peer_version != PROTOCOL_VERSION {
5237 return Err(format!(
5238 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
5239 ));
5240 }
5241 Ok(PROTOCOL_VERSION)
5242}
5243
5244fn pong(frame: &Frame) -> Result<Frame, RouterError> {
5245 Frame::build_with_version(
5246 response_version(frame),
5247 FrameType::Pong,
5248 frame.header.flags,
5249 0,
5250 0,
5251 frame.header.corr,
5252 Vec::new(),
5253 )
5254 .map_err(RouterError::FrameBuild)
5255}
5256
5257fn control_error_frame(
5258 frame: &Frame,
5259 code: &'static str,
5260 message: impl Into<String>,
5261) -> Result<Frame, RouterError> {
5262 control_error_body_frame(
5263 frame,
5264 ErrorBody {
5265 code: code.to_string(),
5266 message: message.into(),
5267 detail: None,
5268 },
5269 )
5270}
5271
5272fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
5273 let body = serde_json::to_vec(&error).map_err(|err| {
5274 RouterError::backend(
5275 0,
5276 frame.header.corr,
5277 format!("failed to encode control ERROR: {err}"),
5278 )
5279 })?;
5280
5281 Frame::build_with_version(
5282 response_version(frame),
5283 FrameType::Error,
5284 control_flags(),
5285 0,
5286 0,
5287 frame.header.corr,
5288 body,
5289 )
5290 .map_err(RouterError::FrameBuild)
5291}
5292
5293fn control_response_body_frame<T: Serialize>(
5294 frame: &Frame,
5295 reply: &T,
5296 label: &'static str,
5297) -> Result<Frame, RouterError> {
5298 let body = serde_json::to_vec(reply).map_err(|err| {
5299 RouterError::backend(
5300 0,
5301 frame.header.corr,
5302 format!("failed to encode {label}: {err}"),
5303 )
5304 })?;
5305
5306 Frame::build_with_version(
5307 response_version(frame),
5308 FrameType::Response,
5309 control_flags(),
5310 0,
5311 0,
5312 frame.header.corr,
5313 body,
5314 )
5315 .map_err(RouterError::FrameBuild)
5316}
5317
5318fn forwarding_error_code(err: &ForwardingError) -> &'static str {
5348 match err {
5349 ForwardingError::NoModuleConnection => "target_unavailable",
5350 ForwardingError::ModuleReloading { .. } => "module_reloading",
5351 ForwardingError::ClientRouteChannelExhausted { .. }
5352 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
5353 ForwardingError::StaleModuleEndpoint
5354 | ForwardingError::UnknownReservation { .. }
5355 | ForwardingError::ConnectionClosing { .. }
5356 | ForwardingError::ClientEgressClosed { .. }
5357 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
5358 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
5361 ForwardingError::RelayCorrelationExhausted
5362 | ForwardingError::RouteOpenBuild(_)
5363 | ForwardingError::Poisoned => "forwarding_error",
5364 }
5365}
5366
5367fn response_version(frame: &Frame) -> u8 {
5368 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
5369 frame.header.ver
5370 } else {
5371 PROTOCOL_VERSION
5372 }
5373}
5374
5375fn control_flags() -> Flags {
5376 Flags::new(false, Priority::Passive, false)
5377}
5378
5379fn send_goodbye_target_best_effort(
5384 counters: &DaemonCounters,
5385 target: &GoodbyeTarget,
5386 context: &'static str,
5387) {
5388 let Ok(frame) = Frame::build_with_version(
5389 target.negotiated_ver,
5390 FrameType::Goodbye,
5391 control_flags(),
5392 target.channel,
5393 target.epoch,
5394 0,
5395 Vec::new(),
5396 ) else {
5397 return;
5398 };
5399 crate::forwarding::send_module_route_goodbye(
5400 counters,
5401 &target.sink,
5402 frame,
5403 target.module_id.as_deref(),
5404 context,
5405 );
5406}
5407
5408pub(crate) fn send_route_control_pushes(
5409 forwarding: &ForwardingTable,
5410 routes: Vec<EndpointRoute>,
5411 push: ClientControlPush,
5412) {
5413 let body = match serde_json::to_vec(&push) {
5414 Ok(body) => body,
5415 Err(err) => {
5416 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
5417 return;
5418 }
5419 };
5420 let mut targets = Vec::new();
5421 for route in routes {
5422 let target = route.goodbye_target;
5423 if let Some(existing) = targets
5424 .iter()
5425 .find(|existing: &&GoodbyeTarget| existing.connection_id == target.connection_id)
5426 {
5427 debug_assert_eq!(
5428 existing.negotiated_ver, target.negotiated_ver,
5429 "one connection cannot negotiate multiple frame versions"
5430 );
5431 continue;
5432 }
5433 targets.push(target);
5434 }
5435 for target in targets {
5436 let frame = match Frame::build_with_version(
5437 target.negotiated_ver,
5438 FrameType::Push,
5439 control_flags(),
5440 0,
5441 0,
5442 0,
5443 body.clone(),
5444 ) {
5445 Ok(frame) => frame,
5446 Err(err) => {
5447 warn!(
5448 route_channel = target.channel,
5449 error = %err,
5450 "failed to build route lifecycle control PUSH frame"
5451 );
5452 continue;
5453 }
5454 };
5455 if let Err(err) = target.sink.try_send(frame) {
5456 if target.close_on_delivery_failure() {
5457 warn!(
5458 target_connection_id = target.connection_id.get(),
5459 route_channel = target.channel,
5460 error = %err,
5461 "route lifecycle control PUSH was not delivered to client; closing target connection"
5462 );
5463 let _ = forwarding.escalate_client_delivery_failure(
5464 target.connection_id,
5465 target.channel,
5466 target.epoch,
5467 CloseReason::new(
5468 "route_lifecycle_push_delivery_failed",
5469 format!(
5470 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
5471 target.channel
5472 ),
5473 ),
5474 crate::forwarding::UndeliveredFrame {
5475 module_id: target.module_id.as_deref(),
5476 sink: &target.sink,
5477 },
5478 );
5479 }
5480 }
5481 }
5482}
5483
5484#[cfg(test)]
5485mod tests {
5486 use std::{
5487 collections::BTreeMap,
5488 fmt,
5489 path::PathBuf,
5490 sync::{Arc, Mutex},
5491 time::Duration,
5492 };
5493
5494 use serde_json::{json, Value};
5495 use subc_protocol::{
5496 manifest::{
5497 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
5498 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
5499 },
5500 session::HealthStatus,
5501 FrameType,
5502 };
5503
5504 use super::*;
5505 use crate::{
5506 forwarding::{DataRoute, DataRouteState},
5507 registry::ChannelState,
5508 router::FrameSink,
5509 stderr_tail::DEFAULT_MAX_LINE_BYTES,
5510 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
5511 test_support::TestTempDir,
5512 RouteCtx, Router,
5513 };
5514 use tokio::{
5515 sync::mpsc,
5516 time::{sleep, Instant},
5517 };
5518 use tracing::{
5519 field::{Field, Visit},
5520 Event, Subscriber,
5521 };
5522 use tracing_subscriber::{layer::Context, prelude::*, Layer};
5523
5524 fn fake_aft_stub_path() -> PathBuf {
5543 let mut path = std::env::current_exe().expect("current_exe available in tests");
5544 path.pop(); path.pop(); path.push(if cfg!(windows) {
5547 "fake-aft-stub.exe"
5548 } else {
5549 "fake-aft-stub"
5550 });
5551 assert!(
5552 path.exists(),
5553 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
5554 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
5555 path.display()
5556 );
5557 path
5558 }
5559
5560 fn client_retries(code: &str) -> bool {
5565 subc_protocol::error_codes::is_retryable_route_open(code)
5566 }
5567
5568 #[test]
5579 fn retryability_of_forwarding_codes_matches_the_failure() {
5580 let transient = [
5583 ForwardingError::NoModuleConnection,
5584 ForwardingError::ModuleReloading {
5585 module_id: "m".into(),
5586 },
5587 ForwardingError::StaleModuleEndpoint,
5588 ForwardingError::UnknownReservation {
5589 client_channel: 1,
5590 module_channel: 1,
5591 },
5592 ForwardingError::ConnectionClosing {
5593 connection_id: ConnectionId::new(1),
5594 },
5595 ForwardingError::ClientEgressClosed {
5596 connection_id: ConnectionId::new(1),
5597 },
5598 ForwardingError::ModuleEgressUnavailable {
5599 connection_id: ConnectionId::new(1),
5600 },
5601 ];
5602 for err in transient {
5603 let code = forwarding_error_code(&err);
5604 assert!(
5605 client_retries(code),
5606 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
5607 );
5608 }
5609
5610 let permanent = [
5615 ForwardingError::ClientRouteChannelExhausted {
5616 connection_id: ConnectionId::new(1),
5617 },
5618 ForwardingError::ModuleRouteChannelExhausted {
5619 endpoint: ModuleEndpointId {
5620 connection_id: ConnectionId::new(1),
5621 generation: 1,
5622 },
5623 },
5624 ForwardingError::RelayCorrelationExhausted,
5625 ForwardingError::RouteOpenBuild("x".into()),
5626 ForwardingError::Poisoned,
5627 ];
5628 for err in permanent {
5629 let code = forwarding_error_code(&err);
5630 assert!(
5631 !client_retries(code),
5632 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
5633 );
5634 }
5635 }
5636
5637 #[tokio::test]
5665 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
5666 let handler = ControlHandler::default();
5667 let frame =
5668 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
5669
5670 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
5674 assert_eq!(
5675 stamped,
5676 Principal::Direct,
5677 "a caller that proved nothing must not be stamped as a supervised module"
5678 );
5679
5680 let forged = handler
5685 .route_open_principal(
5686 &frame,
5687 Some(ConsumerIdentity {
5688 module_id: "aft".to_string(),
5689 launch_nonce: "not-a-real-nonce".to_string(),
5690 }),
5691 )
5692 .unwrap();
5693 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
5694 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
5695 }
5696
5697 #[test]
5714 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
5715 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
5716 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
5717 let ClientControlRequest::RouteOpen {
5718 consumer_identity, ..
5719 } = parsed
5720 else {
5721 panic!("route.open body must parse as RouteOpen");
5722 };
5723 assert_eq!(
5724 consumer_identity,
5725 Some(ConsumerIdentity {
5726 module_id: "aft".to_string(),
5727 launch_nonce: "n".to_string(),
5728 }),
5729 "the wire field name must reach the value route_open_principal reads"
5730 );
5731 }
5732
5733 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
5734 ModuleManifest::builder(module_id, "0.1.0")
5735 .protocol_ver(protocol_ver)
5736 .provides(vec![ProviderRole::ToolProvider {
5737 tools: vec![Tool {
5738 name: "read".to_string(),
5739 description: None,
5740 execution_mode: ExecutionMode::Pure,
5741 schema: json!({"type": "object"}),
5742 }],
5743 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
5744 concurrency: Concurrency::ModuleManaged,
5745 emits_push: true,
5746 sub_supervises: true,
5747 }])
5748 .build()
5749 }
5750
5751 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
5752 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
5753 }
5754
5755 fn hello_frame_with_control_ops(
5756 module_id: &str,
5757 protocol_ver: u8,
5758 corr: u64,
5759 control_ops: Option<Vec<String>>,
5760 ) -> Frame {
5761 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
5762 }
5763
5764 fn hello_frame_with_nonce(
5765 module_id: &str,
5766 protocol_ver: u8,
5767 corr: u64,
5768 launch_nonce: Option<&str>,
5769 ) -> Frame {
5770 hello_frame_full(
5771 module_id,
5772 protocol_ver,
5773 corr,
5774 None,
5775 launch_nonce.map(ToOwned::to_owned),
5776 )
5777 }
5778
5779 fn hello_frame_full(
5780 module_id: &str,
5781 protocol_ver: u8,
5782 corr: u64,
5783 control_ops: Option<Vec<String>>,
5784 launch_nonce: Option<String>,
5785 ) -> Frame {
5786 let body = serde_json::to_vec(&ModuleHelloBody {
5787 manifest: manifest(module_id, protocol_ver),
5788 protocol_ver,
5789 control_ops,
5790 launch_nonce,
5791 })
5792 .unwrap();
5793 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
5794 }
5795
5796 fn non_routable_hello_frame_with_control_ops(
5797 module_id: &str,
5798 corr: u64,
5799 control_ops: Option<Vec<String>>,
5800 ) -> Frame {
5801 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
5802 manifest.provides.clear();
5803 let body = serde_json::to_vec(&ModuleHelloBody {
5804 manifest,
5805 protocol_ver: PROTOCOL_VERSION,
5806 control_ops,
5807 launch_nonce: None,
5808 })
5809 .unwrap();
5810 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
5811 }
5812
5813 fn capability_grammar_hello_frame(
5814 capabilities: Value,
5815 runtime_computed: Option<Value>,
5816 corr: u64,
5817 ) -> Frame {
5818 let mut body = serde_json::to_value(ModuleHelloBody {
5819 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
5820 protocol_ver: PROTOCOL_VERSION,
5821 control_ops: None,
5822 launch_nonce: None,
5823 })
5824 .expect("HELLO body serializes");
5825 body["manifest"]["capabilities"] = capabilities;
5826 if let Some(runtime_computed) = runtime_computed {
5827 body["runtime_computed"] = runtime_computed;
5828 }
5829 Frame::build(
5830 FrameType::Hello,
5831 control_flags(),
5832 0,
5833 0,
5834 corr,
5835 serde_json::to_vec(&body).expect("HELLO body reserializes"),
5836 )
5837 .expect("HELLO frame builds")
5838 }
5839
5840 fn channel_request(channel: u16, corr: u64) -> Frame {
5841 Frame::build(
5842 FrameType::Request,
5843 Flags::new(true, Priority::Interactive, false),
5844 channel,
5845 0,
5846 corr,
5847 b"opaque".to_vec(),
5848 )
5849 .unwrap()
5850 }
5851
5852 fn route_ctx(
5853 connection_id: ConnectionId,
5854 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
5855 let (tx, rx) = mpsc::channel(8);
5856 (
5857 RouteCtx {
5858 connection_id,
5859 egress: FrameSink::new(tx),
5860 },
5861 rx,
5862 )
5863 }
5864
5865 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
5866 serde_json::from_slice(&frame.body).unwrap()
5867 }
5868
5869 async fn hello_via_sink(
5874 handler: &ControlHandler,
5875 ctx: &RouteCtx,
5876 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
5877 hello: Frame,
5878 ) -> Frame {
5879 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
5880 assert!(
5881 replies.is_empty(),
5882 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
5883 );
5884 let ack = rx
5885 .try_recv()
5886 .expect("HELLO_ACK is queued on the module sink")
5887 .frame;
5888 assert_eq!(ack.header.ty, FrameType::HelloAck);
5889 ack
5890 }
5891
5892 fn parse_error(frame: &Frame) -> Value {
5893 serde_json::from_slice(&frame.body).unwrap()
5894 }
5895
5896 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
5897 serde_json::from_slice(&frame.body).unwrap()
5898 }
5899
5900 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
5901 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
5902 route_channel,
5903 route_epoch: 0,
5904 kind,
5905 })
5906 .unwrap();
5907 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
5908 }
5909
5910 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
5911 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
5912 module_id: module_id.to_string(),
5913 })
5914 .unwrap();
5915 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
5916 }
5917
5918 fn route_open_frame(corr: u64, module_id: &str, project_root: TestTempDir) -> Frame {
5919 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
5920 }
5921
5922 fn route_open_frame_with_consumer_capabilities(
5923 corr: u64,
5924 module_id: &str,
5925 project_root: TestTempDir,
5926 consumer_capabilities: Option<Vec<String>>,
5927 ) -> Frame {
5928 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
5929 target: RouteTarget::ToolProvider {
5930 module_id: module_id.to_string(),
5931 },
5932 identity: BindIdentity::new(
5933 project_root.path().to_path_buf(),
5934 "unit".to_string(),
5935 "session".to_string(),
5936 ),
5937 consumer_identity: None,
5938 consumer_capabilities,
5939 admission_facts: None,
5940 })
5941 .unwrap();
5942 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
5943 }
5944
5945 fn route_open_frame_with_admission_facts(
5946 corr: u64,
5947 module_id: &str,
5948 project_root: TestTempDir,
5949 consumer_identity: Option<subc_control::ConsumerIdentity>,
5950 facts: Option<Value>,
5951 ) -> Frame {
5952 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
5953 target: RouteTarget::ToolProvider {
5954 module_id: module_id.to_string(),
5955 },
5956 identity: BindIdentity::new(
5957 project_root.path().to_path_buf(),
5958 "unit".to_string(),
5959 format!("session-{corr}"),
5960 ),
5961 consumer_identity,
5962 consumer_capabilities: None,
5963 admission_facts: facts,
5964 })
5965 .unwrap();
5966 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
5967 }
5968
5969 #[derive(Clone, Default)]
5970 struct EventCapture {
5971 events: Arc<Mutex<Vec<CapturedEvent>>>,
5972 }
5973
5974 #[derive(Clone, Debug)]
5975 struct CapturedEvent {
5976 target: String,
5977 fields: BTreeMap<String, String>,
5978 }
5979
5980 impl EventCapture {
5981 fn events(&self) -> Vec<CapturedEvent> {
5982 self.events.lock().unwrap().clone()
5983 }
5984 }
5985
5986 impl<S> Layer<S> for EventCapture
5987 where
5988 S: Subscriber,
5989 {
5990 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
5991 let mut visitor = EventFieldVisitor::default();
5992 event.record(&mut visitor);
5993 self.events.lock().unwrap().push(CapturedEvent {
5994 target: event.metadata().target().to_string(),
5995 fields: visitor.fields,
5996 });
5997 }
5998 }
5999
6000 #[derive(Default)]
6001 struct EventFieldVisitor {
6002 fields: BTreeMap<String, String>,
6003 }
6004
6005 impl Visit for EventFieldVisitor {
6006 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
6007 self.fields
6008 .insert(field.name().to_string(), format!("{value:?}"));
6009 }
6010 }
6011
6012 fn health_response(corr: u64, status: HealthStatus) -> Frame {
6013 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
6014 status,
6015 detail: Some("warming".to_string()),
6016 metrics: Some(json!({"queue_depth": 3})),
6017 })
6018 .unwrap();
6019 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
6020 }
6021
6022 fn route_bind_ack(corr: u64) -> Frame {
6023 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
6024 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
6025 }
6026
6027 fn unique_project_root(label: &str) -> TestTempDir {
6028 TestTempDir::new(label)
6029 }
6030
6031 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
6032 match parse_route_poll(frame) {
6033 ClientControlResponse::RoutePoll {
6034 status: None,
6035 live: Some(live),
6036 ..
6037 } => assert_eq!(live, expected_live),
6038 other => panic!("unexpected route.poll response: {other:?}"),
6039 }
6040 }
6041
6042 fn bind_liveness_route(
6043 registry: &Registry,
6044 forwarding: &ForwardingTable,
6045 module_id: &str,
6046 ) -> (RouteCtx, u16, u32) {
6047 let module_connection = ConnectionId::new(101);
6048 let client_connection = ConnectionId::new(202);
6049 let registration = registry
6050 .register_with_control_ops(
6051 manifest(module_id, PROTOCOL_VERSION),
6052 PROTOCOL_VERSION,
6053 module_connection,
6054 module_baseline_control_ops(),
6055 )
6056 .unwrap();
6057 let (module_tx, _module_rx) = mpsc::channel(8);
6058 let endpoint = forwarding
6059 .register_module_connection(
6060 module_connection,
6061 module_id.to_string(),
6062 PROTOCOL_VERSION,
6063 manifest_concurrency(®istration.manifest),
6064 FrameSink::new(module_tx),
6065 )
6066 .unwrap();
6067 let (client_ctx, _client_rx) = route_ctx(client_connection);
6068 let pending = forwarding
6069 .begin_route_bind_relay_for_test(
6070 client_connection,
6071 client_ctx.egress.clone(),
6072 1,
6073 module_id,
6074 )
6075 .unwrap();
6076 assert_eq!(pending.endpoint, endpoint);
6077 let route_channel = pending.client_channel;
6078 let route_epoch = pending.client_epoch;
6079 forwarding
6080 .complete_pending_relay(
6081 module_connection,
6082 pending.corr,
6083 RouteBindRelayOutcome::Accepted,
6084 )
6085 .unwrap();
6086 (client_ctx, route_channel, route_epoch)
6087 }
6088
6089 struct FakeProcessLiveness {
6090 live: Option<bool>,
6091 }
6092
6093 impl ModuleProcessLiveness for FakeProcessLiveness {
6094 fn process_live(&self, _module_id: &str) -> Option<bool> {
6095 self.live
6096 }
6097 }
6098
6099 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6100 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
6101 {
6102 let registry = Arc::new(Registry::default());
6103 let supervisor_handle = SupervisorHandle::new();
6104 let supervisor = Supervisor::new(
6105 Arc::clone(®istry),
6106 RestartPolicy::new(1, Duration::from_millis(10)),
6107 )
6108 .with_handle(supervisor_handle.clone());
6109 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
6110 let module = supervisor
6111 .spawn(ModuleSpec {
6112 module_id: "stderr-tail-wire".to_string(),
6113 program: fake_aft_stub_path(),
6114 args: Vec::new(),
6115 env: vec![
6116 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
6117 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
6118 ],
6119 reserved: false,
6120 reserved_prefixes: Vec::new(),
6121 protocol: ModuleProtocol::Subc,
6122 overlap: Default::default(),
6123 })
6124 .unwrap();
6125
6126 let deadline = Instant::now() + Duration::from_secs(5);
6127 loop {
6128 let tail = module.stderr_tail(None, None);
6129 if tail
6130 .entries
6131 .iter()
6132 .any(|entry| matches!(entry, TailEntry::ProcessStart))
6133 && tail.entries.iter().any(|entry| {
6134 matches!(
6135 entry,
6136 TailEntry::Line {
6137 truncated: true,
6138 ..
6139 }
6140 )
6141 })
6142 {
6143 break;
6144 }
6145 assert!(
6146 Instant::now() < deadline,
6147 "module did not produce a truncated line and restart boundary: {tail:?}"
6148 );
6149 sleep(Duration::from_millis(10)).await;
6150 }
6151
6152 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
6153 let request = ClientControlRequest::SupervisorStderrTail {
6154 module_id: "stderr-tail-wire".to_string(),
6155 max_lines: None,
6156 max_bytes: None,
6157 };
6158 let frame = Frame::build(
6159 FrameType::Request,
6160 control_flags(),
6161 0,
6162 0,
6163 1,
6164 serde_json::to_vec(&request).unwrap(),
6165 )
6166 .unwrap();
6167 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6168 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
6169 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
6170 serde_json::from_slice(&responses[0].body).unwrap()
6171 else {
6172 panic!("expected supervisor.stderr_tail response");
6173 };
6174
6175 assert!(
6176 tail.entries
6177 .iter()
6178 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
6179 "the control response lost the restart boundary"
6180 );
6181 let Some(StderrTailEntry::Line { text, truncated }) = tail.entries.iter().find(|entry| {
6182 matches!(
6183 entry,
6184 StderrTailEntry::Line {
6185 truncated: true,
6186 ..
6187 }
6188 )
6189 }) else {
6190 panic!("the control response lost the truncated line");
6191 };
6192 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
6193 assert!(*truncated);
6194 }
6195
6196 #[tokio::test(flavor = "current_thread")]
6201 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
6202 let dir = TestTempDir::new("terminals-off-worker");
6203 let journal_path = dir.join("terminals.jsonl");
6204 let registry = Arc::new(Registry::default());
6205 let supervisor_handle = SupervisorHandle::new();
6206 let supervisor =
6207 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
6208 .with_handle(supervisor_handle.clone())
6209 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
6210 let module = supervisor
6211 .spawn(ModuleSpec {
6212 module_id: "terminal-off-worker".to_string(),
6213 program: fake_aft_stub_path(),
6214 args: Vec::new(),
6215 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
6216 reserved: false,
6217 reserved_prefixes: Vec::new(),
6218 protocol: ModuleProtocol::Subc,
6219 overlap: Default::default(),
6220 })
6221 .unwrap();
6222 let deadline = Instant::now() + Duration::from_secs(5);
6223 while module.terminal_history().entries.len() != 2 {
6224 assert!(Instant::now() < deadline, "module did not record two exits");
6225 sleep(Duration::from_millis(10)).await;
6226 }
6227
6228 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
6229 let handler =
6230 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
6231 let frame = Frame::build(
6232 FrameType::Request,
6233 control_flags(),
6234 0,
6235 0,
6236 1,
6237 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
6238 module_id: "terminal-off-worker".to_string(),
6239 })
6240 .unwrap(),
6241 )
6242 .unwrap();
6243 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6244 let spawned_at = std::time::Instant::now();
6245 let read = tokio::spawn({
6246 let handler = Arc::clone(&handler);
6247 async move { handler.handle_control_frame(&ctx, frame).await }
6248 });
6249 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
6252 .await
6253 .unwrap()
6254 .expect("the history read reached its pause");
6255 let elapsed = spawned_at.elapsed();
6256 assert!(
6257 elapsed < Duration::from_secs(2) && !read.is_finished(),
6258 "this task could not run while the history read was paused \
6259 (resumed after {elapsed:?}, read finished: {})",
6260 read.is_finished()
6261 );
6262
6263 drop(release);
6264 let responses = read.await.unwrap().unwrap();
6265 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
6266 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
6267 panic!("expected supervisor.terminals response");
6268 };
6269 assert_eq!(terminals.entries.len(), 2);
6270 assert_eq!(terminals.journal_skipped_lines, 0);
6271 assert_eq!(terminals.journal_read_errors, 0);
6272 }
6273
6274 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6275 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
6276 let registry = Arc::new(Registry::default());
6277 let supervisor_handle = SupervisorHandle::new();
6278 let supervisor =
6279 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
6280 .with_handle(supervisor_handle.clone());
6281 let module = supervisor
6282 .spawn(ModuleSpec {
6283 module_id: "terminal-golden".to_string(),
6284 program: fake_aft_stub_path(),
6285 args: Vec::new(),
6286 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
6287 reserved: false,
6288 reserved_prefixes: Vec::new(),
6289 protocol: ModuleProtocol::Subc,
6290 overlap: Default::default(),
6291 })
6292 .unwrap();
6293
6294 let deadline = Instant::now() + Duration::from_secs(5);
6295 while module.terminal_history().entries.len() != 2 {
6296 assert!(
6297 Instant::now() < deadline,
6298 "module did not retain two terminal exits: {:?}",
6299 module.terminal_history()
6300 );
6301 sleep(Duration::from_millis(10)).await;
6302 }
6303
6304 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
6305 let request = ClientControlRequest::SupervisorTerminals {
6306 module_id: "terminal-golden".to_string(),
6307 };
6308 let frame = Frame::build(
6309 FrameType::Request,
6310 control_flags(),
6311 0,
6312 0,
6313 1,
6314 serde_json::to_vec(&request).unwrap(),
6315 )
6316 .unwrap();
6317 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6318 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
6319 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
6320 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
6321 panic!("expected supervisor.terminals response");
6322 };
6323 assert_eq!(terminals.entries.len(), 2);
6324 assert_eq!(terminals.dropped, 0);
6325
6326 let mut rendered = serde_json::to_value(response).unwrap();
6327 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
6330 for (index, entry) in rendered["entries"]
6331 .as_array_mut()
6332 .expect("terminal response entries array")
6333 .iter_mut()
6334 .enumerate()
6335 {
6336 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
6337 }
6338
6339 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
6340 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
6341 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
6342 if std::env::var_os("UPDATE_GOLDEN").is_some() {
6343 std::fs::write(&golden_path, &serialized).unwrap();
6344 }
6345 let expected: Value =
6346 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
6347 assert_eq!(rendered, expected);
6348 }
6349
6350 #[test]
6351 fn hello_registers_manifest_and_returns_ack() {
6352 let registry = Arc::new(Registry::default());
6353 let handler = ControlHandler::new(Arc::clone(®istry));
6354 let conn = ConnectionId::new(1);
6355
6356 let responses = handler
6357 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
6358 .unwrap();
6359
6360 assert_eq!(responses.len(), 1);
6361 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
6362 assert_eq!(responses[0].header.channel, 0);
6363 assert_eq!(responses[0].header.corr, 7);
6364 let ack = parse_ack(&responses[0]);
6365 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
6366 assert!(ack
6367 .subc_capabilities
6368 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
6369 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
6370 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
6371 assert!(ack
6372 .subc_ops
6373 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
6374 assert!(ack
6375 .subc_ops
6376 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
6377
6378 let registration = registry.get_module("aft").unwrap().unwrap();
6379 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
6380 assert_eq!(registration.state, ChannelState::Active);
6381 assert_eq!(registration.connection_id, conn);
6382 assert_eq!(registration.control_ops, module_baseline_control_ops());
6383 }
6384
6385 #[test]
6386 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
6387 let invalid_identifiers = [
6388 ("case_change", "credentials-Provider/v1"),
6389 ("leading_zero", "credentials-provider/v01"),
6390 ("trailing_hyphen", "credentials-provider-/v1"),
6391 ("consecutive_hyphens", "credentials--provider/v1"),
6392 ("uppercase", "Credentials-provider/v1"),
6393 ("missing_v", "credentials-provider/1"),
6394 ("whitespace", "credentials provider/v1"),
6395 ("zero_version", "credentials-provider/v0"),
6396 ("out_of_range_version", "credentials-provider/v4294967296"),
6397 (
6398 "overlength_name",
6399 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
6400 ),
6401 ];
6402 let mut cases = invalid_identifiers
6403 .into_iter()
6404 .map(|(name, identifier)| {
6405 (
6406 format!("identifier_{name}"),
6407 "capabilities.provides[0]".to_string(),
6408 identifier.to_string(),
6409 json!({ "provides": [identifier] }),
6410 None,
6411 )
6412 })
6413 .collect::<Vec<_>>();
6414 cases.extend([
6415 (
6416 "unknown_need".to_string(),
6417 "capabilities.requires[0].need".to_string(),
6418 "deferred".to_string(),
6419 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
6420 None,
6421 ),
6422 (
6423 "duplicate_provides".to_string(),
6424 "capabilities.provides[1]".to_string(),
6425 "credentials-provider/v1".to_string(),
6426 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
6427 None,
6428 ),
6429 (
6430 "duplicate_must_never_reach".to_string(),
6431 "capabilities.must_never_reach[1]".to_string(),
6432 "credentials-provider/v1".to_string(),
6433 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
6434 None,
6435 ),
6436 (
6437 "duplicate_requires_same_need".to_string(),
6438 "capabilities.requires[1]".to_string(),
6439 "credentials-provider/v1".to_string(),
6440 json!({ "requires": [
6441 { "capability": "credentials-provider/v1", "need": "required" },
6442 { "capability": "credentials-provider/v1", "need": "required" }
6443 ] }),
6444 None,
6445 ),
6446 (
6447 "duplicate_requires_conflicting_need".to_string(),
6448 "capabilities.requires[1]".to_string(),
6449 "credentials-provider/v1".to_string(),
6450 json!({ "requires": [
6451 { "capability": "credentials-provider/v1", "need": "required" },
6452 { "capability": "credentials-provider/v1", "need": "optional" }
6453 ] }),
6454 None,
6455 ),
6456 (
6457 "capabilities_root_pointer".to_string(),
6458 "runtime_computed[0]".to_string(),
6459 "/capabilities".to_string(),
6460 json!({}),
6461 Some(json!(["/capabilities"])),
6462 ),
6463 (
6464 "capabilities_descendant_pointer".to_string(),
6465 "runtime_computed[0]".to_string(),
6466 "/capabilities/provides".to_string(),
6467 json!({}),
6468 Some(json!(["/capabilities/provides"])),
6469 ),
6470 (
6471 "malformed_pointer_without_leading_slash".to_string(),
6472 "runtime_computed[0]".to_string(),
6473 "capabilities".to_string(),
6474 json!({}),
6475 Some(json!(["capabilities"])),
6476 ),
6477 (
6478 "malformed_pointer_escape".to_string(),
6479 "runtime_computed[0]".to_string(),
6480 "/roles/~2/tools".to_string(),
6481 json!({}),
6482 Some(json!(["/roles/~2/tools"])),
6483 ),
6484 (
6485 "unknown_capabilities_field".to_string(),
6486 "capabilities.future".to_string(),
6487 "<array>".to_string(),
6488 json!({ "future": [] }),
6489 None,
6490 ),
6491 ]);
6492
6493 for (index, (name, field, value, capabilities, runtime_computed)) in
6494 cases.into_iter().enumerate()
6495 {
6496 let registry = Arc::new(Registry::default());
6497 let handler = ControlHandler::new(Arc::clone(®istry));
6498 let response = handler
6499 .handle_control(
6500 ConnectionId::new((index + 1) as u64),
6501 capability_grammar_hello_frame(
6502 capabilities,
6503 runtime_computed,
6504 index as u64 + 1,
6505 ),
6506 )
6507 .expect("invalid HELLO returns a refusal");
6508
6509 assert_eq!(response.len(), 1, "{name} must emit one refusal");
6510 let error = parse_error(&response[0]);
6511 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
6512 let message = error["message"]
6513 .as_str()
6514 .expect("error message is a string");
6515 assert!(
6516 message.contains(&field),
6517 "{name}: field missing from {message}"
6518 );
6519 assert!(
6520 message.contains(&value),
6521 "{name}: value missing from {message}"
6522 );
6523 assert_eq!(
6524 registry
6525 .active_registration_count()
6526 .expect("registry reads"),
6527 0,
6528 "{name}: refused HELLO must not create a catalog entry"
6529 );
6530 }
6531 }
6532
6533 #[test]
6534 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
6535 let registry = Arc::new(Registry::default());
6536 let handler = ControlHandler::new(Arc::clone(®istry));
6537 let capabilities = json!({
6538 "provides": ["credentials-provider/v1"],
6539 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
6540 "must_never_reach": ["federation-transport/v1"]
6541 });
6542 let response = handler
6543 .handle_control(
6544 ConnectionId::new(99),
6545 capability_grammar_hello_frame(
6546 capabilities.clone(),
6547 Some(json!(["/roles/0/tools"])),
6548 99,
6549 ),
6550 )
6551 .expect("valid HELLO registers");
6552 assert_eq!(response[0].header.ty, FrameType::HelloAck);
6553
6554 let request = Frame::build(
6555 FrameType::Request,
6556 control_flags(),
6557 0,
6558 0,
6559 100,
6560 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
6561 .expect("catalog request serializes"),
6562 )
6563 .expect("catalog request frame builds");
6564 let response = handler
6565 .handle_catalog_list(request, None)
6566 .expect("catalog list succeeds");
6567 let ClientControlResponse::CatalogList { modules, .. } =
6568 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
6569 else {
6570 panic!("catalog request must return catalog.list");
6571 };
6572 assert_eq!(modules.len(), 1);
6573 assert_eq!(
6574 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
6575 capabilities
6576 );
6577 }
6578
6579 #[test]
6580 fn catalog_list_mirrors_management_operation_description() {
6581 let registry = Arc::new(Registry::default());
6582 let handler = ControlHandler::new(Arc::clone(®istry));
6583 let description = "List managed records and return their identifiers and metadata.";
6584 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
6585 manifest.provides = vec![ProviderRole::ManagementSurface {
6586 operations: vec![ManagementOperation {
6587 name: "records.list".to_string(),
6588 kind: ManagementOperationKind::Query,
6589 description: Some(description.to_string()),
6590 }],
6591 config_schema: json!({"type": "object"}),
6592 observability: vec![ObservabilitySurface {
6593 name: "records.stats".to_string(),
6594 kind: ObservabilityKind::Snapshot,
6595 }],
6596 identity_scope: vec![IdentityScope::Project],
6597 concurrency: Concurrency::ModuleManaged,
6598 }];
6599 registry
6600 .register_with_control_ops(
6601 manifest,
6602 PROTOCOL_VERSION,
6603 ConnectionId::new(99),
6604 Vec::new(),
6605 )
6606 .expect("described management manifest registers");
6607
6608 let request = Frame::build(
6609 FrameType::Request,
6610 control_flags(),
6611 0,
6612 0,
6613 100,
6614 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
6615 .expect("catalog request serializes"),
6616 )
6617 .expect("catalog request frame builds");
6618 let response = handler
6619 .handle_catalog_list(request, None)
6620 .expect("catalog list succeeds");
6621 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
6622 assert_eq!(
6623 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
6624 "catalog.list must preserve the declared operation description verbatim"
6625 );
6626 }
6627
6628 #[test]
6629 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
6630 let registry = Arc::new(Registry::default());
6631 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
6632 [("vault".to_string(), true), ("squatter".to_string(), true)],
6633 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
6634 );
6635 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
6636 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
6637 provides: vec!["credentials-provider/v1".to_string()],
6638 requires: Vec::new(),
6639 must_never_reach: Vec::new(),
6640 });
6641 let frame = Frame::build(
6642 FrameType::Hello,
6643 control_flags(),
6644 0,
6645 0,
6646 77,
6647 serde_json::to_vec(&ModuleHelloBody {
6648 manifest: squatter,
6649 protocol_ver: PROTOCOL_VERSION,
6650 control_ops: None,
6651 launch_nonce: None,
6652 })
6653 .expect("HELLO serializes"),
6654 )
6655 .expect("HELLO frame builds");
6656 let response = handler
6657 .handle_control(ConnectionId::new(77), frame)
6658 .expect("reserved claim receives a typed refusal");
6659 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
6660 assert_eq!(
6661 registry
6662 .active_registration_count()
6663 .expect("registry reads"),
6664 0,
6665 "a reserved capability refusal must not leave a catalog entry"
6666 );
6667 }
6668
6669 #[test]
6670 fn server_describe_surfaces_required_capability_verdict_fields() {
6671 let registry = Arc::new(Registry::default());
6672 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
6673 [
6674 ("consumer".to_string(), true),
6675 ("provider".to_string(), false),
6676 ],
6677 BTreeMap::new(),
6678 );
6679 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
6680 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
6681 provides: Vec::new(),
6682 requires: vec![subc_protocol::manifest::CapabilityRequirement {
6683 capability: "credentials-provider/v1".to_string(),
6684 need: subc_protocol::manifest::CapabilityNeed::Required,
6685 }],
6686 must_never_reach: Vec::new(),
6687 });
6688 let hello = Frame::build(
6689 FrameType::Hello,
6690 control_flags(),
6691 0,
6692 0,
6693 78,
6694 serde_json::to_vec(&ModuleHelloBody {
6695 manifest: consumer,
6696 protocol_ver: PROTOCOL_VERSION,
6697 control_ops: None,
6698 launch_nonce: None,
6699 })
6700 .expect("HELLO serializes"),
6701 )
6702 .expect("HELLO frame builds");
6703 handler
6704 .handle_control(ConnectionId::new(78), hello)
6705 .expect("consumer registers");
6706 let describe = Frame::build(
6707 FrameType::Request,
6708 control_flags(),
6709 0,
6710 0,
6711 79,
6712 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
6713 .expect("request serializes"),
6714 )
6715 .expect("describe frame builds");
6716 let response = handler
6717 .handle_server_describe(describe)
6718 .expect("server.describe succeeds");
6719 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
6720 let requirement = &rendered["capability_requirements"][0];
6721 assert_eq!(requirement["consumer"], "consumer");
6722 assert_eq!(requirement["verdict"], "never_provided");
6723 assert_eq!(requirement["episode_seq"], 1);
6724 assert_eq!(requirement["config_satisfiable"], false);
6725 assert_eq!(requirement["runtime_available"], false);
6726 assert!(requirement["detail"]
6727 .as_str()
6728 .expect("detail string")
6729 .contains("credentials-provider/v1"));
6730 }
6731
6732 #[test]
6733 fn catalog_list_omits_capabilities_for_legacy_manifest() {
6734 let registry = Arc::new(Registry::default());
6735 let handler = ControlHandler::new(Arc::clone(®istry));
6736 let hello = handler
6737 .handle_control(
6738 ConnectionId::new(101),
6739 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
6740 )
6741 .expect("legacy HELLO registers");
6742 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
6743
6744 let request = Frame::build(
6745 FrameType::Request,
6746 control_flags(),
6747 0,
6748 0,
6749 102,
6750 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
6751 .expect("catalog request serializes"),
6752 )
6753 .expect("catalog request frame builds");
6754 let response = handler
6755 .handle_catalog_list(request, None)
6756 .expect("catalog list succeeds");
6757 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
6758 assert!(
6759 body["modules"][0].get("capabilities").is_none(),
6760 "legacy manifest must retain an absent capabilities field on catalog.list"
6761 );
6762 }
6763
6764 #[test]
6765 fn hello_ack_omits_storage_when_no_storage_config() {
6766 let registry = Arc::new(Registry::default());
6767 let handler = ControlHandler::new(Arc::clone(®istry));
6768 let responses = handler
6769 .handle_control(
6770 ConnectionId::new(1),
6771 hello_frame("aft", PROTOCOL_VERSION, 7),
6772 )
6773 .unwrap();
6774 let ack = parse_ack(&responses[0]);
6775 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
6776 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
6777 }
6778
6779 #[tokio::test]
6780 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
6781 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
6782 let registry = Arc::new(Registry::default());
6783 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
6784 let responses = handler
6785 .handle_control(
6786 ConnectionId::new(1),
6787 hello_frame("aft", PROTOCOL_VERSION, 7),
6788 )
6789 .unwrap();
6790 let ack = parse_ack(&responses[0]);
6791 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
6792
6793 let described = handler
6794 .handle_control_frame(
6795 &route_ctx(ConnectionId::new(2)).0,
6796 Frame::build(
6797 FrameType::Request,
6798 control_flags(),
6799 0,
6800 0,
6801 9,
6802 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
6803 )
6804 .unwrap(),
6805 )
6806 .await
6807 .unwrap();
6808 let ClientControlResponse::ServerDescribe { machine_id, .. } =
6809 serde_json::from_slice(&described[0].body).unwrap()
6810 else {
6811 panic!("server.describe answered with another shape");
6812 };
6813 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
6814 }
6815
6816 #[test]
6817 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
6818 let registry = Arc::new(Registry::default());
6821 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
6822 crate::daemon_config::StorageConfig::Sqlite {
6823 data_home: std::path::PathBuf::from("/data"),
6824 },
6825 ));
6826
6827 let responses = handler
6828 .handle_control(
6829 ConnectionId::new(1),
6830 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
6831 )
6832 .unwrap();
6833 let ack = parse_ack(&responses[0]);
6834 assert_eq!(
6835 ack.storage,
6836 Some(serde_json::json!({
6837 "module_id": "alfonso-routing",
6838 "storage_namespace": "default",
6839 "isolation": { "kind": "module" },
6840 "backend": {
6841 "backend": "sqlite",
6842 "path": "/data/cortexkit/alfonso-routing/store.db"
6843 }
6844 })),
6845 "the delivered descriptor is the module's own sqlite store path"
6846 );
6847 }
6848
6849 #[test]
6850 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
6851 let registry = Arc::new(Registry::default());
6852 let handler = ControlHandler::new(Arc::clone(®istry));
6853 let conn = ConnectionId::new(1);
6854 let responses = handler
6855 .handle_control(
6856 conn,
6857 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
6858 )
6859 .unwrap();
6860 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
6861 let registration = registry.get_module("aft").unwrap().unwrap();
6862 assert_eq!(registration.control_ops, module_baseline_control_ops());
6863
6864 let frame =
6865 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
6866 assert!(handler
6867 .guard_module_control_op(&frame, "aft", "route.bind")
6868 .unwrap()
6869 .is_none());
6870 let error = handler
6871 .guard_module_control_op(&frame, "aft", "test.synthetic")
6872 .unwrap()
6873 .expect("synthetic ungranted op should be rejected");
6874 assert_eq!(error.header.ty, FrameType::Error);
6875 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
6876 }
6877
6878 #[test]
6879 fn hello_control_ops_some_adds_optional_grants() {
6880 let registry = Arc::new(Registry::default());
6881 let handler = ControlHandler::new(Arc::clone(®istry));
6882 handler
6883 .handle_control(
6884 ConnectionId::new(1),
6885 hello_frame_with_control_ops(
6886 "aft",
6887 PROTOCOL_VERSION,
6888 7,
6889 Some(vec![
6890 "future.synthetic".to_string(),
6891 "route.bind".to_string(),
6892 ]),
6893 ),
6894 )
6895 .unwrap();
6896 let registration = registry.get_module("aft").unwrap().unwrap();
6897 assert_eq!(
6898 registration.control_ops,
6899 vec![
6900 "route.bind".to_string(),
6901 "route.status".to_string(),
6902 "future.synthetic".to_string(),
6903 ]
6904 );
6905 let frame =
6906 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
6907 assert!(handler
6908 .guard_module_control_op(&frame, "aft", "future.synthetic")
6909 .unwrap()
6910 .is_none());
6911 }
6912
6913 #[tokio::test]
6914 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
6915 let registry = Arc::new(Registry::default());
6916 let forwarding = Arc::new(ForwardingTable::default());
6917 let handler =
6918 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
6919 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
6920 hello_via_sink(
6921 &handler,
6922 &module_ctx,
6923 &mut module_rx,
6924 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
6925 )
6926 .await;
6927
6928 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
6929 let responses = handler
6930 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
6931 .await
6932 .unwrap();
6933 assert_eq!(responses.len(), 1);
6934 assert_eq!(responses[0].header.ty, FrameType::Error);
6935 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
6936 assert!(module_rx.try_recv().is_err());
6937 }
6938
6939 #[tokio::test]
6940 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
6941 let registry = Arc::new(Registry::default());
6942 let forwarding = Arc::new(ForwardingTable::default());
6943 let handler =
6944 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
6945 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
6946 hello_via_sink(
6947 &handler,
6948 &module_ctx,
6949 &mut module_rx,
6950 hello_frame_with_control_ops(
6951 "aft",
6952 PROTOCOL_VERSION,
6953 7,
6954 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
6955 ),
6956 )
6957 .await;
6958
6959 let project_root = unique_project_root("demux");
6960 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
6961 let route_handler = handler.clone();
6962 let route_task = tokio::spawn(async move {
6963 route_handler
6964 .handle_control_frame(
6965 &route_client_ctx,
6966 route_open_frame(100, "aft", project_root),
6967 )
6968 .await
6969 .unwrap()
6970 });
6971 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
6972 .await
6973 .unwrap()
6974 .unwrap();
6975 assert!(matches!(
6976 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
6977 ModuleControlRequest::RouteBind { .. }
6978 ));
6979
6980 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
6981 let health_handler = handler.clone();
6982 let health_task = tokio::spawn(async move {
6983 health_handler
6984 .handle_control_frame(
6985 &health_client_ctx,
6986 supervisor_health_probe_frame(101, "aft"),
6987 )
6988 .await
6989 .unwrap()
6990 });
6991 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
6992 .await
6993 .unwrap()
6994 .unwrap();
6995 assert_eq!(
6996 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
6997 ModuleControlRequest::HealthCheck {}
6998 );
6999
7000 handler
7001 .handle_control_frame(
7002 &module_ctx,
7003 health_response(health_frame.header.corr, HealthStatus::Degraded),
7004 )
7005 .await
7006 .unwrap();
7007 let health_response = health_task.await.unwrap();
7008 assert_eq!(health_response.len(), 1);
7009 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
7010 ClientControlResponse::SupervisorHealthProbe {
7011 module_id,
7012 status,
7013 detail,
7014 metrics,
7015 } => {
7016 assert_eq!(module_id, "aft");
7017 assert_eq!(status, HealthStatus::Degraded);
7018 assert_eq!(detail.as_deref(), Some("warming"));
7019 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
7020 }
7021 other => panic!("unexpected health response: {other:?}"),
7022 }
7023
7024 handler
7025 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
7026 .await
7027 .unwrap();
7028 let route_response = route_task.await.unwrap();
7029 assert!(route_response.is_empty());
7030 let published = route_client_rx.recv().await.unwrap();
7031 assert!(matches!(
7032 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
7033 ClientControlResponse::RouteOpen { .. }
7034 ));
7035 }
7036
7037 async fn relay_route_open(
7042 handler: &ControlHandler,
7043 client_connection: ConnectionId,
7044 client_egress: &FrameSink,
7045 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
7046 corr: u64,
7047 module_id: &str,
7048 project_root_label: &str,
7049 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
7050 let ctx = RouteCtx {
7051 connection_id: client_connection,
7052 egress: client_egress.clone(),
7053 };
7054 let handler = handler.clone();
7055 let project_root = unique_project_root(project_root_label);
7056 let module_id = module_id.to_string();
7057 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
7058 let task = tokio::spawn(async move {
7059 let _guard = tracing::dispatcher::set_default(&dispatch);
7060 handler
7061 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
7062 .await
7063 .unwrap()
7064 });
7065 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
7066 .await
7067 .expect("module receives the relayed route.bind")
7068 .expect("module egress is open");
7069 (task, bind.frame)
7070 }
7071
7072 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
7073 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
7074 ModuleControlRequest::RouteBind {
7075 route_channel,
7076 epoch,
7077 ..
7078 } => (route_channel, epoch),
7079 other => panic!("expected a route.bind request, got {other:?}"),
7080 }
7081 }
7082
7083 fn published_route(frame: &Frame) -> (u16, u32) {
7084 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
7085 ClientControlResponse::RouteOpen {
7086 route_channel,
7087 route_epoch,
7088 } => (route_channel, route_epoch),
7089 other => panic!("expected a route.open response, got {other:?}"),
7090 }
7091 }
7092
7093 #[tokio::test]
7113 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
7114 let registry = Arc::new(Registry::default());
7115 let forwarding = Arc::new(ForwardingTable::default());
7116 let handler =
7117 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
7118
7119 let module_connection = ConnectionId::new(30);
7120 let (module_ctx, mut module_rx) = route_ctx(module_connection);
7121 hello_via_sink(
7122 &handler,
7123 &module_ctx,
7124 &mut module_rx,
7125 hello_frame("aft", PROTOCOL_VERSION, 7),
7126 )
7127 .await;
7128
7129 let dying_client = ConnectionId::new(31);
7130 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
7131
7132 let (first_task, first_bind) = relay_route_open(
7135 &handler,
7136 dying_client,
7137 &dying_ctx.egress,
7138 &mut module_rx,
7139 100,
7140 "aft",
7141 "closing-first",
7142 )
7143 .await;
7144 handler
7145 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
7146 .await
7147 .unwrap();
7148 assert!(first_task.await.unwrap().is_empty());
7149 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
7150
7151 let (second_task, second_bind) = relay_route_open(
7153 &handler,
7154 dying_client,
7155 &dying_ctx.egress,
7156 &mut module_rx,
7157 101,
7158 "aft",
7159 "closing-second",
7160 )
7161 .await;
7162 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
7163
7164 assert!(forwarding
7167 .escalate_client_delivery_failure(
7168 dying_client,
7169 first_channel,
7170 first_epoch,
7171 CloseReason::new(
7172 "module_to_client_delivery_failed",
7173 "client egress refused a module frame",
7174 ),
7175 crate::forwarding::UndeliveredFrame {
7176 module_id: None,
7177 sink: &dying_ctx.egress,
7178 },
7179 )
7180 .unwrap());
7181 assert!(!dying_ctx.egress.is_closed());
7182
7183 let ack = handler
7185 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
7186 .await;
7187 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
7188 if module_loop_error.is_some() {
7189 handler.cleanup_connection(module_connection).unwrap();
7193 }
7194 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7198 .await
7199 .ok()
7200 .flatten();
7201
7202 assert!(
7204 registry
7205 .get_module_by_connection(module_connection)
7206 .unwrap()
7207 .is_some(),
7208 "one client's closing connection ended the shared module connection: \
7209 {module_loop_error:?}"
7210 );
7211 let cotenant = ConnectionId::new(32);
7213 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
7214 let (cotenant_task, cotenant_bind) = relay_route_open(
7215 &handler,
7216 cotenant,
7217 &cotenant_ctx.egress,
7218 &mut module_rx,
7219 102,
7220 "aft",
7221 "closing-cotenant",
7222 )
7223 .await;
7224 handler
7225 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
7226 .await
7227 .unwrap();
7228 assert!(cotenant_task.await.unwrap().is_empty());
7229 let (cotenant_channel, cotenant_epoch) =
7230 published_route(&cotenant_rx.recv().await.unwrap());
7231 assert!(matches!(
7232 forwarding
7233 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
7234 .unwrap(),
7235 DataRoute::Client(DataRouteState::Bound(_))
7236 ));
7237
7238 let goodbye = post_ack_module_frame
7241 .expect("module receives a GOODBYE for the abandoned route channel");
7242 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
7243 assert_eq!(goodbye.header.channel, abandoned_channel);
7244 assert_eq!(goodbye.header.epoch, abandoned_epoch);
7245
7246 assert!(dying_rx.try_recv().is_err());
7250 let second_response = second_task.await.unwrap();
7251 assert_eq!(second_response.len(), 1);
7252 assert_eq!(
7253 parse_error(&second_response[0])["code"],
7254 "target_unavailable"
7255 );
7256 }
7257
7258 #[test]
7265 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
7266 let handler = ControlHandler::default();
7267 let module_connection = ConnectionId::new(30);
7268 let client_connection = ConnectionId::new(31);
7269
7270 handler
7271 .refuse_to_end_module_connection_for_a_client(
7272 module_connection,
7273 77,
7274 ForwardingError::ConnectionClosing {
7275 connection_id: client_connection,
7276 },
7277 )
7278 .expect("a closing client must never end the module connection");
7279
7280 assert!(matches!(
7281 handler.refuse_to_end_module_connection_for_a_client(
7282 module_connection,
7283 78,
7284 ForwardingError::ConnectionClosing {
7285 connection_id: module_connection,
7286 },
7287 ),
7288 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
7289 connection_id
7290 })) if connection_id == module_connection
7291 ));
7292 assert!(matches!(
7293 handler.refuse_to_end_module_connection_for_a_client(
7294 module_connection,
7295 79,
7296 ForwardingError::Poisoned,
7297 ),
7298 Err(RouterError::Forwarding(ForwardingError::Poisoned))
7299 ));
7300 assert!(matches!(
7301 handler.refuse_to_end_module_connection_for_a_client(
7302 module_connection,
7303 80,
7304 ForwardingError::StaleModuleEndpoint,
7305 ),
7306 Err(RouterError::Forwarding(
7307 ForwardingError::StaleModuleEndpoint
7308 ))
7309 ));
7310 }
7311
7312 #[tokio::test]
7325 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
7326 let registry = Arc::new(Registry::default());
7327 let forwarding = Arc::new(ForwardingTable::default());
7328 let supervisor = SupervisorHandle::new();
7329 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7330 let handler =
7331 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
7332 .with_supervisor(supervisor);
7333
7334 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
7335 hello_via_sink(
7336 &handler,
7337 &target_ctx,
7338 &mut target_rx,
7339 hello_frame("target", PROTOCOL_VERSION, 1),
7340 )
7341 .await;
7342
7343 let wrong_nonce = handler
7347 .handle_control_frame(
7348 &route_ctx(ConnectionId::new(91)).0,
7349 route_open_frame_with_admission_facts(
7350 20,
7351 "target",
7352 unique_project_root("admission-facts"),
7353 Some(subc_control::ConsumerIdentity {
7354 module_id: "fed".to_string(),
7355 launch_nonce: "not-the-real-nonce".to_string(),
7356 }),
7357 None,
7358 ),
7359 )
7360 .await
7361 .unwrap();
7362 assert_eq!(
7363 parse_error(&wrong_nonce[0])["code"],
7364 "bad_consumer_identity",
7365 "a mismatched launch nonce must be refused, not stamped Reserved"
7366 );
7367
7368 let never_spawned = handler
7372 .handle_control_frame(
7373 &route_ctx(ConnectionId::new(92)).0,
7374 route_open_frame_with_admission_facts(
7375 21,
7376 "target",
7377 unique_project_root("admission-facts"),
7378 Some(subc_control::ConsumerIdentity {
7379 module_id: "never-spawned".to_string(),
7380 launch_nonce: "any-nonce".to_string(),
7381 }),
7382 None,
7383 ),
7384 )
7385 .await
7386 .unwrap();
7387 assert_eq!(
7388 parse_error(&never_spawned[0])["code"],
7389 "bad_consumer_identity",
7390 "an unspawned module_id must be refused rather than accepted for lack of a record"
7391 );
7392 }
7393
7394 #[tokio::test]
7417 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
7418 let registry = Arc::new(Registry::default());
7419 let forwarding = Arc::new(ForwardingTable::default());
7420 let supervisor = SupervisorHandle::new();
7421 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7422 let handler =
7423 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
7424 .with_supervisor(supervisor);
7425
7426 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
7427 hello_via_sink(
7428 &handler,
7429 &target_ctx,
7430 &mut target_rx,
7431 hello_frame("target", PROTOCOL_VERSION, 1),
7432 )
7433 .await;
7434
7435 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
7436 let route_handler = handler.clone();
7437 let route_task = tokio::spawn(async move {
7438 route_handler
7439 .handle_control_frame(
7440 &client_ctx,
7441 route_open_frame_with_admission_facts(
7442 30,
7443 "target",
7444 unique_project_root("admission-facts"),
7445 Some(subc_control::ConsumerIdentity {
7446 module_id: "fed".to_string(),
7447 launch_nonce: "fed-nonce".to_string(),
7448 }),
7449 None,
7450 ),
7451 )
7452 .await
7453 .unwrap()
7454 });
7455
7456 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7463 .await
7464 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
7465 .expect("module control channel closed before route.bind");
7466 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
7467 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
7468 panic!("expected route.bind")
7469 };
7470 assert_eq!(
7471 principal,
7472 Some(Principal::Reserved {
7473 module_id: "fed".to_string()
7474 }),
7475 "a correctly attested consumer must be stamped Reserved for its own id"
7476 );
7477
7478 handler
7479 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
7480 .await
7481 .unwrap();
7482 assert!(route_task.await.unwrap().is_empty());
7483 assert!(
7484 matches!(
7485 serde_json::from_slice::<ClientControlResponse>(
7486 &client_rx.recv().await.unwrap().body
7487 )
7488 .unwrap(),
7489 ClientControlResponse::RouteOpen { .. }
7490 ),
7491 "the route must actually open, not merely avoid an error"
7492 );
7493 }
7494
7495 #[tokio::test(start_paused = true)]
7496 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
7497 let registry = Arc::new(Registry::default());
7498 let forwarding = Arc::new(ForwardingTable::default());
7499 let supervisor = SupervisorHandle::new();
7500 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7501 let handler =
7502 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
7503 .with_supervisor(supervisor);
7504
7505 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
7506 hello_via_sink(
7507 &handler,
7508 &target_ctx,
7509 &mut target_rx,
7510 hello_frame("target", PROTOCOL_VERSION, 1),
7511 )
7512 .await;
7513
7514 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
7515 let direct_handler = handler.clone();
7516 let direct_open = tokio::spawn(async move {
7517 direct_handler
7518 .handle_control_frame(
7519 &direct_ctx,
7520 route_open_frame(2, "target", unique_project_root("route-census-direct")),
7521 )
7522 .await
7523 .unwrap()
7524 });
7525 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7526 .await
7527 .expect("no direct route.bind within 5s")
7528 .expect("target control channel closed before direct route.bind");
7529 handler
7530 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
7531 .await
7532 .unwrap();
7533 assert!(direct_open.await.unwrap().is_empty());
7534 let _ = direct_rx.recv().await.unwrap();
7535
7536 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
7537 let reserved_handler = handler.clone();
7538 let reserved_open = tokio::spawn(async move {
7539 reserved_handler
7540 .handle_control_frame(
7541 &reserved_ctx,
7542 route_open_frame_with_admission_facts(
7543 3,
7544 "target",
7545 unique_project_root("admission-facts"),
7546 Some(ConsumerIdentity {
7547 module_id: "fed".to_string(),
7548 launch_nonce: "fed-nonce".to_string(),
7549 }),
7550 None,
7551 ),
7552 )
7553 .await
7554 .unwrap()
7555 });
7556 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7557 .await
7558 .expect("no reserved route.bind within 5s")
7559 .expect("target control channel closed before reserved route.bind");
7560 handler
7561 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
7562 .await
7563 .unwrap();
7564 assert!(reserved_open.await.unwrap().is_empty());
7565 let _ = reserved_rx.recv().await.unwrap();
7566
7567 forwarding
7568 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
7569 .unwrap();
7570 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
7571 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
7572 module_id: Some("target".to_string()),
7573 })
7574 .unwrap();
7575 let census_frame =
7576 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
7577 let response = handler
7578 .handle_control_frame(&census_ctx, census_frame)
7579 .await
7580 .unwrap()
7581 .pop()
7582 .unwrap();
7583 let actual: Value = serde_json::from_slice(&response.body).unwrap();
7584 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
7585 assert!(matches!(
7586 decoded,
7587 ClientControlResponse::SupervisorRoutes { .. }
7588 ));
7589 let routes = actual["modules"][0]["routes"].as_array().unwrap();
7590 assert_eq!(routes.len(), 2);
7591 assert!(routes.iter().all(|route| route["draining"] == true));
7592 assert!(
7595 routes.iter().all(|route| route["drain_reason"] == "reload"),
7596 "draining routes must name the drain's reason: {routes:?}"
7597 );
7598 assert!(routes.iter().any(|route| {
7599 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
7600 }));
7601 assert!(routes.iter().any(|route| {
7602 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
7603 }));
7604
7605 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
7606 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
7607 if std::env::var_os("UPDATE_GOLDEN").is_some() {
7608 std::fs::write(
7609 &golden_path,
7610 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
7611 )
7612 .unwrap();
7613 }
7614 let expected: Value =
7615 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
7616 assert_eq!(actual, expected);
7617 }
7618
7619 async fn query_live_roots(
7620 handler: &ControlHandler,
7621 module_ctx: &RouteCtx,
7622 ) -> ModuleControlResponseToModule {
7623 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
7624 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
7625 let response = handler
7626 .handle_control_frame(module_ctx, frame)
7627 .await
7628 .unwrap()
7629 .pop()
7630 .unwrap();
7631 serde_json::from_slice(&response.body).unwrap()
7632 }
7633
7634 #[tokio::test(start_paused = true)]
7635 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
7636 let registry = Arc::new(Registry::default());
7637 let forwarding = Arc::new(ForwardingTable::default());
7638 let handler = ControlHandler::with_forwarding(registry, forwarding);
7639 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
7640 hello_via_sink(
7641 &handler,
7642 &target_ctx,
7643 &mut target_rx,
7644 hello_frame("target", PROTOCOL_VERSION, 1),
7645 )
7646 .await;
7647 let root = unique_project_root("live-roots-known");
7648 let path = ProjectRootId::from_path_allowing_missing(root.path())
7649 .unwrap()
7650 .as_path()
7651 .to_path_buf();
7652 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
7653 let open_handler = handler.clone();
7654 let opened = tokio::spawn(async move {
7655 open_handler
7656 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
7657 .await
7658 .unwrap()
7659 });
7660 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7661 .await
7662 .unwrap()
7663 .unwrap();
7664 handler
7665 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
7666 .await
7667 .unwrap();
7668 assert!(opened.await.unwrap().is_empty());
7669 let _ = client_rx.recv().await.unwrap();
7670
7671 let root = unique_project_root("live-roots-pending");
7672 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
7673 .unwrap()
7674 .as_path()
7675 .to_path_buf();
7676 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
7677 let open_handler = handler.clone();
7678 let pending = tokio::spawn(async move {
7679 open_handler
7680 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
7681 .await
7682 .unwrap()
7683 });
7684 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7685 .await
7686 .unwrap()
7687 .unwrap();
7688 let actual = query_live_roots(&handler, &target_ctx).await;
7689 let ModuleControlResponseToModule::LiveRoots {
7690 roots,
7691 unknown_root_bindings,
7692 total_bindings,
7693 } = actual
7694 else {
7695 panic!("expected live roots")
7696 };
7697 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
7698 assert_eq!(unknown_root_bindings, 0);
7699 assert_eq!(
7700 roots.len(),
7701 2,
7702 "root-known arm must retain each canonical root"
7703 );
7704 assert_eq!(
7705 total_bindings,
7706 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
7707 );
7708 let counts = roots
7709 .iter()
7710 .map(|root| (root.project_root.clone(), root.bound, root.pending))
7711 .collect::<Vec<_>>();
7712 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
7713 expected.sort_by(|a, b| a.0.cmp(&b.0));
7714 assert_eq!(
7715 counts, expected,
7716 "roots must sort by path and count pending separately"
7717 );
7718 handler
7719 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
7720 .await
7721 .unwrap();
7722 assert!(pending.await.unwrap().is_empty());
7723 }
7724
7725 #[tokio::test(start_paused = true)]
7726 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
7727 let forwarding = Arc::new(ForwardingTable::default());
7728 let handler =
7729 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
7730 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
7731 hello_via_sink(
7732 &handler,
7733 &target_ctx,
7734 &mut target_rx,
7735 hello_frame("target", PROTOCOL_VERSION, 1),
7736 )
7737 .await;
7738 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
7739 let pending = forwarding
7740 .begin_route_bind_relay_for_test(
7741 client_ctx.connection_id,
7742 client_ctx.egress.clone(),
7743 2,
7744 "target",
7745 )
7746 .unwrap();
7747 forwarding
7748 .complete_pending_relay(
7749 target_ctx.connection_id,
7750 pending.corr,
7751 RouteBindRelayOutcome::Accepted,
7752 )
7753 .unwrap();
7754 let actual = query_live_roots(&handler, &target_ctx).await;
7755 let ModuleControlResponseToModule::LiveRoots {
7756 roots,
7757 unknown_root_bindings,
7758 total_bindings,
7759 } = actual
7760 else {
7761 panic!("expected live roots")
7762 };
7763 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
7764 assert_eq!(
7765 unknown_root_bindings, 1,
7766 "unknown-root arm must not read as no bindings"
7767 );
7768 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
7769 assert_eq!(
7770 total_bindings,
7771 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
7772 );
7773 }
7774
7775 #[tokio::test(start_paused = true)]
7783 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
7784 let forwarding = Arc::new(ForwardingTable::default());
7785 let handler =
7786 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
7787 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
7788 let replies = handler
7789 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
7790 .await
7791 .unwrap();
7792 let queued_by_hello = module_rx.len();
7793
7794 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
7795 let open_handler = handler.clone();
7796 let open = tokio::spawn(async move {
7797 open_handler
7798 .handle_control_frame(
7799 &client_ctx,
7800 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
7801 )
7802 .await
7803 .unwrap()
7804 });
7805 let mut spins = 0;
7807 while module_rx.len() == queued_by_hello {
7808 spins += 1;
7809 assert!(spins < 10_000, "route.open never queued a route.bind");
7810 tokio::task::yield_now().await;
7811 }
7812
7813 for reply in replies {
7815 module_ctx.egress.send(reply).await.unwrap();
7816 }
7817
7818 let first = module_rx.recv().await.unwrap().frame;
7819 assert_eq!(
7820 first.header.ty,
7821 FrameType::HelloAck,
7822 "the first frame a registering module reads must be its HELLO_ACK"
7823 );
7824 assert_eq!(first.header.corr, 7);
7825 let second = module_rx.recv().await.unwrap().frame;
7826 assert_eq!(second.header.ty, FrameType::Request);
7827 assert!(
7828 matches!(
7829 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
7830 ModuleControlRequest::RouteBind { .. }
7831 ),
7832 "the route.bind follows the ack"
7833 );
7834 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
7835
7836 handler
7837 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
7838 .await
7839 .unwrap();
7840 assert!(open.await.unwrap().is_empty());
7841 let _ = client_rx.recv().await.unwrap();
7842 }
7843
7844 #[tokio::test(start_paused = true)]
7845 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
7846 let handler = ControlHandler::with_forwarding(
7847 Arc::new(Registry::default()),
7848 Arc::new(ForwardingTable::default()),
7849 );
7850 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
7851 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
7852 hello_via_sink(
7853 &handler,
7854 &first_ctx,
7855 &mut first_rx,
7856 hello_frame("first", PROTOCOL_VERSION, 1),
7857 )
7858 .await;
7859 hello_via_sink(
7860 &handler,
7861 &second_ctx,
7862 &mut second_rx,
7863 hello_frame("second", PROTOCOL_VERSION, 2),
7864 )
7865 .await;
7866 let root = unique_project_root("second-only");
7867 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
7868 let cloned = handler.clone();
7869 let open = tokio::spawn(async move {
7870 cloned
7871 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
7872 .await
7873 .unwrap()
7874 });
7875 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
7876 .await
7877 .unwrap()
7878 .unwrap();
7879 let first = query_live_roots(&handler, &first_ctx).await;
7880 let second = query_live_roots(&handler, &second_ctx).await;
7881 assert!(
7882 matches!(
7883 first,
7884 ModuleControlResponseToModule::LiveRoots {
7885 total_bindings: 0,
7886 ..
7887 }
7888 ),
7889 "cross-module scope must not expose another module's roots"
7890 );
7891 assert!(
7892 matches!(
7893 second,
7894 ModuleControlResponseToModule::LiveRoots {
7895 total_bindings: 1,
7896 ..
7897 }
7898 ),
7899 "second module must see its pending route"
7900 );
7901 handler
7902 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
7903 .await
7904 .unwrap();
7905 assert!(open.await.unwrap().is_empty());
7906 }
7907
7908 #[tokio::test(start_paused = true)]
7909 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
7910 let handler = ControlHandler::with_forwarding(
7911 Arc::new(Registry::default()),
7912 Arc::new(ForwardingTable::default()),
7913 );
7914 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
7915 hello_via_sink(
7916 &handler,
7917 &target_ctx,
7918 &mut target_rx,
7919 hello_frame("target", PROTOCOL_VERSION, 1),
7920 )
7921 .await;
7922 let actual = query_live_roots(&handler, &target_ctx).await;
7923 let ModuleControlResponseToModule::LiveRoots {
7924 roots,
7925 unknown_root_bindings,
7926 total_bindings,
7927 } = actual
7928 else {
7929 panic!("expected live roots")
7930 };
7931 assert!(roots.is_empty());
7932 assert_eq!(unknown_root_bindings, 0);
7933 assert_eq!(total_bindings, 0);
7934 assert_eq!(
7935 total_bindings,
7936 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
7937 );
7938 }
7939
7940 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
7947 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
7948 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
7949 let text = std::fs::read_to_string(&path)
7950 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
7951 let vectors: Vec<(String, Value)> = text
7952 .lines()
7953 .filter(|line| !line.trim().is_empty())
7954 .map(|line| {
7955 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
7956 let id = entry["corpus_id"]
7957 .as_str()
7958 .expect("every vector carries a corpus_id")
7959 .to_string();
7960 (id, entry["package"].clone())
7961 })
7962 .collect();
7963 assert_eq!(
7967 vectors.len(),
7968 3,
7969 "vendored fed corpus changed size; re-sync from subc-federation"
7970 );
7971
7972 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
7987 let richest = vectors
7988 .iter()
7989 .filter_map(|(_, package)| package.as_object())
7990 .map(|object| {
7991 object
7992 .keys()
7993 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
7994 .count()
7995 })
7996 .max()
7997 .unwrap_or(0);
7998 assert!(
7999 richest >= 2,
8000 "vendored corpus no longer carries a package with unmodelled fields, \
8001 so the relay test can no longer distinguish a verbatim relay from a lossy one"
8002 );
8003
8004 vectors
8005 }
8006
8007 #[tokio::test]
8015 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
8016 for (corpus_id, package) in fed_admission_facts_vectors() {
8017 let registry = Arc::new(Registry::default());
8018 let forwarding = Arc::new(ForwardingTable::default());
8019 let supervisor = SupervisorHandle::new();
8020 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8021 let handler =
8022 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8023 .with_supervisor(supervisor)
8024 .with_admission_facts_config(
8025 Some("fed".to_string()),
8026 Some(vec!["target".to_string()]),
8027 );
8028
8029 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
8030 hello_via_sink(
8031 &handler,
8032 &target_ctx,
8033 &mut target_rx,
8034 hello_frame("target", PROTOCOL_VERSION, 1),
8035 )
8036 .await;
8037
8038 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
8039 let route_handler = handler.clone();
8040 let expected = package.clone();
8041 let route_task = tokio::spawn(async move {
8042 route_handler
8043 .handle_control_frame(
8044 &client_ctx,
8045 route_open_frame_with_admission_facts(
8046 20,
8047 "target",
8048 unique_project_root("admission-facts"),
8049 Some(subc_control::ConsumerIdentity {
8050 module_id: "fed".to_string(),
8051 launch_nonce: "fed-nonce".to_string(),
8052 }),
8053 Some(package),
8054 ),
8055 )
8056 .await
8057 .unwrap()
8058 });
8059
8060 let bind_frame = target_rx.recv().await.unwrap();
8061 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8062 let ModuleControlRequest::RouteBind {
8063 admission_facts, ..
8064 } = bind
8065 else {
8066 panic!("{corpus_id}: expected route.bind")
8067 };
8068 assert_eq!(
8069 admission_facts,
8070 Some(expected),
8071 "{corpus_id}: relay must not add, drop or reshape any field"
8072 );
8073
8074 handler
8075 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8076 .await
8077 .unwrap();
8078 route_task.await.unwrap();
8079 }
8080 }
8081
8082 #[tokio::test]
8083 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
8084 let registry = Arc::new(Registry::default());
8085 let forwarding = Arc::new(ForwardingTable::default());
8086 let supervisor = SupervisorHandle::new();
8087 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8088 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
8089 let handler =
8090 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8091 .with_supervisor(supervisor)
8092 .with_admission_facts_config(
8093 Some("fed".to_string()),
8094 Some(vec!["target".to_string()]),
8095 );
8096
8097 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
8098 hello_via_sink(
8099 &handler,
8100 &target_ctx,
8101 &mut target_rx,
8102 hello_frame("target", PROTOCOL_VERSION, 1),
8103 )
8104 .await;
8105 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
8106 hello_via_sink(
8107 &handler,
8108 &other_ctx,
8109 &mut other_rx,
8110 hello_frame("other", PROTOCOL_VERSION, 2),
8111 )
8112 .await;
8113
8114 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
8115 let expected_facts = facts.clone();
8116 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
8117 let route_handler = handler.clone();
8118 let route_task = tokio::spawn(async move {
8119 route_handler
8120 .handle_control_frame(
8121 &client_ctx,
8122 route_open_frame_with_admission_facts(
8123 10,
8124 "target",
8125 unique_project_root("admission-facts"),
8126 Some(subc_control::ConsumerIdentity {
8127 module_id: "fed".to_string(),
8128 launch_nonce: "fed-nonce".to_string(),
8129 }),
8130 Some(facts.clone()),
8131 ),
8132 )
8133 .await
8134 .unwrap()
8135 });
8136 let bind_frame = target_rx.recv().await.unwrap();
8137 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8138 let ModuleControlRequest::RouteBind {
8139 admission_facts, ..
8140 } = bind
8141 else {
8142 panic!("expected route.bind")
8143 };
8144 assert_eq!(admission_facts, Some(expected_facts));
8145 handler
8146 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8147 .await
8148 .unwrap();
8149 assert!(route_task.await.unwrap().is_empty());
8150 assert!(matches!(
8151 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
8152 .unwrap(),
8153 ClientControlResponse::RouteOpen { .. }
8154 ));
8155
8156 let direct = handler
8157 .handle_control_frame(
8158 &route_ctx(ConnectionId::new(73)).0,
8159 route_open_frame_with_admission_facts(
8160 11,
8161 "target",
8162 unique_project_root("admission-facts"),
8163 None,
8164 Some(json!({"x": 1})),
8165 ),
8166 )
8167 .await
8168 .unwrap();
8169 assert_eq!(
8170 parse_error(&direct[0])["code"],
8171 "admission_facts_not_permitted"
8172 );
8173
8174 let different_reserved = handler
8175 .handle_control_frame(
8176 &route_ctx(ConnectionId::new(77)).0,
8177 route_open_frame_with_admission_facts(
8178 15,
8179 "target",
8180 unique_project_root("admission-facts"),
8181 Some(subc_control::ConsumerIdentity {
8182 module_id: "other".to_string(),
8183 launch_nonce: "other-nonce".to_string(),
8184 }),
8185 Some(json!({"x": 1})),
8186 ),
8187 )
8188 .await
8189 .unwrap();
8190 assert_eq!(
8191 parse_error(&different_reserved[0])["code"],
8192 "admission_facts_not_permitted"
8193 );
8194
8195 let other_target = handler
8196 .handle_control_frame(
8197 &route_ctx(ConnectionId::new(74)).0,
8198 route_open_frame_with_admission_facts(
8199 12,
8200 "other",
8201 unique_project_root("admission-facts"),
8202 Some(subc_control::ConsumerIdentity {
8203 module_id: "fed".to_string(),
8204 launch_nonce: "fed-nonce".to_string(),
8205 }),
8206 Some(json!({"x": 1})),
8207 ),
8208 )
8209 .await
8210 .unwrap();
8211 assert_eq!(
8212 parse_error(&other_target[0])["code"],
8213 "admission_facts_target_not_allowed"
8214 );
8215
8216 let nonexistent = handler
8217 .handle_control_frame(
8218 &route_ctx(ConnectionId::new(75)).0,
8219 route_open_frame_with_admission_facts(
8220 13,
8221 "missing",
8222 unique_project_root("admission-facts"),
8223 None,
8224 Some(json!({"x": 1})),
8225 ),
8226 )
8227 .await
8228 .unwrap();
8229 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
8230
8231 let described = handler
8232 .handle_control_frame(
8233 &route_ctx(ConnectionId::new(76)).0,
8234 Frame::build(
8235 FrameType::Request,
8236 control_flags(),
8237 0,
8238 0,
8239 14,
8240 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
8241 )
8242 .unwrap(),
8243 )
8244 .await
8245 .unwrap();
8246 let ClientControlResponse::ServerDescribe { capabilities, .. } =
8247 serde_json::from_slice(&described[0].body).unwrap()
8248 else {
8249 panic!("expected server.describe response")
8250 };
8251 assert!(capabilities
8252 .iter()
8253 .any(|cap| cap == "admission_facts_relay_v1"));
8254 }
8255
8256 #[tokio::test]
8257 async fn admission_facts_without_configured_carrier_are_rejected() {
8258 let registry = Arc::new(Registry::default());
8259 let forwarding = Arc::new(ForwardingTable::default());
8260 let handler = ControlHandler::with_forwarding(registry, forwarding);
8261 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
8262 hello_via_sink(
8263 &handler,
8264 &target_ctx,
8265 &mut target_rx,
8266 hello_frame("target", PROTOCOL_VERSION, 1),
8267 )
8268 .await;
8269
8270 let responses = handler
8271 .handle_control_frame(
8272 &route_ctx(ConnectionId::new(79)).0,
8273 route_open_frame_with_admission_facts(
8274 16,
8275 "target",
8276 unique_project_root("admission-facts"),
8277 None,
8278 Some(json!({"x": 1})),
8279 ),
8280 )
8281 .await
8282 .unwrap();
8283 assert_eq!(
8284 parse_error(&responses[0])["code"],
8285 "admission_facts_not_permitted"
8286 );
8287 }
8288
8289 #[tokio::test]
8290 async fn route_open_relays_consumer_capabilities_verbatim() {
8291 let registry = Arc::new(Registry::default());
8292 let forwarding = Arc::new(ForwardingTable::default());
8293 let handler =
8294 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8295 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
8296 hello_via_sink(
8297 &handler,
8298 &module_ctx,
8299 &mut module_rx,
8300 hello_frame("aft", PROTOCOL_VERSION, 7),
8301 )
8302 .await;
8303
8304 let expected = vec!["elicitation".to_string(), "roots".to_string()];
8305 let expected_for_request = expected.clone();
8306 let project_root = unique_project_root("consumer-capabilities-present");
8307 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
8308 let route_handler = handler.clone();
8309 let route_task = tokio::spawn(async move {
8310 route_handler
8311 .handle_control_frame(
8312 &client_ctx,
8313 route_open_frame_with_consumer_capabilities(
8314 401,
8315 "aft",
8316 project_root,
8317 Some(expected_for_request),
8318 ),
8319 )
8320 .await
8321 .unwrap()
8322 });
8323 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8324 .await
8325 .unwrap()
8326 .unwrap();
8327 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8328 let ModuleControlRequest::RouteBind {
8329 consumer_capabilities,
8330 ..
8331 } = bind
8332 else {
8333 panic!("expected route.bind request, got {bind:?}");
8334 };
8335 assert_eq!(consumer_capabilities, Some(expected.clone()));
8336
8337 handler
8338 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8339 .await
8340 .unwrap();
8341 let route_response = route_task.await.unwrap();
8342 assert!(route_response.is_empty());
8343 let published = client_rx.recv().await.unwrap();
8344 assert!(matches!(
8345 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8346 ClientControlResponse::RouteOpen { .. }
8347 ));
8348 }
8349
8350 #[tokio::test]
8351 async fn route_open_without_consumer_capabilities_relays_none() {
8352 let registry = Arc::new(Registry::default());
8353 let forwarding = Arc::new(ForwardingTable::default());
8354 let handler =
8355 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8356 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
8357 hello_via_sink(
8358 &handler,
8359 &module_ctx,
8360 &mut module_rx,
8361 hello_frame("aft", PROTOCOL_VERSION, 7),
8362 )
8363 .await;
8364
8365 let project_root = unique_project_root("consumer-capabilities-absent");
8366 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
8367 let route_handler = handler.clone();
8368 let route_task = tokio::spawn(async move {
8369 route_handler
8370 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
8371 .await
8372 .unwrap()
8373 });
8374 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8375 .await
8376 .unwrap()
8377 .unwrap();
8378 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8379 let ModuleControlRequest::RouteBind {
8380 consumer_capabilities,
8381 ..
8382 } = bind
8383 else {
8384 panic!("expected route.bind request, got {bind:?}");
8385 };
8386 assert_eq!(consumer_capabilities, None);
8387
8388 handler
8389 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8390 .await
8391 .unwrap();
8392 let route_response = route_task.await.unwrap();
8393 assert!(route_response.is_empty());
8394 let published = client_rx.recv().await.unwrap();
8395 assert!(matches!(
8396 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8397 ClientControlResponse::RouteOpen { .. }
8398 ));
8399 }
8400
8401 #[tokio::test]
8402 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
8403 let registry = Arc::new(Registry::default());
8404 let forwarding = Arc::new(ForwardingTable::default());
8405 let handler =
8406 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8407 .with_health_probe_timeout(Duration::from_secs(5));
8408 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
8409 hello_via_sink(
8410 &handler,
8411 &module_ctx,
8412 &mut module_rx,
8413 non_routable_hello_frame_with_control_ops(
8414 "mcp",
8415 300,
8416 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
8417 ),
8418 )
8419 .await;
8420 assert!(registry
8421 .get_module("mcp")
8422 .unwrap()
8423 .unwrap()
8424 .manifest
8425 .provides
8426 .is_empty());
8427
8428 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
8429 let route_response = handler
8430 .handle_control_frame(
8431 &route_client_ctx,
8432 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
8433 )
8434 .await
8435 .unwrap();
8436 assert_eq!(route_response[0].header.ty, FrameType::Error);
8437 assert_eq!(
8438 parse_error(&route_response[0])["code"],
8439 "target_unavailable"
8440 );
8441 assert!(parse_error(&route_response[0])["message"]
8442 .as_str()
8443 .unwrap()
8444 .contains("does not provide the requested target"));
8445 assert!(module_rx.try_recv().is_err());
8446
8447 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
8448 let health_handler = handler.clone();
8449 let health_task = tokio::spawn(async move {
8450 health_handler
8451 .handle_control_frame(
8452 &health_client_ctx,
8453 supervisor_health_probe_frame(302, "mcp"),
8454 )
8455 .await
8456 .unwrap()
8457 });
8458 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8459 .await
8460 .unwrap()
8461 .unwrap();
8462 assert_eq!(
8463 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
8464 ModuleControlRequest::HealthCheck {}
8465 );
8466 handler
8467 .handle_control_frame(
8468 &module_ctx,
8469 health_response(health_frame.header.corr, HealthStatus::Ok),
8470 )
8471 .await
8472 .unwrap();
8473 let health_response = health_task.await.unwrap();
8474 assert_eq!(health_response[0].header.ty, FrameType::Response);
8475 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
8476 ClientControlResponse::SupervisorHealthProbe {
8477 module_id, status, ..
8478 } => {
8479 assert_eq!(module_id, "mcp");
8480 assert_eq!(status, HealthStatus::Ok);
8481 }
8482 other => panic!("unexpected health response: {other:?}"),
8483 }
8484
8485 forwarding
8490 .cleanup_connection(module_ctx.connection_id)
8491 .unwrap();
8492 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
8493 let cleanup_response = tokio::time::timeout(
8494 Duration::from_millis(200),
8495 handler.handle_control_frame(
8496 &cleanup_probe_ctx,
8497 supervisor_health_probe_frame(303, "mcp"),
8498 ),
8499 )
8500 .await
8501 .expect("probe should fail immediately when the control lane is gone")
8502 .unwrap();
8503 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
8504 assert_eq!(
8505 parse_error(&cleanup_response[0])["code"],
8506 "target_unavailable"
8507 );
8508 assert!(parse_error(&cleanup_response[0])["message"]
8509 .as_str()
8510 .unwrap()
8511 .contains("no module connection"));
8512
8513 handler
8514 .cleanup_connection(module_ctx.connection_id)
8515 .unwrap();
8516 }
8517
8518 #[tokio::test]
8519 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
8520 let registry = Arc::new(Registry::default());
8521 let supervisor_handle = SupervisorHandle::new();
8522 let supervisor =
8523 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
8524 .with_handle(supervisor_handle.clone())
8525 .with_connection_file_path(
8526 std::env::temp_dir()
8527 .join(format!("subc-route-open-warming-{}", std::process::id())),
8528 );
8529 let module = supervisor
8530 .supervise_configured(
8531 ModuleSpec {
8532 module_id: "warming".to_string(),
8533 program: fake_aft_stub_path(),
8534 args: Vec::new(),
8535 env: Vec::new(),
8536 reserved: false,
8537 reserved_prefixes: Vec::new(),
8538 protocol: ModuleProtocol::Subc,
8539 overlap: Default::default(),
8540 },
8541 true,
8542 )
8543 .unwrap();
8544 assert_eq!(module.state().unwrap(), ModuleState::Running);
8545
8546 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8547 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
8548 let response = handler
8549 .handle_control_frame(
8550 &ctx,
8551 route_open_frame(304, "warming", unique_project_root("warming")),
8552 )
8553 .await
8554 .unwrap();
8555 module.stop().await.unwrap();
8556
8557 assert_eq!(response[0].header.ty, FrameType::Error);
8558 let error = parse_error(&response[0]);
8559 assert_eq!(error["code"], "module_warming");
8560 assert!(error["message"]
8561 .as_str()
8562 .unwrap()
8563 .contains("state=running, enabled=true, live=false"));
8564 }
8565
8566 #[tokio::test]
8571 async fn route_open_refusal_names_the_check_that_refused() {
8572 let handler = ControlHandler::new(Arc::new(Registry::default()));
8573 let capture = EventCapture::default();
8574 let _subscriber =
8575 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8576 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
8577 let response = handler
8578 .handle_control_frame(
8579 &ctx,
8580 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
8581 )
8582 .await
8583 .unwrap();
8584
8585 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
8586 let event = capture
8587 .events()
8588 .into_iter()
8589 .find(|event| {
8590 event.target == "control"
8591 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
8592 })
8593 .expect("route.open refusal event");
8594 assert_eq!(
8595 event.fields.get("reason"),
8596 Some(&"\"not_registered\"".to_string())
8597 );
8598 }
8599
8600 #[tokio::test]
8601 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
8602 let registry = Arc::new(Registry::default());
8603 let supervisor_handle = SupervisorHandle::new();
8604 let supervisor =
8605 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
8606 .with_handle(supervisor_handle.clone())
8607 .with_connection_file_path(std::env::temp_dir().join(format!(
8608 "subc-route-open-refusal-info-{}",
8609 std::process::id()
8610 )));
8611 let module = supervisor
8612 .supervise_configured(
8613 ModuleSpec {
8614 module_id: "warming".to_string(),
8615 program: fake_aft_stub_path(),
8616 args: Vec::new(),
8617 env: Vec::new(),
8618 reserved: false,
8619 reserved_prefixes: Vec::new(),
8620 protocol: ModuleProtocol::Subc,
8621 overlap: Default::default(),
8622 },
8623 true,
8624 )
8625 .unwrap();
8626 assert_eq!(module.state().unwrap(), ModuleState::Running);
8627
8628 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8629 assert!(handler
8630 .counters()
8631 .snapshot()
8632 .get("route_open_refused_by_code")
8633 .is_none());
8634 let capture = EventCapture::default();
8635 let _subscriber =
8636 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8637 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
8638 let response = handler
8639 .handle_control_frame(
8640 &ctx,
8641 route_open_frame(394, "warming", unique_project_root("refusal-info")),
8642 )
8643 .await
8644 .unwrap();
8645 module.stop().await.unwrap();
8646
8647 assert_eq!(parse_error(&response[0])["code"], "module_warming");
8648 let event = capture
8649 .events()
8650 .into_iter()
8651 .find(|event| {
8652 event.target == "control"
8653 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
8654 })
8655 .expect("route.open refusal event");
8656 assert_eq!(
8657 event.fields.get("module_id"),
8658 Some(&"\"warming\"".to_string())
8659 );
8660 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
8661 assert_eq!(
8662 event.fields.get("reason"),
8663 Some(&"\"supervised_not_registered\"".to_string())
8664 );
8665 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
8666 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
8667 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
8668 assert_eq!(
8669 handler.counters().snapshot()["route_open_refused_by_code"],
8670 json!({ "module_warming": 1 })
8671 );
8672 }
8673
8674 #[tokio::test(flavor = "current_thread")]
8675 async fn route_open_unknown_module_escapes_target_module_id() {
8676 let handler = ControlHandler::new(Arc::new(Registry::default()));
8677 let capture = EventCapture::default();
8678 let _subscriber =
8679 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8680 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
8681 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
8682 let response = handler
8683 .handle_control_frame(
8684 &ctx,
8685 route_open_frame(
8686 395,
8687 hostile_module_id,
8688 unique_project_root("hostile-target-module-id"),
8689 ),
8690 )
8691 .await
8692 .unwrap();
8693
8694 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
8695 let event = capture
8696 .events()
8697 .into_iter()
8698 .find(|event| {
8699 event.target == "control"
8700 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
8701 })
8702 .expect("route.open unknown-module refusal event");
8703 let logged = event.fields.get("module_id").expect("module_id field");
8704 assert!(!logged.bytes().any(|byte| byte < 0x20));
8705 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
8706 }
8707
8708 #[tokio::test(flavor = "current_thread")]
8709 async fn route_open_module_rejection_uses_daemon_counter_key() {
8710 let registry = Arc::new(Registry::default());
8711 let forwarding = Arc::new(ForwardingTable::default());
8712 let handler =
8713 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8714 let module_connection = ConnectionId::new(95);
8715 let (module_ctx, mut module_rx) = route_ctx(module_connection);
8716 hello_via_sink(
8717 &handler,
8718 &module_ctx,
8719 &mut module_rx,
8720 hello_frame("aft", PROTOCOL_VERSION, 395),
8721 )
8722 .await;
8723
8724 let client_connection = ConnectionId::new(96);
8725 let (client_ctx, _client_rx) = route_ctx(client_connection);
8726 let capture = EventCapture::default();
8727 let _subscriber =
8728 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8729 let (route_task, bind) = relay_route_open(
8730 &handler,
8731 client_connection,
8732 &client_ctx.egress,
8733 &mut module_rx,
8734 396,
8735 "aft",
8736 "hostile-module-code",
8737 )
8738 .await;
8739 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
8740 let rejection = Frame::build(
8741 FrameType::Error,
8742 control_flags(),
8743 0,
8744 0,
8745 bind.header.corr,
8746 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
8747 )
8748 .unwrap();
8749 handler
8750 .handle_control_frame(&module_ctx, rejection)
8751 .await
8752 .unwrap();
8753
8754 let response = route_task.await.unwrap();
8755 assert_eq!(parse_error(&response[0])["code"], hostile_code);
8756 let counters = handler.counters().snapshot();
8757 assert_eq!(
8758 counters["route_open_refused_by_code"],
8759 json!({ "module_rejected": 1 })
8760 );
8761 assert!(counters["route_open_refused_by_code"]
8762 .get(hostile_code)
8763 .is_none());
8764
8765 let event = capture
8766 .events()
8767 .into_iter()
8768 .find(|event| {
8769 event.target == "control"
8770 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
8771 })
8772 .expect("route.open module-rejection refusal event");
8773 let logged = event.fields.get("module_code").expect("module_code field");
8774 assert!(!logged.bytes().any(|byte| byte < 0x20));
8775 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
8776 }
8777
8778 #[tokio::test]
8779 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
8780 let registry = Arc::new(Registry::default());
8781 let supervisor_handle = SupervisorHandle::new();
8782 let missing_program = std::env::temp_dir().join(format!(
8783 "subc-route-open-missing-program-{}",
8784 std::process::id()
8785 ));
8786 let supervisor =
8787 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
8788 .with_handle(supervisor_handle.clone());
8789 let module = supervisor
8790 .supervise_configured(
8791 ModuleSpec {
8792 module_id: "failed".to_string(),
8793 program: missing_program,
8794 args: Vec::new(),
8795 env: Vec::new(),
8796 reserved: false,
8797 reserved_prefixes: Vec::new(),
8798 protocol: ModuleProtocol::Subc,
8799 overlap: Default::default(),
8800 },
8801 true,
8802 )
8803 .unwrap();
8804 assert_eq!(module.state().unwrap(), ModuleState::Failed);
8805
8806 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8807 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
8808 let response = handler
8809 .handle_control_frame(
8810 &ctx,
8811 route_open_frame(305, "failed", unique_project_root("failed")),
8812 )
8813 .await
8814 .unwrap();
8815
8816 assert_eq!(response[0].header.ty, FrameType::Error);
8817 let error = parse_error(&response[0]);
8818 assert_eq!(error["code"], "target_unavailable");
8819 assert!(error["message"]
8820 .as_str()
8821 .unwrap()
8822 .contains("state=failed, enabled=true, live=false"));
8823 }
8824
8825 #[tokio::test]
8826 async fn route_open_role_mismatch_remains_target_unavailable() {
8827 let registry = Arc::new(Registry::default());
8828 let handler = ControlHandler::new(Arc::clone(®istry));
8829 handler
8830 .handle_control(
8831 ConnectionId::new(41),
8832 non_routable_hello_frame_with_control_ops("health-only", 306, None),
8833 )
8834 .unwrap();
8835
8836 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
8837 let response = handler
8838 .handle_control_frame(
8839 &ctx,
8840 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
8841 )
8842 .await
8843 .unwrap();
8844
8845 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
8846 assert!(parse_error(&response[0])["message"]
8847 .as_str()
8848 .unwrap()
8849 .contains("does not provide the requested target"));
8850 }
8851
8852 #[tokio::test]
8853 async fn route_open_inactive_registration_remains_target_unavailable() {
8854 let registry = Arc::new(Registry::default());
8855 let handler = ControlHandler::new(Arc::clone(®istry));
8856 handler
8857 .handle_control(
8858 ConnectionId::new(43),
8859 hello_frame("inactive", PROTOCOL_VERSION, 308),
8860 )
8861 .unwrap();
8862 assert!(registry
8863 .set_module_state_for_test("inactive", ChannelState::Closed)
8864 .unwrap());
8865
8866 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
8867 let response = handler
8868 .handle_control_frame(
8869 &ctx,
8870 route_open_frame(309, "inactive", unique_project_root("inactive")),
8871 )
8872 .await
8873 .unwrap();
8874
8875 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
8876 assert!(parse_error(&response[0])["message"]
8877 .as_str()
8878 .unwrap()
8879 .contains("is not active"));
8880 }
8881
8882 #[tokio::test]
8883 async fn late_health_reply_is_recorded_through_the_module_response_path() {
8884 let registry = Arc::new(Registry::default());
8885 let forwarding = Arc::new(ForwardingTable::default());
8886 let supervisor_handle = SupervisorHandle::new();
8887 let supervisor = Supervisor::new(Arc::clone(®istry), crate::RestartPolicy::default())
8888 .with_forwarding(Arc::clone(&forwarding))
8889 .with_handle(supervisor_handle.clone());
8890 let module = supervisor
8891 .supervise_configured(
8892 crate::ModuleSpec {
8893 module_id: "late-health-response".to_string(),
8894 program: PathBuf::from("disabled-module"),
8895 args: Vec::new(),
8896 env: Vec::new(),
8897 reserved: false,
8898 reserved_prefixes: Vec::new(),
8899 protocol: ModuleProtocol::Subc,
8900 overlap: Default::default(),
8901 },
8902 false,
8903 )
8904 .unwrap();
8905 let handler =
8906 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8907 .with_supervisor(supervisor_handle);
8908 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
8909 handler
8910 .handle_control_frame(
8911 &module_ctx,
8912 hello_frame_with_control_ops(
8913 "late-health-response",
8914 PROTOCOL_VERSION,
8915 7,
8916 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
8917 ),
8918 )
8919 .await
8920 .unwrap();
8921 let probe_started_at = Instant::now() - Duration::from_millis(80);
8922 let pending = forwarding
8923 .begin_health_probe_rpc_for(
8924 "late-health-response",
8925 MODULE_CONTROL_OP_HEALTH_CHECK,
8926 probe_started_at,
8927 Instant::now() - Duration::from_millis(1),
8928 )
8929 .unwrap();
8930 assert!(forwarding
8931 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
8932 .unwrap());
8933
8934 let responses = handler
8935 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
8936 .await
8937 .unwrap();
8938
8939 assert!(responses.is_empty());
8940 let health = module.status().unwrap().health;
8941 assert_eq!(health.late_answer_count, 1);
8942 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
8943 }
8944
8945 #[tokio::test]
8946 async fn health_probe_timeout_and_module_death_are_typed() {
8947 let registry = Arc::new(Registry::default());
8948 let forwarding = Arc::new(ForwardingTable::default());
8949 let handler =
8950 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8951 .with_health_probe_timeout(Duration::from_millis(50));
8952 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
8953 hello_via_sink(
8954 &handler,
8955 &module_ctx,
8956 &mut module_rx,
8957 hello_frame_with_control_ops(
8958 "aft",
8959 PROTOCOL_VERSION,
8960 7,
8961 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
8962 ),
8963 )
8964 .await;
8965
8966 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
8967 let responses = handler
8968 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
8969 .await
8970 .unwrap();
8971 assert_eq!(responses[0].header.ty, FrameType::Error);
8972 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
8973 let _ = module_rx.try_recv();
8974
8975 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
8976 let health_handler = handler.clone();
8977 let death_task = tokio::spawn(async move {
8978 health_handler
8979 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
8980 .await
8981 .unwrap()
8982 });
8983 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8984 .await
8985 .unwrap()
8986 .unwrap();
8987 handler
8988 .cleanup_connection(module_ctx.connection_id)
8989 .unwrap();
8990 let responses = death_task.await.unwrap();
8991 assert_eq!(responses[0].header.ty, FrameType::Error);
8992 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
8993 }
8994
8995 #[test]
8996 fn hello_requires_exact_protocol_version() {
8997 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
8998 let registry = Arc::new(Registry::default());
8999 let handler = ControlHandler::new(Arc::clone(®istry));
9000 let responses = handler
9001 .handle_control(
9002 ConnectionId::new(connection),
9003 hello_frame("aft", offered, 9),
9004 )
9005 .unwrap();
9006
9007 assert_eq!(responses.len(), 1);
9008 assert_eq!(responses[0].header.ty, FrameType::Error);
9009 let error = parse_error(&responses[0]);
9010 assert_eq!(error["code"], "version_unsupported");
9011 assert!(registry.get_module("aft").unwrap().is_none());
9012 assert_eq!(registry.active_registration_count().unwrap(), 0);
9013 }
9014 }
9015
9016 #[test]
9017 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
9018 let registry = Arc::new(Registry::default());
9019 let forwarding = Arc::new(ForwardingTable::default());
9020 let handler =
9021 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9022 let module_connection = ConnectionId::new(301);
9023 let registration = registry
9024 .register_with_control_ops(
9025 manifest("aft-push", PROTOCOL_VERSION),
9026 PROTOCOL_VERSION,
9027 module_connection,
9028 module_baseline_control_ops(),
9029 )
9030 .unwrap();
9031 let (module_tx, _module_rx) = mpsc::channel(8);
9032 let endpoint = forwarding
9033 .register_module_connection(
9034 module_connection,
9035 "aft-push".to_string(),
9036 PROTOCOL_VERSION,
9037 manifest_concurrency(®istration.manifest),
9038 FrameSink::new(module_tx),
9039 )
9040 .unwrap();
9041
9042 let unknown = Frame::build(
9044 FrameType::Push,
9045 control_flags(),
9046 0,
9047 0,
9048 5,
9049 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
9050 )
9051 .unwrap();
9052 let out = handler.handle_status_update(endpoint, unknown).unwrap();
9053 assert!(
9054 out.is_empty(),
9055 "unknown push op must be ignored, got {out:?}"
9056 );
9057
9058 let malformed = Frame::build(
9060 FrameType::Push,
9061 control_flags(),
9062 0,
9063 0,
9064 6,
9065 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
9066 )
9067 .unwrap();
9068 let out = handler.handle_status_update(endpoint, malformed).unwrap();
9069 assert_eq!(out.len(), 1);
9070 assert_eq!(out[0].header.ty, FrameType::Error);
9071 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
9072 }
9073
9074 #[test]
9075 fn hello_rejected_when_connection_already_owns_client_routes() {
9076 let registry = Arc::new(Registry::default());
9077 let forwarding = Arc::new(ForwardingTable::default());
9078 let handler =
9079 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9080 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
9082 let client_connection = ConnectionId::new(202);
9083
9084 let responses = handler
9087 .handle_control(
9088 client_connection,
9089 hello_frame("aft-second", PROTOCOL_VERSION, 9),
9090 )
9091 .unwrap();
9092 assert_eq!(responses[0].header.ty, FrameType::Error);
9093 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
9094 assert!(registry.get_module("aft-second").unwrap().is_none());
9095 }
9096
9097 #[test]
9098 fn reserved_module_hello_requires_matching_launch_nonce() {
9099 let registry = Arc::new(Registry::default());
9100 let supervisor = SupervisorHandle::new();
9101 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
9104 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9105
9106 let no_nonce = handler
9108 .handle_control(
9109 ConnectionId::new(1),
9110 hello_frame("vault", PROTOCOL_VERSION, 1),
9111 )
9112 .unwrap();
9113 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
9114 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
9115 assert!(registry.get_module("vault").unwrap().is_none());
9116
9117 let wrong = handler
9119 .handle_control(
9120 ConnectionId::new(2),
9121 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
9122 )
9123 .unwrap();
9124 assert_eq!(wrong[0].header.ty, FrameType::Error);
9125 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
9126 assert!(registry.get_module("vault").unwrap().is_none());
9127
9128 let ok = handler
9130 .handle_control(
9131 ConnectionId::new(3),
9132 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
9133 )
9134 .unwrap();
9135 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
9136 assert!(registry.get_module("vault").unwrap().is_some());
9137 }
9138
9139 #[test]
9140 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
9141 let registry = Arc::new(Registry::default());
9142 let supervisor = SupervisorHandle::new();
9143 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
9144 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
9145 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9146
9147 let squat = handler
9148 .handle_control(
9149 ConnectionId::new(1),
9150 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
9151 )
9152 .unwrap();
9153 assert_eq!(squat[0].header.ty, FrameType::Error);
9154 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
9155 assert!(parse_error(&squat[0])["message"]
9156 .as_str()
9157 .unwrap()
9158 .contains("fed:"));
9159
9160 let accepted_peer = handler
9161 .handle_control(
9162 ConnectionId::new(2),
9163 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
9164 )
9165 .unwrap();
9166 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
9167
9168 let accepted_short = handler
9169 .handle_control(
9170 ConnectionId::new(3),
9171 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
9172 )
9173 .unwrap();
9174 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
9175
9176 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
9177 let response = handler
9178 .handle_control(
9179 ConnectionId::new(conn),
9180 hello_frame(module_id, PROTOCOL_VERSION, conn),
9181 )
9182 .unwrap();
9183 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
9184 }
9185 }
9186
9187 #[test]
9188 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
9189 let registry = Arc::new(Registry::default());
9190 let supervisor = SupervisorHandle::new();
9191 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
9192 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
9193 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
9194 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9195
9196 let owner_nonce = handler
9197 .handle_control(
9198 ConnectionId::new(1),
9199 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
9200 )
9201 .unwrap();
9202 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
9203 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
9204 assert!(registry.get_module("fed:special").unwrap().is_none());
9205
9206 let exact_nonce = handler
9207 .handle_control(
9208 ConnectionId::new(2),
9209 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
9210 )
9211 .unwrap();
9212 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
9213 assert!(registry.get_module("fed:special").unwrap().is_some());
9214 }
9215
9216 #[test]
9217 fn non_reserved_module_ignores_launch_nonce() {
9218 let registry = Arc::new(Registry::default());
9219 let handler = ControlHandler::new(Arc::clone(®istry));
9222 let no_nonce = handler
9223 .handle_control(
9224 ConnectionId::new(1),
9225 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
9226 )
9227 .unwrap();
9228 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
9229 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
9230
9231 let echoed_nonce = handler
9232 .handle_control(
9233 ConnectionId::new(2),
9234 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
9235 )
9236 .unwrap();
9237 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
9238 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
9239 }
9240
9241 #[test]
9242 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
9243 let handler = ControlHandler::default();
9244 let conn = ConnectionId::new(1);
9245 let malformed = Frame::build(
9246 FrameType::Hello,
9247 control_flags(),
9248 0,
9249 0,
9250 3,
9251 b"{not json".to_vec(),
9252 )
9253 .unwrap();
9254
9255 let error = handler.handle_control(conn, malformed).unwrap();
9256 assert_eq!(error[0].header.ty, FrameType::Error);
9257 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
9258
9259 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
9260 let pong = handler.handle_control(conn, ping).unwrap();
9261 assert_eq!(pong[0].header.ty, FrameType::Pong);
9262 assert_eq!(pong[0].header.corr, 4);
9263 }
9264
9265 #[test]
9266 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
9267 let registry = Arc::new(Registry::default());
9268 let handler = ControlHandler::new(Arc::clone(®istry));
9269
9270 handler
9271 .handle_control(
9272 ConnectionId::new(1),
9273 hello_frame("aft", PROTOCOL_VERSION, 1),
9274 )
9275 .unwrap();
9276 let duplicate = handler
9277 .handle_control(
9278 ConnectionId::new(2),
9279 hello_frame("aft", PROTOCOL_VERSION, 2),
9280 )
9281 .unwrap();
9282
9283 assert_eq!(duplicate[0].header.ty, FrameType::Error);
9284 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
9285 let registration = registry.get_module("aft").unwrap().unwrap();
9286 assert_eq!(registration.connection_id, ConnectionId::new(1));
9287 }
9288
9289 #[test]
9290 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
9291 let registry = Arc::new(Registry::default());
9292 let forwarding = Arc::new(ForwardingTable::default());
9293 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
9294 let handler =
9295 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9296 .with_process_liveness(process_liveness);
9297 let (ctx, route_channel, route_epoch) =
9298 bind_liveness_route(®istry, &forwarding, "aft-dead");
9299 let responses = handler
9300 .handle_route_poll(
9301 &ctx,
9302 route_poll_frame(41, PollKind::Liveness, route_channel),
9303 route_channel,
9304 route_epoch,
9305 PollKind::Liveness,
9306 )
9307 .unwrap();
9308
9309 assert_eq!(responses.len(), 1);
9310 assert_eq!(responses[0].header.ty, FrameType::Response);
9311 assert_route_poll_liveness(&responses[0], false);
9312 }
9313
9314 #[test]
9315 fn liveness_poll_without_process_source_uses_bound_route() {
9316 let registry = Arc::new(Registry::default());
9317 let forwarding = Arc::new(ForwardingTable::default());
9318 let handler =
9319 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9320 let (ctx, route_channel, route_epoch) =
9321 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
9322 let responses = handler
9323 .handle_route_poll(
9324 &ctx,
9325 route_poll_frame(42, PollKind::Liveness, route_channel),
9326 route_channel,
9327 route_epoch,
9328 PollKind::Liveness,
9329 )
9330 .unwrap();
9331
9332 assert_route_poll_liveness(&responses[0], true);
9333 }
9334
9335 #[test]
9336 fn liveness_poll_untracked_process_source_uses_bound_route() {
9337 let registry = Arc::new(Registry::default());
9338 let forwarding = Arc::new(ForwardingTable::default());
9339 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
9340 let handler =
9341 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9342 .with_process_liveness(process_liveness);
9343 let (ctx, route_channel, route_epoch) =
9344 bind_liveness_route(®istry, &forwarding, "aft-untracked");
9345 let responses = handler
9346 .handle_route_poll(
9347 &ctx,
9348 route_poll_frame(43, PollKind::Liveness, route_channel),
9349 route_channel,
9350 route_epoch,
9351 PollKind::Liveness,
9352 )
9353 .unwrap();
9354
9355 assert_route_poll_liveness(&responses[0], true);
9356 }
9357
9358 #[tokio::test]
9359 async fn unknown_op_returns_unknown_control_op() {
9360 let handler = ControlHandler::default();
9361 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
9362 let request = Frame::build(
9363 FrameType::Request,
9364 control_flags(),
9365 0,
9366 0,
9367 55,
9368 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
9369 )
9370 .unwrap();
9371
9372 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
9373
9374 assert_eq!(response.len(), 1);
9375 assert_eq!(response[0].header.ty, FrameType::Error);
9376 assert_eq!(response[0].header.corr, 55);
9377 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
9378 }
9379
9380 #[tokio::test]
9381 async fn supervisor_provenance_rejects_unknown_exact_module() {
9382 let handler = ControlHandler::default();
9383 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
9384 let request = Frame::build(
9385 FrameType::Request,
9386 control_flags(),
9387 0,
9388 0,
9389 57,
9390 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
9391 )
9392 .unwrap();
9393
9394 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
9395
9396 assert_eq!(response.len(), 1);
9397 assert_eq!(response[0].header.ty, FrameType::Error);
9398 assert_eq!(response[0].header.corr, 57);
9399 let error = parse_error(&response[0]);
9400 assert_eq!(error["code"], "unknown_module");
9401 assert_eq!(error["message"], "module_id 'missing' is not supervised");
9402 }
9403
9404 #[test]
9405 fn provenance_probe_override_keeps_handler_tests_deterministic() {
9406 let expected = subc_control::RunningImageAgreement::Unavailable {
9407 reason: subc_control::RunningImageUnavailableReason::HashFailed,
9408 };
9409 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
9410 assert_eq!(handler.provenance_probe_override, Some(expected));
9411 }
9412
9413 #[test]
9414 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
9415 let verdict = reload_verdict(
9416 std::path::Path::new("/bin/new"),
9417 Some(std::path::Path::new("/bin/old")),
9418 subc_control::RunningImageAgreement::Unavailable {
9419 reason: subc_control::RunningImageUnavailableReason::HashFailed,
9420 },
9421 );
9422 assert!(matches!(
9423 verdict.path,
9424 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
9425 if configured == std::path::Path::new("/bin/new")
9426 && spawned_from == std::path::Path::new("/bin/old")
9427 ));
9428 }
9429
9430 #[test]
9431 fn reload_verdict_detects_replaced_image_at_same_path() {
9432 let image = subc_control::RunningImageAgreement::Mismatch {
9433 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
9434 digest: "old".into(),
9435 },
9436 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
9437 digest: "new".into(),
9438 },
9439 };
9440 let verdict = reload_verdict(
9441 std::path::Path::new("/bin/same"),
9442 Some(std::path::Path::new("/bin/same")),
9443 image.clone(),
9444 );
9445 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
9446 assert_eq!(verdict.image, image);
9447 }
9448
9449 #[test]
9450 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
9451 let image = subc_control::RunningImageAgreement::Unavailable {
9452 reason: subc_control::RunningImageUnavailableReason::NotRunning,
9453 };
9454 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
9455 assert_eq!(
9456 verdict.path,
9457 subc_control::ReloadPathAgreement::Unavailable {
9458 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
9459 }
9460 );
9461 assert_eq!(verdict.image, image);
9462
9463 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
9464 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
9465 };
9466 let verdict = reload_verdict(
9467 std::path::Path::new("/bin/same"),
9468 Some(std::path::Path::new("/bin/same")),
9469 unconfirmed.clone(),
9470 );
9471 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
9472 assert_eq!(verdict.image, unconfirmed);
9473 }
9474
9475 #[test]
9476 fn reload_verdict_preserves_each_image_unavailability_reason() {
9477 use subc_control::RunningImageUnavailableReason as Reason;
9478
9479 for reason in [
9480 Reason::NotRunning,
9481 Reason::UnsupportedPlatform,
9482 Reason::RunningExecutableUnreadable,
9483 Reason::SpawnedPathUnreadable,
9484 Reason::HashFailed,
9485 Reason::ProcessIdentityUnconfirmed,
9486 Reason::Unknown("future_probe_reason".to_string()),
9487 ] {
9488 let image = subc_control::RunningImageAgreement::Unavailable {
9489 reason: reason.clone(),
9490 };
9491 let verdict = reload_verdict(
9492 std::path::Path::new("/bin/same"),
9493 Some(std::path::Path::new("/bin/same")),
9494 image.clone(),
9495 );
9496 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
9497 assert_eq!(verdict.image, image, "{reason:?}");
9498 }
9499 }
9500
9501 #[tokio::test]
9502 async fn malformed_control_bodies_return_invalid_control_body() {
9503 let handler = ControlHandler::default();
9504 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
9505
9506 for (corr, body) in [
9507 (56, br#"{"route_channel":1}"#.as_slice()),
9508 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
9509 (
9510 58,
9511 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
9512 ),
9513 ] {
9514 let request = Frame::build(
9515 FrameType::Request,
9516 control_flags(),
9517 0,
9518 0,
9519 corr,
9520 body.to_vec(),
9521 )
9522 .unwrap();
9523 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
9524
9525 assert_eq!(response.len(), 1);
9526 assert_eq!(response[0].header.ty, FrameType::Error);
9527 assert_eq!(response[0].header.corr, corr);
9528 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
9529 }
9530 }
9531
9532 #[tokio::test]
9533 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
9534 let registry = Arc::new(Registry::default());
9535 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
9536 let router = Router::with_control_handler(Arc::clone(&control));
9537 let connection = router.begin_connection();
9538 let (ctx, mut rx) = route_ctx(connection.id());
9539
9540 router
9541 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
9542 .await
9543 .unwrap();
9544 let response = rx.recv().await.unwrap();
9545 let ack = parse_ack(&response);
9546 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9547 let channel = 1;
9548
9549 let goodbye =
9550 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
9551 router.route_for_connection(&ctx, goodbye).await.unwrap();
9552 assert!(rx.try_recv().is_err());
9553 assert!(registry.get_module("aft").unwrap().is_none());
9554
9555 router
9556 .route_for_connection(&ctx, channel_request(channel, 13))
9557 .await
9558 .unwrap();
9559 let error_frame = rx.recv().await.unwrap();
9560 assert_eq!(error_frame.header.ty, FrameType::Error);
9561 assert_eq!(error_frame.header.channel, channel);
9562 }
9563
9564 #[tokio::test]
9565 async fn dropping_router_connection_releases_registration() {
9566 let registry = Arc::new(Registry::default());
9567 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
9568 let router = Router::with_control_handler(control);
9569 let connection = router.begin_connection();
9570 let (ctx, mut rx) = route_ctx(connection.id());
9571
9572 router
9573 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
9574 .await
9575 .unwrap();
9576 let response = rx.recv().await.unwrap();
9577 let ack = parse_ack(&response);
9578 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9579 assert!(registry.get_module("aft").unwrap().is_some());
9580
9581 drop(connection);
9582
9583 assert!(registry.get_module("aft").unwrap().is_none());
9584 assert_eq!(registry.active_registration_count().unwrap(), 0);
9585 }
9586
9587 fn capability_manifest(
9588 module_id: &str,
9589 provides: &[&str],
9590 must_never_reach: &[&str],
9591 ) -> ModuleManifest {
9592 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
9593 manifest.capabilities = Some(CapabilityDeclarations {
9594 provides: provides
9595 .iter()
9596 .map(|capability| (*capability).to_string())
9597 .collect(),
9598 requires: Vec::new(),
9599 must_never_reach: must_never_reach
9600 .iter()
9601 .map(|capability| (*capability).to_string())
9602 .collect(),
9603 });
9604 manifest
9605 }
9606
9607 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
9608 Frame::build(
9609 FrameType::Hello,
9610 control_flags(),
9611 0,
9612 0,
9613 corr,
9614 serde_json::to_vec(&ModuleHelloBody {
9615 protocol_ver: manifest.protocol_ver,
9616 manifest,
9617 control_ops: None,
9618 launch_nonce: None,
9619 })
9620 .expect("capability test HELLO serializes"),
9621 )
9622 .expect("capability test HELLO frame builds")
9623 }
9624
9625 fn catalog_update_with_capabilities_frame(
9626 corr: u64,
9627 capabilities: CapabilityDeclarations,
9628 ) -> Frame {
9629 Frame::build(
9630 FrameType::Request,
9631 control_flags(),
9632 0,
9633 0,
9634 corr,
9635 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
9636 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
9637 capabilities: Some(capabilities),
9638 ready: None,
9639 })
9640 .expect("capability catalog.update serializes"),
9641 )
9642 .expect("capability catalog.update frame builds")
9643 }
9644
9645 async fn register_capability_manifest(
9646 handler: &ControlHandler,
9647 ctx: &RouteCtx,
9648 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9649 manifest: ModuleManifest,
9650 corr: u64,
9651 ) {
9652 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
9653 }
9654
9655 async fn open_route_for_capability_test(
9656 handler: &ControlHandler,
9657 target_ctx: &RouteCtx,
9658 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9659 client_connection_id: u64,
9660 corr: u64,
9661 target_module_id: &str,
9662 consumer_identity: Option<ConsumerIdentity>,
9663 ) -> (
9664 mpsc::Receiver<crate::router::OutboundFrame>,
9665 ModuleControlRequest,
9666 ) {
9667 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
9668 let route_handler = handler.clone();
9669 let target_module_id = target_module_id.to_string();
9670 let route_task = tokio::spawn(async move {
9671 route_handler
9672 .handle_control_frame(
9673 &client_ctx,
9674 route_open_frame_with_admission_facts(
9675 corr,
9676 &target_module_id,
9677 unique_project_root("admission-facts"),
9678 consumer_identity,
9679 None,
9680 ),
9681 )
9682 .await
9683 .expect("capability test route.open succeeds")
9684 });
9685 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
9686 .await
9687 .expect("capability test route.open must reach route.bind")
9688 .expect("target control receiver stays open");
9689 let bind_request: ModuleControlRequest =
9690 serde_json::from_slice(&bind.body).expect("route.bind decodes");
9691 handler
9692 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
9693 .await
9694 .expect("capability test route.bind ACK succeeds");
9695 assert!(route_task.await.expect("route.open task joins").is_empty());
9696 let opened = client_rx
9697 .recv()
9698 .await
9699 .expect("successful route.open publishes a response");
9700 assert!(matches!(
9701 serde_json::from_slice::<ClientControlResponse>(&opened.body),
9702 Ok(ClientControlResponse::RouteOpen { .. })
9703 ));
9704 (client_rx, bind_request)
9705 }
9706
9707 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
9708 assert_eq!(frame.header.ty, FrameType::Push);
9709 assert_eq!(frame.header.channel, 0);
9710 assert_eq!(
9711 serde_json::from_slice::<ClientControlPush>(&frame.body)
9712 .expect("route.closed control push decodes"),
9713 ClientControlPush::RouteClosed {
9714 module_id: target_module_id.to_string(),
9715 reason: RouteCloseReason::CapabilityDenied,
9716 drained: false,
9717 abandoned: 0,
9718 excluded_subscriptions: 0,
9719 terminal: Some(false),
9720 }
9721 );
9722 }
9723
9724 #[tokio::test]
9725 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
9726 let registry = Arc::new(Registry::default());
9727 let forwarding = Arc::new(ForwardingTable::default());
9728 let supervisor = SupervisorHandle::new();
9729 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9730 let handler =
9731 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9732 .with_supervisor(supervisor);
9733 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
9734 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
9735 register_capability_manifest(
9736 &handler,
9737 &target_ctx,
9738 &mut target_rx,
9739 capability_manifest("target", &["credentials-provider/v1"], &[]),
9740 1,
9741 )
9742 .await;
9743 register_capability_manifest(
9744 &handler,
9745 &opener_ctx,
9746 &mut opener_rx,
9747 capability_manifest("opener", &[], &["credentials-provider/v1"]),
9748 2,
9749 )
9750 .await;
9751
9752 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
9753 let replies = handler
9754 .handle_control_frame(
9755 &client_ctx,
9756 route_open_frame_with_admission_facts(
9757 3,
9758 "target",
9759 unique_project_root("admission-facts"),
9760 Some(ConsumerIdentity {
9761 module_id: "opener".to_string(),
9762 launch_nonce: "opener-nonce".to_string(),
9763 }),
9764 None,
9765 ),
9766 )
9767 .await
9768 .expect("denied route.open returns a typed frame");
9769 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
9770 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
9771 assert!(
9772 target_rx.try_recv().is_err(),
9773 "forbidden route.open must not relay route.bind"
9774 );
9775 }
9776
9777 #[tokio::test]
9778 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
9779 let registry = Arc::new(Registry::default());
9780 let forwarding = Arc::new(ForwardingTable::default());
9781 let supervisor = SupervisorHandle::new();
9782 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9783 let handler =
9784 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9785 .with_supervisor(supervisor);
9786 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
9787 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
9788 register_capability_manifest(
9789 &handler,
9790 &target_ctx,
9791 &mut target_rx,
9792 capability_manifest("target", &["credentials-provider/v1"], &[]),
9793 1,
9794 )
9795 .await;
9796 register_capability_manifest(
9797 &handler,
9798 &old_opener_ctx,
9799 &mut old_opener_rx,
9800 capability_manifest("opener", &[], &[]),
9801 2,
9802 )
9803 .await;
9804 let (mut client_rx, _) = open_route_for_capability_test(
9805 &handler,
9806 &target_ctx,
9807 &mut target_rx,
9808 712,
9809 3,
9810 "target",
9811 Some(ConsumerIdentity {
9812 module_id: "opener".to_string(),
9813 launch_nonce: "opener-nonce".to_string(),
9814 }),
9815 )
9816 .await;
9817 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
9818
9819 handler
9820 .cleanup_connection(old_opener_ctx.connection_id)
9821 .expect("old opener registration cleans up");
9822 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
9823 register_capability_manifest(
9824 &handler,
9825 &new_opener_ctx,
9826 &mut new_opener_rx,
9827 capability_manifest("opener", &[], &["credentials-provider/v1"]),
9828 4,
9829 )
9830 .await;
9831
9832 assert_capability_denied_push(
9833 client_rx
9834 .try_recv()
9835 .expect("HELLO deny addition must emit route.closed")
9836 .frame,
9837 "target",
9838 );
9839 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
9840 assert!(matches!(
9841 target_rx.try_recv(),
9842 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
9843 ));
9844 }
9845
9846 #[tokio::test]
9847 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
9848 let registry = Arc::new(Registry::default());
9849 let forwarding = Arc::new(ForwardingTable::default());
9850 let supervisor = SupervisorHandle::new();
9851 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9852 let handler =
9853 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9854 .with_supervisor(supervisor);
9855 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
9856 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
9857 register_capability_manifest(
9858 &handler,
9859 &target_ctx,
9860 &mut target_rx,
9861 capability_manifest("target", &[], &[]),
9862 1,
9863 )
9864 .await;
9865 register_capability_manifest(
9866 &handler,
9867 &opener_ctx,
9868 &mut opener_rx,
9869 capability_manifest("opener", &[], &["credentials-provider/v1"]),
9870 2,
9871 )
9872 .await;
9873 let (mut client_rx, _) = open_route_for_capability_test(
9874 &handler,
9875 &target_ctx,
9876 &mut target_rx,
9877 722,
9878 3,
9879 "target",
9880 Some(ConsumerIdentity {
9881 module_id: "opener".to_string(),
9882 launch_nonce: "opener-nonce".to_string(),
9883 }),
9884 )
9885 .await;
9886 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
9887
9888 let replies = handler
9889 .handle_control_frame(
9890 &target_ctx,
9891 catalog_update_with_capabilities_frame(
9892 4,
9893 CapabilityDeclarations {
9894 provides: vec!["credentials-provider/v1".to_string()],
9895 requires: Vec::new(),
9896 must_never_reach: Vec::new(),
9897 },
9898 ),
9899 )
9900 .await
9901 .expect("claim catalog.update succeeds");
9902 assert!(matches!(
9903 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
9904 Ok(ModuleControlResponseToModule::CatalogUpdate {})
9905 ));
9906 assert_capability_denied_push(
9907 client_rx
9908 .try_recv()
9909 .expect("claim addition must emit route.closed")
9910 .frame,
9911 "target",
9912 );
9913 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
9914 assert!(matches!(
9915 target_rx.try_recv(),
9916 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
9917 ));
9918 }
9919
9920 #[tokio::test]
9921 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
9922 let registry = Arc::new(Registry::default());
9923 let forwarding = Arc::new(ForwardingTable::default());
9924 let supervisor = SupervisorHandle::new();
9925 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9926 let handler =
9927 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9928 .with_supervisor(supervisor);
9929 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
9930 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
9931 register_capability_manifest(
9932 &handler,
9933 &target_ctx,
9934 &mut target_rx,
9935 capability_manifest("target", &["credentials-provider/v1"], &[]),
9936 1,
9937 )
9938 .await;
9939 register_capability_manifest(
9940 &handler,
9941 &opener_ctx,
9942 &mut opener_rx,
9943 capability_manifest("opener", &[], &[]),
9944 2,
9945 )
9946 .await;
9947 let (mut client_rx, _) = open_route_for_capability_test(
9948 &handler,
9949 &target_ctx,
9950 &mut target_rx,
9951 732,
9952 3,
9953 "target",
9954 Some(ConsumerIdentity {
9955 module_id: "opener".to_string(),
9956 launch_nonce: "opener-nonce".to_string(),
9957 }),
9958 )
9959 .await;
9960 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
9961
9962 handler
9963 .handle_control_frame(
9964 &target_ctx,
9965 catalog_update_with_capabilities_frame(
9966 4,
9967 CapabilityDeclarations {
9968 provides: Vec::new(),
9969 requires: Vec::new(),
9970 must_never_reach: Vec::new(),
9971 },
9972 ),
9973 )
9974 .await
9975 .expect("claim removal catalog.update succeeds");
9976 assert_eq!(
9977 forwarding.active_binding_count().unwrap(),
9978 1,
9979 "removing an attested target claim must leave the route census unchanged"
9980 );
9981 assert!(
9982 client_rx.try_recv().is_err(),
9983 "claim removal must not emit route.closed capability_denied"
9984 );
9985 assert!(
9986 target_rx.try_recv().is_err(),
9987 "claim removal must not send the target a route GOODBYE"
9988 );
9989 }
9990
9991 #[tokio::test]
9994 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
9995 let registry = Arc::new(Registry::default());
9996 let forwarding = Arc::new(ForwardingTable::default());
9997 let supervisor = SupervisorHandle::new();
9998 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9999 let handler =
10000 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10001 .with_supervisor(supervisor);
10002 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
10003 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
10004 register_capability_manifest(
10005 &handler,
10006 &target_ctx,
10007 &mut target_rx,
10008 capability_manifest("target", &["credentials-provider/v1"], &[]),
10009 1,
10010 )
10011 .await;
10012 register_capability_manifest(
10013 &handler,
10014 &opener_ctx,
10015 &mut opener_rx,
10016 capability_manifest("opener", &[], &["credentials-provider/v1"]),
10017 2,
10018 )
10019 .await;
10020
10021 let (_client_rx, bind) = open_route_for_capability_test(
10022 &handler,
10023 &target_ctx,
10024 &mut target_rx,
10025 742,
10026 3,
10027 "target",
10028 None,
10029 )
10030 .await;
10031 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
10032 panic!("direct scope-honesty route must bind");
10033 };
10034 assert_eq!(principal, Some(Principal::Direct));
10035 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
10036 }
10037
10038 #[tokio::test]
10041 async fn must_never_reach_self_route_is_capability_forbidden() {
10042 let registry = Arc::new(Registry::default());
10043 let forwarding = Arc::new(ForwardingTable::default());
10044 let supervisor = SupervisorHandle::new();
10045 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
10046 let handler =
10047 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10048 .with_supervisor(supervisor);
10049 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
10050 register_capability_manifest(
10051 &handler,
10052 &self_ctx,
10053 &mut self_rx,
10054 capability_manifest(
10055 "self-provider",
10056 &["credentials-provider/v1"],
10057 &["credentials-provider/v1"],
10058 ),
10059 1,
10060 )
10061 .await;
10062
10063 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
10064 let replies = handler
10065 .handle_control_frame(
10066 &client_ctx,
10067 route_open_frame_with_admission_facts(
10068 2,
10069 "self-provider",
10070 unique_project_root("admission-facts"),
10071 Some(ConsumerIdentity {
10072 module_id: "self-provider".to_string(),
10073 launch_nonce: "self-nonce".to_string(),
10074 }),
10075 None,
10076 ),
10077 )
10078 .await
10079 .expect("self-route refusal returns a typed frame");
10080 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
10081 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
10082 assert!(
10083 self_rx.try_recv().is_err(),
10084 "self denial must not relay route.bind"
10085 );
10086 }
10087
10088 #[test]
10089 fn unsupported_channel_zero_frame_returns_error() {
10090 let handler = ControlHandler::default();
10091 let request = Frame::build(
10092 FrameType::Request,
10093 control_flags(),
10094 0,
10095 0,
10096 21,
10097 b"opaque".to_vec(),
10098 )
10099 .unwrap();
10100
10101 let response = handler
10102 .handle_control(ConnectionId::new(1), request)
10103 .unwrap();
10104
10105 assert_eq!(response[0].header.ty, FrameType::Error);
10106 assert_eq!(
10107 parse_error(&response[0])["code"],
10108 "unsupported_control_frame"
10109 );
10110 }
10111
10112 mod swap {
10117 use super::*;
10118
10119 const INCUMBENT: ConnectionId = ConnectionId::new(30);
10120 const CANDIDATE: ConnectionId = ConnectionId::new(40);
10121
10122 struct Swap {
10123 registry: Arc<Registry>,
10124 forwarding: Arc<ForwardingTable>,
10125 handler: ControlHandler,
10126 incumbent_ctx: RouteCtx,
10127 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
10128 candidate_ctx: RouteCtx,
10129 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
10130 }
10131
10132 async fn swap_with_incumbent() -> Swap {
10133 let registry = Arc::new(Registry::default());
10134 let forwarding = Arc::new(ForwardingTable::default());
10135 let handler =
10136 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10137 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
10138 hello_via_sink(
10139 &handler,
10140 &incumbent_ctx,
10141 &mut incumbent_rx,
10142 hello_frame("aft", PROTOCOL_VERSION, 7),
10143 )
10144 .await;
10145 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
10146 Swap {
10147 registry,
10148 forwarding,
10149 handler,
10150 incumbent_ctx,
10151 incumbent_rx,
10152 candidate_ctx,
10153 candidate_rx,
10154 }
10155 }
10156
10157 fn register_candidate(swap: &Swap, ready: Option<bool>) {
10158 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
10159 candidate_manifest.ready = ready;
10160 let registration = swap
10161 .registry
10162 .register_candidate_with_control_ops(
10163 candidate_manifest,
10164 PROTOCOL_VERSION,
10165 CANDIDATE,
10166 module_baseline_control_ops(),
10167 )
10168 .unwrap();
10169 swap.forwarding
10170 .register_candidate_module_connection(
10171 CANDIDATE,
10172 "aft".to_string(),
10173 PROTOCOL_VERSION,
10174 manifest_concurrency(®istration.manifest),
10175 swap.candidate_ctx.egress.clone(),
10176 )
10177 .unwrap();
10178 }
10179
10180 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
10181 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
10182 swap.registry.promote_candidate("aft").unwrap().unwrap();
10183 cutover.incumbent.unwrap()
10184 }
10185
10186 fn keyed_total(counters: &Value, key: &str) -> u64 {
10187 counters[key]
10188 .as_object()
10189 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
10190 .unwrap_or(0)
10191 }
10192
10193 #[tokio::test]
10199 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
10200 let mut swap = swap_with_incumbent().await;
10201 let handler = swap.handler.clone();
10202
10203 let cotenant = ConnectionId::new(31);
10205 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
10206 let (cotenant_task, cotenant_bind) = relay_route_open(
10207 &handler,
10208 cotenant,
10209 &cotenant_ctx.egress,
10210 &mut swap.incumbent_rx,
10211 100,
10212 "aft",
10213 "swap-cotenant",
10214 )
10215 .await;
10216 handler
10217 .handle_control_frame(
10218 &swap.incumbent_ctx,
10219 route_bind_ack(cotenant_bind.header.corr),
10220 )
10221 .await
10222 .unwrap();
10223 assert!(cotenant_task.await.unwrap().is_empty());
10224 let (cotenant_channel, cotenant_epoch) =
10225 published_route(&cotenant_rx.recv().await.unwrap());
10226
10227 let caller = ConnectionId::new(32);
10229 let (caller_ctx, mut caller_rx) = route_ctx(caller);
10230 let (caller_task, caller_bind) = relay_route_open(
10231 &handler,
10232 caller,
10233 &caller_ctx.egress,
10234 &mut swap.incumbent_rx,
10235 101,
10236 "aft",
10237 "swap-caller",
10238 )
10239 .await;
10240 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
10241
10242 register_candidate(&swap, None);
10243 cutover(&swap);
10244
10245 let ack = handler
10247 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
10248 .await;
10249 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
10250 if module_loop_error.is_some() {
10251 handler.cleanup_connection(INCUMBENT).unwrap();
10254 }
10255
10256 assert!(
10258 cotenant_rx.try_recv().is_err(),
10259 "the co-tenant route on the incumbent was torn down by one late ack: \
10260 {module_loop_error:?}"
10261 );
10262 assert!(matches!(
10263 swap.forwarding
10264 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
10265 .unwrap(),
10266 DataRoute::Client(DataRouteState::Bound(_))
10267 ));
10268 assert_eq!(module_loop_error, None);
10269 assert!(swap
10270 .registry
10271 .get_module_by_connection(INCUMBENT)
10272 .unwrap()
10273 .is_some());
10274
10275 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
10277 .await
10278 .expect("the incumbent is told to drop the abandoned binding")
10279 .unwrap()
10280 .frame;
10281 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10282 assert_eq!(goodbye.header.channel, abandoned_channel);
10283 assert_eq!(goodbye.header.epoch, abandoned_epoch);
10284 assert!(swap.incumbent_rx.try_recv().is_err());
10285
10286 let response = caller_task.await.unwrap();
10288 assert_eq!(response.len(), 1);
10289 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
10290 assert!(caller_rx.try_recv().is_err());
10291
10292 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
10296 let counters = handler.counters().snapshot();
10297 assert_eq!(
10298 keyed_total(&counters, "route_open_accepted_by_principal"),
10299 1
10300 );
10301 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
10302 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
10303 }
10304
10305 #[tokio::test]
10309 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
10310 let mut swap = swap_with_incumbent().await;
10311 register_candidate(&swap, None);
10312 let incumbent = cutover(&swap);
10313 swap.forwarding
10314 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
10315 .unwrap()
10316 .expect("the incumbent is still registered");
10317
10318 let client = ConnectionId::new(33);
10319 let (client_ctx, mut client_rx) = route_ctx(client);
10320 let route_handler = swap.handler.clone();
10321 let open_ctx = RouteCtx {
10322 connection_id: client,
10323 egress: client_ctx.egress.clone(),
10324 };
10325 let mut route_task = tokio::spawn(async move {
10326 route_handler
10327 .handle_control_frame(
10328 &open_ctx,
10329 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
10330 )
10331 .await
10332 .unwrap()
10333 });
10334 let bind = tokio::select! {
10335 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
10336 response = &mut route_task => {
10337 let response = response.unwrap();
10338 panic!(
10339 "post-cutover route.open was refused instead of relayed to the candidate: {}",
10340 parse_error(&response[0])["code"]
10341 );
10342 }
10343 };
10344 swap.handler
10345 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
10346 .await
10347 .unwrap();
10348 assert!(route_task.await.unwrap().is_empty());
10349 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
10350 match swap
10351 .forwarding
10352 .lookup_data_route(client, channel, epoch)
10353 .unwrap()
10354 {
10355 DataRoute::Client(DataRouteState::Bound(route)) => {
10356 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
10357 }
10358 other => panic!("expected a bound route on the candidate, got {other:?}"),
10359 }
10360 assert!(swap.incumbent_rx.try_recv().is_err());
10361 }
10362
10363 #[tokio::test]
10368 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
10369 let swap = swap_with_incumbent().await;
10370 register_candidate(&swap, Some(false));
10371 let update = Frame::build(
10372 FrameType::Request,
10373 control_flags(),
10374 0,
10375 0,
10376 55,
10377 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
10378 provides: manifest("aft", PROTOCOL_VERSION).provides,
10379 capabilities: None,
10380 ready: Some(true),
10381 })
10382 .unwrap(),
10383 )
10384 .unwrap();
10385
10386 let replies = swap
10387 .handler
10388 .handle_control_frame(&swap.candidate_ctx, update)
10389 .await
10390 .unwrap();
10391
10392 assert_eq!(replies.len(), 1);
10393 assert_eq!(
10394 replies[0].header.ty,
10395 FrameType::Response,
10396 "candidate catalog.update was refused: {:?}",
10397 serde_json::from_slice::<Value>(&replies[0].body).ok()
10398 );
10399 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
10400 assert_eq!(
10401 swap.registry
10402 .get_module("aft")
10403 .unwrap()
10404 .unwrap()
10405 .connection_id,
10406 INCUMBENT
10407 );
10408 }
10409 }
10410
10411 mod swap_admission {
10415 use super::*;
10416
10417 const INCUMBENT_NONCE: &str = "incumbent-nonce";
10418 const CANDIDATE_NONCE: &str = "candidate-nonce";
10419
10420 fn handler_with_incumbent(
10421 module_id: &str,
10422 reserved: bool,
10423 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
10424 let registry = Arc::new(Registry::default());
10425 let supervisor = SupervisorHandle::new();
10426 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
10427 if reserved {
10428 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
10429 }
10430 let handler =
10431 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
10432 let incumbent = handler
10433 .handle_control(
10434 ConnectionId::new(1),
10435 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
10436 )
10437 .unwrap();
10438 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
10439 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
10440 (registry, supervisor, handler)
10441 }
10442
10443 #[test]
10449 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
10450 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
10451
10452 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
10453 let replies = handler
10454 .handle_control(
10455 ConnectionId::new(connection),
10456 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
10457 )
10458 .unwrap();
10459 assert_eq!(replies[0].header.ty, FrameType::Error);
10460 assert_eq!(
10461 parse_error(&replies[0])["code"],
10462 "swap_token_invalid",
10463 "nonce {nonce:?}"
10464 );
10465 }
10466 assert!(registry.get_candidate("aft").unwrap().is_none());
10467 assert_eq!(
10468 registry.get_module("aft").unwrap().unwrap().connection_id,
10469 ConnectionId::new(1)
10470 );
10471
10472 let admitted = handler
10475 .handle_control(
10476 ConnectionId::new(4),
10477 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
10478 )
10479 .unwrap();
10480 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
10481 assert_eq!(
10482 registry
10483 .get_candidate("aft")
10484 .unwrap()
10485 .unwrap()
10486 .connection_id,
10487 ConnectionId::new(4)
10488 );
10489 assert_eq!(
10490 registry.get_module("aft").unwrap().unwrap().connection_id,
10491 ConnectionId::new(1),
10492 "the candidate must not take the active slot"
10493 );
10494 let replayed = handler
10495 .handle_control(
10496 ConnectionId::new(5),
10497 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
10498 )
10499 .unwrap();
10500 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
10501
10502 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
10506 let squatter = handler
10507 .handle_control(
10508 ConnectionId::new(6),
10509 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
10510 )
10511 .unwrap();
10512 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
10513 assert!(
10514 registry.get_module("aft").unwrap().is_none(),
10515 "a squatter took the active slot of an id being swapped"
10516 );
10517 }
10518
10519 #[test]
10524 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
10525 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
10526
10527 let replies = handler
10528 .handle_control(
10529 ConnectionId::new(2),
10530 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
10531 )
10532 .unwrap();
10533
10534 assert_eq!(
10535 replies[0].header.ty,
10536 FrameType::HelloAck,
10537 "reserved candidate refused: {:?}",
10538 serde_json::from_slice::<Value>(&replies[0].body).ok()
10539 );
10540 assert_eq!(
10541 registry
10542 .get_candidate("vault")
10543 .unwrap()
10544 .unwrap()
10545 .connection_id,
10546 ConnectionId::new(2)
10547 );
10548 }
10549
10550 #[test]
10553 fn without_an_open_swap_the_ordinary_gates_decide() {
10554 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
10555 supervisor.close_swap("vault");
10556
10557 let candidate = handler
10558 .handle_control(
10559 ConnectionId::new(2),
10560 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
10561 )
10562 .unwrap();
10563 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
10564 let duplicate = handler
10565 .handle_control(
10566 ConnectionId::new(3),
10567 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
10568 )
10569 .unwrap();
10570 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
10571 assert!(registry.get_candidate("vault").unwrap().is_none());
10572 }
10573 }
10574}
10575
10576#[cfg(test)]
10577mod concurrency_default_exposure_tests {
10578 use super::*;
10579
10580 fn hello_body(role_json: &str) -> Vec<u8> {
10581 format!(
10582 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
10583 )
10584 .into_bytes()
10585 }
10586
10587 fn manifest_from(body: &[u8]) -> ModuleManifest {
10588 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
10589 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
10590 .expect("manifest parses")
10591 }
10592
10593 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
10594
10595 #[test]
10596 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
10597 let body = hello_body(&format!(
10598 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
10599 ));
10600 let manifest = manifest_from(&body);
10601 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
10604 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
10605 }
10606
10607 #[test]
10608 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
10609 let body = hello_body(&format!(
10610 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
10611 ));
10612 let manifest = manifest_from(&body);
10613 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
10614 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
10615 }
10616
10617 #[test]
10618 fn non_management_roles_are_never_reported() {
10619 let body = hello_body(
10620 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
10621 );
10622 let manifest = manifest_from(&body);
10623 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
10624 }
10625}