1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 path::{Path, PathBuf},
5 sync::{Arc, Mutex},
6 time::{Duration, Instant as StdInstant},
7};
8
9use serde::{Deserialize, Serialize};
10use subc_control::{
11 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
12 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
13 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
14 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
15 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
16 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
17 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
18};
19use subc_protocol::{
20 error_codes,
21 manifest::{
22 validate_hello_capability_grammar, validate_hello_self_signal_declarations,
23 CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
24 ProviderRole,
25 },
26 session::{
27 HealthReport, ModuleControlPush, ModuleControlRequest, ModuleControlRequestFromModule,
28 ModuleControlResponse, ModuleControlResponseToModule, MODULE_CONTROL_OP_HEALTH_CHECK,
29 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
30 },
31 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
32 Priority, RouteTarget, PROTOCOL_VERSION,
33};
34use tokio::time::{timeout_at, Instant};
35use tracing::{debug, info, warn};
36
37use crate::{
38 capability_requirements::{
39 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
40 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
41 RuntimeModule,
42 },
43 daemon_config::RestartRequiredSection,
44 forwarding::{
45 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
46 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
47 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
48 },
49 observability::{
50 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
51 },
52 provenance::{
53 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
54 },
55 registry::{ChannelState, ConnectionId, Registry, RegistryError},
56 router::{RouteCtx, RouterError},
57 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
58 stderr_tail::{CaptureState, TailEntry},
59 supervise::{
60 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
61 SupervisorHandle, SwapHelloAdmission,
62 },
63 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
64};
65
66pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
72
73const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
74const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
75const CAP_PING_PONG: &str = "ping_pong_v1";
76const CAP_SESSION_ATTACH: &str = "session_attach_v1";
77const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
78
79const SUBC_CONTROL_OPS: &[&str] = &[
80 ops::SERVER_DESCRIBE,
81 ops::CATALOG_LIST,
82 ops::ROUTE_OPEN,
83 ops::ROUTE_POLL,
84 ops::ROUTE_CLOSING,
85 ops::ROUTE_CLOSED,
86 ops::SUPERVISOR_LIST,
87 ops::SUPERVISOR_RESTART,
88 ops::SUPERVISOR_SWAP,
89 ops::SUPERVISOR_RELOAD,
90 ops::SUPERVISOR_RESCAN,
91 ops::SUPERVISOR_RELEASE_RESERVED,
92 ops::SUPERVISOR_SET_ENABLED,
93 ops::SUPERVISOR_HEALTH_PROBE,
94 ops::SUPERVISOR_HEALTH,
95 ops::SUPERVISOR_STDERR_TAIL,
96 ops::SUPERVISOR_TERMINALS,
97 ops::SUPERVISOR_ROUTES,
98 ops::SUPERVISOR_PROVENANCE,
99 ops::SUPERVISOR_SPAWN_SNAPSHOT,
100 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
101];
102
103const MODULE_TO_SUBC_CONTROL_OPS: &[&str] =
104 &[MODULE_TO_SUBC_OP_CATALOG_UPDATE, "supervisor.live_roots"];
105
106const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
107
108pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
116
117pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
132
133pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
151
152const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
153const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
154
155fn reload_verdict(
156 configured: &Path,
157 spawned_from: Option<&Path>,
158 image: subc_control::RunningImageAgreement,
159) -> PendingReloadVerdict {
160 let path = match spawned_from {
161 Some(spawned_from) if configured == spawned_from => ReloadPathAgreement::Match,
162 Some(spawned_from) => ReloadPathAgreement::Mismatch {
163 configured: configured.to_path_buf(),
164 spawned_from: spawned_from.to_path_buf(),
165 },
166 None => ReloadPathAgreement::Unavailable {
167 reason: if matches!(
168 image,
169 subc_control::RunningImageAgreement::Unavailable {
170 reason: subc_control::RunningImageUnavailableReason::NotRunning
171 }
172 ) {
173 ReloadPathUnavailableReason::NotRunning
174 } else {
175 ReloadPathUnavailableReason::SpawnedPathUnavailable
176 },
177 },
178 };
179 PendingReloadVerdict { path, image }
180}
181
182#[derive(Clone)]
183struct DaemonProvenanceFacts {
184 build: DaemonBuildProvenance,
185 pid: Option<u32>,
186 started_at_ms: Option<u64>,
187 start_clock: Option<crate::clock::StartClock>,
188 executable_path: Option<PathBuf>,
189 executable_identity: Option<SpawnedFileIdentity>,
190 process_start_time: Option<u64>,
191 probe: ExecutableIdentityProbe,
192}
193
194impl Default for DaemonProvenanceFacts {
195 fn default() -> Self {
196 Self {
197 build: DaemonBuildProvenance {
198 build_git_sha: None,
199 build_lock_digest: None,
200 },
201 pid: None,
202 started_at_ms: None,
203 start_clock: None,
204 executable_path: None,
205 executable_identity: None,
206 process_start_time: None,
207 probe: ExecutableIdentityProbe::default(),
208 }
209 }
210}
211
212#[derive(Debug, Clone)]
213struct SupervisorRescanContext {
214 supervisor: Supervisor,
215 config_path: PathBuf,
216 configured_port: Option<u16>,
217 storage_config: Option<crate::daemon_config::StorageConfig>,
218 admission_facts_carrier_module_id: Option<String>,
219 admission_facts_targets: Option<Vec<String>>,
220}
221
222const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
243 "reloading",
244 "supervisor_not_live",
245 "registration_not_active",
246 "no_forwarding_connection",
247 "relay_send_failed",
248];
249
250#[derive(Clone)]
252pub struct ControlHandler {
253 registry: Arc<Registry>,
254 forwarding: Arc<ForwardingTable>,
255 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
256 supervisor: SupervisorHandle,
257 subc_capabilities: Arc<[String]>,
258 route_bind_relay_timeout: Duration,
262 route_bind_relay_timeouts: BTreeMap<String, Duration>,
266 route_bind_breakers: RouteBindBreakers,
269 route_bind_concurrency: RouteBindConcurrency,
272 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
277 route_bind_breaker_threshold: u32,
279 route_bind_breaker_cooldown: Duration,
281 health_probe_timeout: Duration,
282 storage_config: Option<crate::daemon_config::StorageConfig>,
285 machine_id: Option<crate::machine_id::MachineId>,
289 admission_facts_carrier_module_id: Option<String>,
290 admission_facts_targets: Option<Vec<String>>,
291 rescan: Option<SupervisorRescanContext>,
292 connected_clients: ConnectedClients,
293 counters: DaemonCounters,
294 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
295 daemon_provenance: DaemonProvenanceFacts,
296 #[cfg(test)]
297 control_dispatch_delay: Option<Duration>,
298 #[cfg(test)]
299 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
300}
301
302impl fmt::Debug for ControlHandler {
303 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
304 f.debug_struct("ControlHandler")
305 .field("registry", &self.registry)
306 .field("forwarding", &self.forwarding)
307 .field("process_liveness", &self.process_liveness.is_some())
308 .field("supervisor", &self.supervisor)
309 .field("subc_capabilities", &self.subc_capabilities)
310 .finish()
311 }
312}
313
314struct RouteOpenRequest {
315 target: RouteTarget,
316 identity: BindIdentity,
317 consumer_identity: Option<ConsumerIdentity>,
318 consumer_capabilities: Option<Vec<String>>,
319 admission_facts: Option<serde_json::Value>,
320}
321
322struct RouteBindReservationGuard {
323 forwarding: Arc<ForwardingTable>,
324 endpoint: ModuleEndpointId,
325 relay_corr: u64,
326 armed: bool,
327}
328
329struct ModuleControlRpcGuard {
330 forwarding: Arc<ForwardingTable>,
331 endpoint: ModuleEndpointId,
332 corr: u64,
333 armed: bool,
334}
335
336impl ModuleControlRpcGuard {
337 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
338 Self {
339 forwarding,
340 endpoint,
341 corr,
342 armed: true,
343 }
344 }
345
346 fn disarm(&mut self) {
347 self.armed = false;
348 }
349}
350
351impl Drop for ModuleControlRpcGuard {
352 fn drop(&mut self) {
353 if self.armed {
354 let _ = self
355 .forwarding
356 .cancel_module_control_rpc(self.endpoint, self.corr);
357 }
358 }
359}
360
361impl RouteBindReservationGuard {
362 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
363 Self {
364 forwarding,
365 endpoint,
366 relay_corr,
367 armed: true,
368 }
369 }
370
371 fn release_and_disarm(&mut self) {
372 if !self.armed {
373 return;
374 }
375 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
376 self.endpoint,
377 self.relay_corr,
378 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
379 ) {
380 send_goodbye_target_best_effort(
381 &self.forwarding.counters(),
382 &target,
383 "canceled route.bind",
384 );
385 }
386 self.armed = false;
387 }
388
389 fn disarm(&mut self) {
390 self.armed = false;
391 }
392}
393
394impl Drop for RouteBindReservationGuard {
395 fn drop(&mut self) {
396 self.release_and_disarm();
397 }
398}
399
400#[derive(Debug, Clone, Default)]
430pub(crate) struct RouteBindBreakers {
431 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
432}
433
434#[derive(Debug, Clone, Default)]
435pub(crate) struct RouteBindConcurrency {
436 modules: Arc<Mutex<HashMap<String, usize>>>,
437}
438
439struct RouteBindConcurrencyGuard {
440 concurrency: RouteBindConcurrency,
441 module_id: String,
442}
443
444impl RouteBindConcurrency {
445 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
448 let mut modules = self
449 .modules
450 .lock()
451 .expect("route.bind concurrency mutex poisoned");
452 let in_flight = modules.entry(module_id.to_string()).or_default();
453 if *in_flight >= limit {
454 return Err(*in_flight);
455 }
456 *in_flight += 1;
457 Ok(RouteBindConcurrencyGuard {
458 concurrency: self.clone(),
459 module_id: module_id.to_string(),
460 })
461 }
462}
463
464impl Drop for RouteBindConcurrencyGuard {
465 fn drop(&mut self) {
466 let mut modules = self
467 .concurrency
468 .modules
469 .lock()
470 .expect("route.bind concurrency mutex poisoned");
471 let remove = {
472 let in_flight = modules
473 .get_mut(&self.module_id)
474 .expect("admitted route.bind has a concurrency entry");
475 *in_flight -= 1;
476 *in_flight == 0
477 };
478 if remove {
479 modules.remove(&self.module_id);
480 }
481 }
482}
483
484#[derive(Debug, Default)]
485struct ModuleBreakerState {
486 consecutive_timeouts: u32,
488 cooldown_until: Option<Instant>,
491 probe_in_flight: bool,
496}
497
498enum RouteBindAdmission<'a> {
500 Admitted {
501 guard: RouteBindBreakerGuard<'a>,
502 probe: bool,
505 },
506 Refused {
507 consecutive_timeouts: u32,
508 retry_in: Duration,
512 probe_in_flight: bool,
513 },
514}
515
516struct RouteBindBreakerGuard<'a> {
523 breakers: RouteBindBreakers,
524 module_id: &'a str,
525 settled: bool,
526}
527
528impl RouteBindBreakerGuard<'_> {
529 fn record_accepted(&mut self) -> bool {
533 self.settled = true;
534 self.breakers.record_accepted(self.module_id)
535 }
536
537 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
540 self.settled = true;
541 self.breakers
542 .record_timeout(self.module_id, threshold, cooldown)
543 }
544
545 fn record_inconclusive(&mut self) {
554 self.settled = true;
555 self.breakers.record_inconclusive(self.module_id);
556 }
557}
558
559impl Drop for RouteBindBreakerGuard<'_> {
560 fn drop(&mut self) {
561 if !self.settled {
562 self.breakers.record_inconclusive(self.module_id);
563 }
564 }
565}
566
567struct BreakerOpened {
571 consecutive_timeouts: u32,
572 reopened_after_probe: bool,
575}
576
577impl RouteBindBreakers {
578 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
579 self.modules
580 .lock()
581 .expect("route.bind breaker mutex poisoned")
582 }
583
584 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
587 let admitted = |probe| RouteBindAdmission::Admitted {
588 guard: RouteBindBreakerGuard {
589 breakers: self.clone(),
590 module_id,
591 settled: false,
592 },
593 probe,
594 };
595
596 let mut modules = self.lock();
597 let Some(state) = modules.get_mut(module_id) else {
598 return admitted(false);
599 };
600 let Some(cooldown_until) = state.cooldown_until else {
601 return admitted(false);
602 };
603 if state.probe_in_flight {
604 return RouteBindAdmission::Refused {
605 consecutive_timeouts: state.consecutive_timeouts,
606 retry_in: Duration::ZERO,
607 probe_in_flight: true,
608 };
609 }
610 let now = Instant::now();
611 if now < cooldown_until {
612 return RouteBindAdmission::Refused {
613 consecutive_timeouts: state.consecutive_timeouts,
614 retry_in: cooldown_until - now,
615 probe_in_flight: false,
616 };
617 }
618 state.probe_in_flight = true;
619 admitted(true)
620 }
621
622 fn record_accepted(&self, module_id: &str) -> bool {
623 self.lock()
624 .remove(module_id)
625 .is_some_and(|state| state.cooldown_until.is_some())
626 }
627
628 fn record_timeout(
629 &self,
630 module_id: &str,
631 threshold: u32,
632 cooldown: Duration,
633 ) -> Option<BreakerOpened> {
634 let mut modules = self.lock();
635 let state = modules.entry(module_id.to_string()).or_default();
636 let was_open = state.cooldown_until.is_some();
637 let was_probe = state.probe_in_flight;
638 state.probe_in_flight = false;
639 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
640 if state.consecutive_timeouts < threshold {
641 return None;
642 }
643 state.cooldown_until = Some(Instant::now() + cooldown);
644 Some(BreakerOpened {
645 consecutive_timeouts: state.consecutive_timeouts,
646 reopened_after_probe: was_open && was_probe,
647 })
648 }
649
650 fn record_inconclusive(&self, module_id: &str) {
651 if let Some(state) = self.lock().get_mut(module_id) {
652 state.probe_in_flight = false;
653 }
654 }
655
656 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
671 self.lock()
672 .remove(module_id)
673 .map(|state| state.consecutive_timeouts)
674 .filter(|discarded| *discarded > 0)
675 }
676
677 fn open_snapshot(&self) -> Option<serde_json::Value> {
684 let now = Instant::now();
685 let modules = self.lock();
686 let open = modules
687 .iter()
688 .filter_map(|(module_id, state)| {
689 let cooldown_until = state.cooldown_until?;
690 Some((
691 module_id.clone(),
692 serde_json::json!({
693 "consecutive_timeouts": state.consecutive_timeouts,
694 "cooldown_remaining_ms":
695 cooldown_until.saturating_duration_since(now).as_millis() as u64,
696 "probe_in_flight": state.probe_in_flight,
697 }),
698 ))
699 })
700 .collect::<serde_json::Map<String, serde_json::Value>>();
701 (!open.is_empty()).then_some(serde_json::Value::Object(open))
702 }
703}
704
705impl ControlHandler {
706 pub fn new(registry: Arc<Registry>) -> Self {
707 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
708 }
709
710 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
711 let counters = forwarding.counters();
712 let route_bind_breakers = forwarding.route_bind_breakers();
716 let route_bind_concurrency = forwarding.route_bind_concurrency();
717 let route_outages = forwarding.route_outages();
718 Self {
719 registry,
720 forwarding,
721 process_liveness: None,
722 supervisor: SupervisorHandle::new(),
723 subc_capabilities: Arc::from([
724 CAP_MANIFEST_REGISTRATION.to_string(),
725 CAP_CHANNEL_LIFECYCLE.to_string(),
726 CAP_PING_PONG.to_string(),
727 CAP_SESSION_ATTACH.to_string(),
728 CAP_ADMISSION_FACTS_RELAY.to_string(),
729 ]),
730 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
731 route_bind_relay_timeouts: BTreeMap::new(),
732 route_bind_breakers,
733 route_bind_concurrency,
734 route_outages,
735 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
736 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
737 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
738 storage_config: None,
739 machine_id: None,
740 admission_facts_carrier_module_id: None,
741 admission_facts_targets: None,
742 rescan: None,
743 connected_clients: ConnectedClients::new(),
744 counters,
745 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
746 daemon_provenance: DaemonProvenanceFacts::default(),
747 #[cfg(test)]
748 control_dispatch_delay: None,
749 #[cfg(test)]
750 provenance_probe_override: None,
751 }
752 }
753
754 pub fn with_storage_config(
757 mut self,
758 storage_config: Option<crate::daemon_config::StorageConfig>,
759 ) -> Self {
760 self.storage_config = storage_config;
761 self
762 }
763
764 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
767 self.machine_id = machine_id;
768 self
769 }
770
771 pub fn with_admission_facts_config(
775 mut self,
776 carrier_module_id: Option<String>,
777 targets: Option<Vec<String>>,
778 ) -> Self {
779 self.admission_facts_carrier_module_id = carrier_module_id;
780 self.admission_facts_targets = targets;
781 self
782 }
783
784 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
787 self.route_bind_relay_timeout = timeout;
788 self
789 }
790
791 pub fn with_route_bind_relay_timeouts(
797 mut self,
798 timeouts: impl IntoIterator<Item = (String, Duration)>,
799 ) -> Self {
800 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
801 self
802 }
803
804 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
810 self.route_bind_relay_timeouts
811 .get(module_id)
812 .copied()
813 .unwrap_or(self.route_bind_relay_timeout)
814 }
815
816 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
824 self.route_bind_breaker_threshold = threshold.max(1);
825 self.route_bind_breaker_cooldown = cooldown;
826 self
827 }
828
829 #[cfg(test)]
830 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
831 self.health_probe_timeout = timeout;
832 self
833 }
834
835 #[cfg(test)]
836 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
837 self.control_dispatch_delay = Some(delay);
838 self
839 }
840
841 pub fn with_process_liveness(
842 mut self,
843 process_liveness: Arc<dyn ModuleProcessLiveness>,
844 ) -> Self {
845 self.process_liveness = Some(process_liveness);
846 self
847 }
848
849 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
850 self.supervisor = supervisor;
851 self
852 }
853
854 pub fn with_daemon_provenance(
855 mut self,
856 pid: u32,
857 started_at_ms: u64,
858 executable_path: Option<PathBuf>,
859 build_git_sha: Option<String>,
860 build_lock_digest: Option<String>,
861 ) -> Self {
862 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
863 let process_start_time = process_start_time(pid);
864 self.daemon_provenance = DaemonProvenanceFacts {
865 build: DaemonBuildProvenance {
866 build_git_sha,
867 build_lock_digest,
868 },
869 pid: Some(pid),
870 started_at_ms: Some(started_at_ms),
871 start_clock: None,
872 executable_path,
873 executable_identity,
874 process_start_time,
875 probe: ExecutableIdentityProbe::default(),
876 };
877 self
878 }
879
880 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
881 self.daemon_provenance.start_clock = Some(clock);
882 self
883 }
884
885 #[cfg(test)]
886 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
887 self.provenance_probe_override = Some(result);
888 self
889 }
890
891 pub fn with_capability_config(
895 self,
896 modules: impl IntoIterator<Item = (String, bool)>,
897 reserved_capabilities: BTreeMap<String, String>,
898 ) -> Self {
899 self.capability_evaluator
900 .configure(modules, reserved_capabilities);
901 self
902 }
903
904 pub fn with_supervisor_rescan(
905 mut self,
906 supervisor: Supervisor,
907 config_path: impl Into<PathBuf>,
908 configured_port: Option<u16>,
909 ) -> Self {
910 self.rescan = Some(SupervisorRescanContext {
911 supervisor,
912 config_path: config_path.into(),
913 configured_port,
914 storage_config: self.storage_config.clone(),
915 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
916 admission_facts_targets: self.admission_facts_targets.clone(),
917 });
918 self
919 }
920
921 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
922 self.connected_clients = connected_clients;
923 self
924 }
925
926 pub fn forwarding(&self) -> Arc<ForwardingTable> {
927 Arc::clone(&self.forwarding)
928 }
929
930 pub(crate) fn counters(&self) -> DaemonCounters {
931 self.counters.clone()
932 }
933
934 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
937 tokio::spawn(async move {
938 loop {
939 self.capability_evaluator
940 .wait_for_change_or_deadline()
941 .await;
942 self.refresh_capability_requirements();
943 }
944 });
945 }
946
947 fn runtime_capability_snapshot(
948 &self,
949 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
950 let runtime = self
951 .supervisor
952 .list()
953 .into_iter()
954 .map(|module| {
955 let status = module.status().map_err(|err| {
956 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
957 })?;
958 Ok(RuntimeModule {
959 module_id: status.module_id,
960 state: status.state,
961 enabled: status.enabled,
962 })
963 })
964 .collect::<Result<Vec<_>, RouterError>>()?;
965 let (_, registrations) = self.registry.list_modules().map_err(|err| {
966 RouterError::backend(
967 0,
968 0,
969 format!("failed to list capability registrations: {err}"),
970 )
971 })?;
972 let registrations = registrations
973 .into_iter()
974 .map(|registration| RegisteredModule {
975 module_id: registration.manifest.module_id,
976 module_version: registration.manifest.module_version,
977 capabilities: registration.manifest.capabilities,
978 })
979 .collect();
980 Ok((runtime, registrations))
981 }
982
983 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
990 let cached_registration = RegisteredModule {
991 module_id: registration.manifest.module_id.clone(),
992 module_version: registration.manifest.module_version.clone(),
993 capabilities: registration.manifest.capabilities.clone(),
994 };
995 if self.capability_evaluator.record_hello(&cached_registration) {
996 warn!(
997 module_id = %cached_registration.module_id,
998 "capability claims drifted from the cached manifest"
999 );
1000 }
1001 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1002 self.enforce_capability_denies();
1003 }
1004 self.refresh_capability_requirements();
1005 }
1006
1007 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1011 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1012 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1013 self.supervisor.set_swap_promotion_observer(observer);
1014 }
1015
1016 pub fn refresh_capability_requirements(&self) {
1017 match self.runtime_capability_snapshot() {
1018 Ok((runtime, registrations)) => {
1019 log_requirement_events(
1020 self.capability_evaluator
1021 .evaluate_now(&runtime, ®istrations),
1022 );
1023 }
1024 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1025 }
1026 }
1027
1028 fn enforce_capability_denies(&self) {
1032 let (_, registrations) = match self.registry.list_modules() {
1033 Ok(snapshot) => snapshot,
1034 Err(err) => {
1035 warn!(error = %err, "failed to read registrations for capability deny census");
1036 return;
1037 }
1038 };
1039 let manifests = registrations
1040 .into_iter()
1041 .map(|registration| {
1042 (
1043 registration.manifest.module_id.clone(),
1044 registration.manifest,
1045 )
1046 })
1047 .collect::<BTreeMap<_, _>>();
1048 let census = match self.forwarding.route_census(None) {
1049 Ok(census) => census,
1050 Err(err) => {
1051 warn!(error = %err, "failed to read route census for capability deny enforcement");
1052 return;
1053 }
1054 };
1055
1056 for (target_module_id, routes) in census {
1057 let Some(target_manifest) = manifests.get(&target_module_id) else {
1058 continue;
1059 };
1060 let mut closed_routes = Vec::new();
1061 let mut module_goodbyes = Vec::new();
1062 for route in routes {
1063 let Principal::Reserved {
1064 module_id: opening_module_id,
1065 } = &route.principal
1066 else {
1067 continue;
1068 };
1069 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1070 continue;
1071 };
1072 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1073 continue;
1074 };
1075
1076 match self.forwarding.release_client_route(
1077 route.goodbye_target.connection_id,
1078 route.goodbye_target.channel,
1079 route.goodbye_target.epoch,
1080 ) {
1081 Ok(RouteRelease::Removed(module_goodbye)) => {
1082 warn!(
1083 opening_module_id,
1084 target_module_id,
1085 capability,
1086 "force-closing route because an attested capability deny edge now matches"
1087 );
1088 closed_routes.push(route);
1089 module_goodbyes.push(module_goodbye);
1090 }
1091 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1092 Err(err) => warn!(
1093 opening_module_id,
1094 target_module_id,
1095 capability,
1096 error = %err,
1097 "failed to force-close capability-denied route"
1098 ),
1099 }
1100 }
1101
1102 if closed_routes.is_empty() {
1103 continue;
1104 }
1105 send_route_control_pushes(
1106 &self.forwarding,
1107 closed_routes,
1108 ClientControlPush::RouteClosed {
1109 module_id: target_module_id,
1110 reason: RouteCloseReason::CapabilityDenied,
1111 drained: false,
1112 abandoned: 0,
1113 excluded_subscriptions: 0,
1114 terminal: Some(false),
1115 },
1116 );
1117 self.emit_route_goodbyes(module_goodbyes);
1118 }
1119 }
1120
1121 fn not_ready_reason(
1127 &self,
1128 registration: &crate::registry::ModuleRegistration,
1129 ) -> Option<NotReadyReason> {
1130 if !registration.ready {
1131 return Some(NotReadyReason {
1132 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1133 capability: None,
1134 });
1135 }
1136 self.first_unprovided_required_capability(registration)
1137 .map(|capability| NotReadyReason {
1138 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1139 capability: Some(capability),
1140 })
1141 }
1142
1143 fn first_unprovided_required_capability(
1161 &self,
1162 registration: &crate::registry::ModuleRegistration,
1163 ) -> Option<String> {
1164 let required = registration
1165 .manifest
1166 .capabilities
1167 .iter()
1168 .flat_map(|declarations| declarations.requires.iter())
1169 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1170 .map(|requirement| requirement.capability.as_str())
1171 .collect::<BTreeSet<_>>();
1172 if required.is_empty() {
1173 return None;
1174 }
1175 let module_id = registration.manifest.module_id.as_str();
1176 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1177 if required
1178 .iter()
1179 .any(|capability| verdict(capability).is_none())
1180 {
1181 self.refresh_capability_requirements();
1182 }
1183 required
1184 .into_iter()
1185 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1186 .map(str::to_string)
1187 }
1188
1189 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1190 self.capability_evaluator
1191 .statuses()
1192 .into_iter()
1193 .map(capability_requirement_status)
1194 .collect()
1195 }
1196
1197 fn deregister_connection(
1204 &self,
1205 connection_id: ConnectionId,
1206 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1207 self.registry.deregister_connection(connection_id)
1208 }
1209
1210 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1211 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1212 return None;
1213 }
1214 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1215 parse_client_control_request(&frame.body)
1216 else {
1217 return None;
1218 };
1219 Some(target_module_id(&target).to_string())
1220 }
1221
1222 pub(crate) fn route_open_capacity_refusal(
1223 &self,
1224 ctx: &RouteCtx,
1225 frame: &Frame,
1226 target_module_id: &str,
1227 in_flight: usize,
1228 limit: usize,
1229 ) -> Result<Frame, RouterError> {
1230 self.route_open_admission_refusal_frame(
1231 ctx,
1232 frame,
1233 target_module_id,
1234 "open_admission_full",
1235 (in_flight, limit),
1236 format!(
1237 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1238 ),
1239 )
1240 }
1241
1242 fn route_open_target_capacity_refusal(
1243 &self,
1244 ctx: &RouteCtx,
1245 frame: &Frame,
1246 target_module_id: &str,
1247 in_flight: usize,
1248 ) -> Result<Frame, RouterError> {
1249 self.route_open_admission_refusal_frame(
1250 ctx,
1251 frame,
1252 target_module_id,
1253 "target_binds_full",
1254 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1255 format!(
1256 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1257 ),
1258 )
1259 }
1260
1261 fn route_open_admission_refusal_frame(
1269 &self,
1270 ctx: &RouteCtx,
1271 frame: &Frame,
1272 target_module_id: &str,
1273 reason: &'static str,
1274 (in_flight, limit): (usize, usize),
1275 message: impl Into<String>,
1276 ) -> Result<Frame, RouterError> {
1277 let code = error_codes::TARGET_UNAVAILABLE;
1278 self.counters.increment_route_open_refused(code);
1279 info!(
1280 target: "control",
1281 code,
1282 reason,
1283 module_id = ?target_module_id,
1284 connection_id = ctx.connection_id.get(),
1285 in_flight,
1286 limit,
1287 "route.open refused"
1288 );
1289 control_error_frame(frame, code, message.into())
1290 }
1291
1292 #[cfg(test)]
1298 pub fn handle_control(
1299 &self,
1300 connection_id: ConnectionId,
1301 frame: Frame,
1302 ) -> Result<Vec<Frame>, RouterError> {
1303 match frame.header.ty {
1304 FrameType::Ping => Ok(vec![pong(&frame)?]),
1305 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1306 FrameType::Goodbye => self.handle_goodbye(connection_id),
1307 ty => Ok(vec![control_error_frame(
1308 &frame,
1309 "unsupported_control_frame",
1310 format!("unsupported channel-0 frame {ty:?}"),
1311 )?]),
1312 }
1313 }
1314
1315 pub async fn handle_control_frame(
1316 &self,
1317 ctx: &RouteCtx,
1318 frame: Frame,
1319 ) -> Result<Vec<Frame>, RouterError> {
1320 self.handle_control_frame_timed(ctx, frame, None).await
1321 }
1322
1323 pub(crate) async fn handle_control_frame_timed(
1324 &self,
1325 ctx: &RouteCtx,
1326 frame: Frame,
1327 dispatch_started_at: Option<StdInstant>,
1328 ) -> Result<Vec<Frame>, RouterError> {
1329 match frame.header.ty {
1330 FrameType::Ping => Ok(vec![pong(&frame)?]),
1331 FrameType::Hello => {
1332 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1333 }
1334 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1335 FrameType::Cancel => {
1336 if self
1337 .supervisor
1338 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1339 {
1340 Ok(Vec::new())
1341 } else {
1342 Ok(vec![control_error_frame(
1343 &frame,
1344 "unknown_subscription",
1345 "no supervisor spawn subscription has this correlation id",
1346 )?])
1347 }
1348 }
1349 FrameType::Request => {
1350 if self
1351 .forwarding
1352 .module_endpoint_for_connection(ctx.connection_id)
1353 .map_err(RouterError::Forwarding)?
1354 .is_some()
1355 {
1356 if !is_known_module_request_op(&frame.body) {
1357 return Ok(vec![control_error_frame(
1358 &frame,
1359 "unsupported_control_frame",
1360 "module-originated channel-0 REQUEST is not supported",
1361 )?]);
1362 }
1363 let request = match parse_module_control_request_from_module(&frame.body) {
1364 Ok(request) => request,
1365 Err((err, ControlRequestBodyError::UnknownOp)) => {
1366 return Ok(vec![control_error_frame(
1367 &frame,
1368 "unsupported_control_frame",
1369 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1370 )?])
1371 }
1372 Err((err, ControlRequestBodyError::InvalidBody)) => {
1373 return Ok(vec![control_error_frame(
1374 &frame,
1375 "invalid_control_body",
1376 format!("malformed module control body: {err}"),
1377 )?])
1378 }
1379 };
1380 let op = module_control_request_op(&request);
1381 let corr = frame.header.corr;
1382 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1383 let result =
1384 self.handle_module_control_request(ctx.connection_id, frame, request);
1385 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1386 return result;
1387 }
1388
1389 if is_known_module_request_op(&frame.body) {
1390 return Ok(vec![control_error_frame(
1391 &frame,
1392 "not_registered",
1393 "catalog.update requires an active module registration owned by this connection",
1394 )?]);
1395 }
1396
1397 let request = match parse_client_control_request(&frame.body) {
1398 Ok(request) => request,
1399 Err((err, ControlRequestBodyError::UnknownOp)) => {
1400 return Ok(vec![control_error_frame(
1401 &frame,
1402 "unknown_control_op",
1403 format!("unknown client control op: {err}"),
1404 )?])
1405 }
1406 Err((err, ControlRequestBodyError::InvalidBody)) => {
1407 return Ok(vec![control_error_frame(
1408 &frame,
1409 "invalid_control_body",
1410 format!("malformed client control body: {err}"),
1411 )?])
1412 }
1413 };
1414 let op = client_control_request_op(&request);
1415 let corr = frame.header.corr;
1416 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1417 #[cfg(test)]
1418 if let Some(delay) = self.control_dispatch_delay {
1419 tokio::time::sleep(delay).await;
1420 }
1421 let result = self
1422 .handle_client_control_request(ctx, frame, request)
1423 .await;
1424 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1425 result
1426 }
1427 FrameType::Push => {
1428 let Some(endpoint) = self
1429 .forwarding
1430 .module_endpoint_for_connection(ctx.connection_id)
1431 .map_err(RouterError::Forwarding)?
1432 else {
1433 return Ok(vec![control_error_frame(
1434 &frame,
1435 "unsupported_control_frame",
1436 "client-originated channel-0 PUSH is not supported",
1437 )?]);
1438 };
1439 self.handle_status_update(endpoint, frame)
1440 }
1441 FrameType::Response | FrameType::Error
1442 if self
1443 .forwarding
1444 .module_endpoint_for_connection(ctx.connection_id)
1445 .map_err(RouterError::Forwarding)?
1446 .is_some() =>
1447 {
1448 self.handle_module_relay_response(ctx.connection_id, frame)
1449 }
1450 ty => Ok(vec![control_error_frame(
1451 &frame,
1452 "unsupported_control_frame",
1453 format!("unsupported channel-0 frame {ty:?}"),
1454 )?]),
1455 }
1456 }
1457
1458 pub fn cleanup_connection(
1459 &self,
1460 connection_id: ConnectionId,
1461 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1462 let crash_closed = self
1463 .registry
1464 .get_module_by_connection(connection_id)?
1465 .and_then(|registration| {
1466 self.forwarding
1467 .module_endpoint_for_connection(connection_id)
1468 .ok()
1469 .flatten()
1470 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1471 .map(|routes| (registration.manifest.module_id, routes))
1472 });
1473 let crash_closed = crash_closed.map(|(module_id, routes)| {
1474 let terminal = match self.supervisor.get(&module_id) {
1475 None => false,
1476 Some(module) => match module.will_recover_after_connection_loss() {
1477 Ok(will_recover) => !will_recover,
1478 Err(err) => {
1479 warn!(
1480 %module_id,
1481 error = %err,
1482 "failed to read crash recovery verdict; reporting non-terminal conservatively"
1483 );
1484 false
1485 }
1486 },
1487 };
1488 let reason = match self.forwarding.is_daemon_draining() {
1492 Ok(true) => RouteCloseReason::Restart,
1493 Ok(false) => RouteCloseReason::Crash,
1494 Err(err) => {
1495 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
1496 RouteCloseReason::Crash
1497 }
1498 };
1499 (module_id, routes, reason, terminal)
1500 });
1501 let registrations = self.deregister_connection(connection_id);
1502 let cleanup = self.forwarding.cleanup_connection_counted(connection_id);
1503 if let Some((module_id, routes, reason, terminal)) = crash_closed {
1508 let abandoned = cleanup
1509 .as_ref()
1510 .map_or(0, |cleanup| cleanup.abandoned_relays);
1511 send_route_control_pushes(
1512 &self.forwarding,
1513 routes,
1514 ClientControlPush::RouteClosed {
1515 module_id,
1516 reason,
1517 drained: false,
1518 abandoned,
1519 excluded_subscriptions: 0,
1520 terminal: Some(terminal),
1521 },
1522 );
1523 }
1524 if let Ok(cleanup) = cleanup {
1525 self.emit_route_goodbyes(cleanup.released);
1526 }
1527 if matches!(®istrations, Ok(r) if !r.is_empty()) {
1531 crate::supervise::notify_registration_release();
1532 self.capability_evaluator.wake_deadline_loop();
1533 self.refresh_capability_requirements();
1534 }
1535 self.supervisor.remove_spawn_subscribers(connection_id);
1536 registrations
1537 }
1538
1539 pub(crate) fn handle_route_goodbye(
1540 &self,
1541 connection_id: ConnectionId,
1542 route_channel: u16,
1543 route_epoch: u32,
1544 ) -> Result<bool, RouterError> {
1545 debug!(
1546 connection_id = connection_id.get(),
1547 route_channel, route_epoch, "handling route GOODBYE"
1548 );
1549 let RouteRelease::Removed(released_route) = self
1550 .forwarding
1551 .release_client_route(connection_id, route_channel, route_epoch)
1552 .map_err(RouterError::Forwarding)?
1553 else {
1554 return Ok(false);
1555 };
1556 self.emit_route_goodbyes(vec![released_route]);
1557 Ok(true)
1558 }
1559
1560 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
1561 for released in released_routes {
1562 let frame = match Frame::build_with_version(
1563 released.negotiated_ver,
1564 FrameType::Goodbye,
1565 control_flags(),
1566 released.channel,
1567 released.epoch,
1568 0,
1569 Vec::new(),
1570 ) {
1571 Ok(frame) => frame,
1572 Err(err) => {
1573 warn!(
1574 route_channel = released.channel,
1575 error = %err,
1576 "failed to build route GOODBYE frame"
1577 );
1578 continue;
1579 }
1580 };
1581 if !released.close_on_delivery_failure() {
1582 crate::forwarding::send_module_route_goodbye(
1583 &self.counters,
1584 &released.sink,
1585 frame,
1586 released.module_id.as_deref(),
1587 "client route released",
1588 );
1589 continue;
1590 }
1591 if let Err(err) = released.sink.try_send(frame) {
1592 warn!(
1593 target_connection_id = released.connection_id.get(),
1594 route_channel = released.channel,
1595 error = %err,
1596 "route GOODBYE was not delivered to client; closing target connection"
1597 );
1598 if self
1599 .forwarding
1600 .escalate_client_delivery_failure(
1601 released.connection_id,
1602 released.channel,
1603 released.epoch,
1604 CloseReason::new(
1605 "route_goodbye_delivery_failed",
1606 format!(
1607 "failed to enqueue route GOODBYE for channel {}: {err}",
1608 released.channel
1609 ),
1610 ),
1611 crate::forwarding::UndeliveredFrame {
1612 module_id: released.module_id.as_deref(),
1613 sink: &released.sink,
1614 },
1615 )
1616 .unwrap_or(false)
1617 {
1618 self.counters.increment_goodbye_relay_client_failed();
1619 }
1620 }
1621 }
1622 }
1623
1624 fn send_abandoned_route_bind_goodbye(
1636 &self,
1637 module_sink: &crate::FrameSink,
1638 negotiated_ver: u8,
1639 module_channel: u16,
1640 module_epoch: u32,
1641 ) {
1642 let frame = match Frame::build_with_version(
1643 negotiated_ver,
1644 FrameType::Goodbye,
1645 control_flags(),
1646 module_channel,
1647 module_epoch,
1648 0,
1649 Vec::new(),
1650 ) {
1651 Ok(frame) => frame,
1652 Err(err) => {
1653 warn!(
1654 route_channel = module_channel,
1655 error = %err,
1656 "failed to build GOODBYE for abandoned route.bind"
1657 );
1658 return;
1659 }
1660 };
1661 crate::forwarding::send_module_route_goodbye(
1662 &self.counters,
1663 module_sink,
1664 frame,
1665 None,
1666 "abandoned route.bind",
1667 );
1668 }
1669
1670 fn handle_hello(
1671 &self,
1672 connection_id: ConnectionId,
1673 sink: Option<crate::FrameSink>,
1674 frame: Frame,
1675 ) -> Result<Vec<Frame>, RouterError> {
1676 debug!(
1677 connection_id = connection_id.get(),
1678 corr = frame.header.corr,
1679 "handling HELLO"
1680 );
1681 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
1682 Ok(value) => value,
1683 Err(err) => {
1684 return Ok(vec![control_error_frame(
1685 &frame,
1686 "invalid_hello",
1687 format!("malformed HELLO body: {err}"),
1688 )?])
1689 }
1690 };
1691 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
1692 return Ok(vec![control_error_frame(
1693 &frame,
1694 "invalid_capability_grammar",
1695 err.to_string(),
1696 )?]);
1697 }
1698 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
1699 return Ok(vec![control_error_frame(
1700 &frame,
1701 "invalid_manifest",
1702 err.to_string(),
1703 )?]);
1704 }
1705 if let Some(provenance) = hello_value
1706 .get("manifest")
1707 .and_then(|manifest| manifest.get("provenance"))
1708 {
1709 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
1710 return Ok(vec![control_error_frame(
1711 &frame,
1712 "invalid_manifest",
1713 format!("malformed manifest provenance: {err}"),
1714 )?]);
1715 }
1716 }
1717 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
1718 Ok(hello) => hello,
1719 Err(err) => {
1720 return Ok(vec![control_error_frame(
1721 &frame,
1722 "invalid_hello",
1723 format!("malformed HELLO body: {err}"),
1724 )?])
1725 }
1726 };
1727
1728 if hello.protocol_ver != hello.manifest.protocol_ver {
1729 return Ok(vec![control_error_frame(
1730 &frame,
1731 "invalid_manifest",
1732 format!(
1733 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
1734 hello.protocol_ver, hello.manifest.protocol_ver
1735 ),
1736 )?]);
1737 }
1738
1739 if hello.manifest.module_id.trim().is_empty() {
1740 return Ok(vec![control_error_frame(
1741 &frame,
1742 "invalid_manifest",
1743 "manifest module_id must not be empty",
1744 )?]);
1745 }
1746
1747 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
1748 Ok(negotiated_ver) => negotiated_ver,
1749 Err(message) => {
1750 return Ok(vec![control_error_frame(
1751 &frame,
1752 "version_unsupported",
1753 message,
1754 )?])
1755 }
1756 };
1757
1758 let swap_admission = self
1766 .supervisor
1767 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
1768 if swap_admission == SwapHelloAdmission::Refused {
1769 warn!(
1770 module_id = %hello.manifest.module_id,
1771 connection_id = connection_id.get(),
1772 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
1773 );
1774 return Ok(vec![control_error_frame(
1775 &frame,
1776 "swap_token_invalid",
1777 format!(
1778 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
1779 hello.manifest.module_id
1780 ),
1781 )?]);
1782 }
1783 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
1784
1785 if let Some(rejection) = (!swap_candidate)
1793 .then(|| {
1794 self.supervisor.reserved_hello_rejection(
1795 &hello.manifest.module_id,
1796 hello.launch_nonce.as_deref(),
1797 )
1798 })
1799 .flatten()
1800 {
1801 let message = match rejection {
1802 ReservedHelloRejection::Exact { module_id } => format!(
1803 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
1804 ),
1805 ReservedHelloRejection::Prefix {
1806 prefix,
1807 owner_module_id,
1808 } => format!(
1809 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
1810 hello.manifest.module_id
1811 ),
1812 };
1813 return Ok(vec![control_error_frame(
1814 &frame,
1815 "reserved_module",
1816 message,
1817 )?]);
1818 }
1819
1820 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
1821 &hello.manifest.module_id,
1822 hello.manifest.capabilities.as_ref(),
1823 );
1824 if let Some(refusal) = reserved_capability_refusals.first() {
1825 let capability = refusal.capability.clone();
1826 let bound_module = refusal.claimants[0].clone();
1827 log_duplicate_claim_events(reserved_capability_refusals);
1828 return Ok(vec![control_error_frame(
1829 &frame,
1830 "reserved_capability",
1831 format!(
1832 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
1833 capability, bound_module, hello.manifest.module_id
1834 ),
1835 )?]);
1836 }
1837
1838 if self
1841 .forwarding
1842 .connection_has_client_routes(connection_id)
1843 .map_err(RouterError::Forwarding)?
1844 {
1845 return Ok(vec![control_error_frame(
1846 &frame,
1847 "invalid_hello",
1848 "connection has open client routes and cannot also register as a module",
1849 )?]);
1850 }
1851
1852 let control_ops = effective_module_control_ops(hello.control_ops);
1853 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
1856 if swap_candidate {
1857 return self.register_swap_candidate(
1858 connection_id,
1859 sink,
1860 &frame,
1861 hello.manifest,
1862 negotiated_ver,
1863 control_ops,
1864 hello_ack,
1865 );
1866 }
1867 let registration = match self.registry.register_with_control_ops(
1868 hello.manifest,
1869 negotiated_ver,
1870 connection_id,
1871 control_ops,
1872 ) {
1873 Ok(registration) => registration,
1874 Err(RegistryError::DuplicateModuleId { module_id }) => {
1875 return Ok(vec![control_error_frame(
1876 &frame,
1877 "duplicate_module_id",
1878 format!(
1879 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
1880 ),
1881 )?])
1882 }
1883 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
1884 return Ok(vec![control_error_frame(
1885 &frame,
1886 "invalid_module_id",
1887 err.to_string(),
1888 )?])
1889 }
1890 Err(err) => {
1891 return Ok(vec![control_error_frame(
1892 &frame,
1893 "registry_error",
1894 err.to_string(),
1895 )?])
1896 }
1897 };
1898
1899 let reply = if let Some(sink) = sink {
1900 let concurrency = manifest_concurrency(®istration.manifest);
1916 if let Err(err) = self.forwarding.register_module_connection_acked(
1917 connection_id,
1918 registration.manifest.module_id.clone(),
1919 negotiated_ver,
1920 concurrency,
1921 sink,
1922 hello_ack,
1923 ) {
1924 if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
1928 crate::supervise::notify_registration_release();
1929 }
1930 return Ok(vec![control_error_frame(
1931 &frame,
1932 forwarding_error_code(&err),
1933 err.to_string(),
1934 )?]);
1935 }
1936 Vec::new()
1937 } else {
1938 vec![hello_ack]
1941 };
1942
1943 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
1952 info!(
1953 module_id = %registration.manifest.module_id,
1954 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
1955 );
1956 }
1957
1958 self.apply_registration_capabilities(®istration);
1959
1960 info!(
1961 module_id = %registration.manifest.module_id,
1962 module_version = %registration.manifest.module_version,
1963 negotiated_ver,
1964 routable_provider = manifest_provides_routable_role(®istration.manifest),
1965 connection_id = connection_id.get(),
1966 "module registered"
1967 );
1968
1969 Ok(reply)
1970 }
1971
1972 #[allow(clippy::too_many_arguments)]
1982 fn register_swap_candidate(
1983 &self,
1984 connection_id: ConnectionId,
1985 sink: Option<crate::FrameSink>,
1986 frame: &Frame,
1987 manifest: ModuleManifest,
1988 negotiated_ver: u8,
1989 control_ops: Vec<String>,
1990 hello_ack: Frame,
1991 ) -> Result<Vec<Frame>, RouterError> {
1992 let module_id = manifest.module_id.clone();
1993 let registration = match self.registry.register_candidate_with_control_ops(
1994 manifest,
1995 negotiated_ver,
1996 connection_id,
1997 control_ops,
1998 ) {
1999 Ok(registration) => registration,
2000 Err(RegistryError::DuplicateModuleId { module_id }) => {
2001 return Ok(vec![control_error_frame(
2002 frame,
2003 "duplicate_module_id",
2004 format!(
2005 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2006 ),
2007 )?])
2008 }
2009 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2010 return Ok(vec![control_error_frame(
2011 frame,
2012 "invalid_module_id",
2013 err.to_string(),
2014 )?])
2015 }
2016 Err(err) => {
2017 return Ok(vec![control_error_frame(
2018 frame,
2019 "registry_error",
2020 err.to_string(),
2021 )?])
2022 }
2023 };
2024 let reply = if let Some(sink) = sink {
2025 let concurrency = manifest_concurrency(®istration.manifest);
2029 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2030 connection_id,
2031 module_id.clone(),
2032 negotiated_ver,
2033 concurrency,
2034 sink,
2035 hello_ack,
2036 ) {
2037 if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
2038 crate::supervise::notify_registration_release();
2039 }
2040 return Ok(vec![control_error_frame(
2041 frame,
2042 forwarding_error_code(&err),
2043 err.to_string(),
2044 )?]);
2045 }
2046 Vec::new()
2047 } else {
2048 vec![hello_ack]
2049 };
2050 self.supervisor.mark_swap_candidate_admitted(&module_id);
2051 info!(
2052 module_id = %module_id,
2053 module_version = %registration.manifest.module_version,
2054 negotiated_ver,
2055 ready = registration.ready,
2056 connection_id = connection_id.get(),
2057 "swap candidate registered; not routable until cutover"
2058 );
2059 Ok(reply)
2060 }
2061
2062 fn build_hello_ack(
2063 &self,
2064 frame: &Frame,
2065 negotiated_ver: u8,
2066 module_id: &str,
2067 ) -> Result<Frame, RouterError> {
2068 let ack = ModuleHelloAckBody {
2069 negotiated_ver,
2070 subc_ops: module_subc_ops(),
2071 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2072 storage: self
2073 .storage_config
2074 .as_ref()
2075 .map(|cfg| cfg.descriptor_for(module_id)),
2076 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2077 };
2078 let body = serde_json::to_vec(&ack).map_err(|err| {
2079 RouterError::backend(
2080 0,
2081 frame.header.corr,
2082 format!("failed to encode HELLO_ACK: {err}"),
2083 )
2084 })?;
2085
2086 Frame::build_with_version(
2087 negotiated_ver,
2088 FrameType::HelloAck,
2089 control_flags(),
2090 0,
2091 0,
2092 frame.header.corr,
2093 body,
2094 )
2095 .map_err(RouterError::FrameBuild)
2096 }
2097
2098 async fn handle_client_control_request(
2099 &self,
2100 ctx: &RouteCtx,
2101 frame: Frame,
2102 request: ClientControlRequest,
2103 ) -> Result<Vec<Frame>, RouterError> {
2104 match request {
2105 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2106 ClientControlRequest::CatalogList { module_id } => {
2107 self.handle_catalog_list(frame, module_id)
2108 }
2109 ClientControlRequest::RouteOpen {
2110 target,
2111 identity,
2112 consumer_identity,
2113 consumer_capabilities,
2114 admission_facts,
2115 } => {
2116 self.handle_route_open(
2117 ctx,
2118 frame,
2119 RouteOpenRequest {
2120 target,
2121 identity,
2122 consumer_identity,
2123 consumer_capabilities,
2124 admission_facts,
2125 },
2126 )
2127 .await
2128 }
2129 ClientControlRequest::RoutePoll {
2130 route_channel,
2131 route_epoch,
2132 kind,
2133 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2134 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2135 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2136 self.handle_supervisor_spawn_snapshot(frame)
2137 }
2138 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2139 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2140 }
2141 ClientControlRequest::SupervisorRestart {
2142 module_id,
2143 drain_timeout_ms,
2144 } => {
2145 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms)
2146 .await
2147 }
2148 ClientControlRequest::SupervisorSwap {
2149 module_id,
2150 ready_timeout_ms,
2151 } => {
2152 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms)
2153 .await
2154 }
2155 ClientControlRequest::SupervisorReload { module_id } => {
2156 self.handle_supervisor_reload(frame, module_id).await
2157 }
2158 ClientControlRequest::SupervisorRescan { preview } => {
2159 self.handle_supervisor_rescan(frame, preview).await
2160 }
2161 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2162 self.handle_supervisor_release_reserved(frame, module_id)
2163 .await
2164 }
2165 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2166 self.handle_supervisor_set_enabled(frame, module_id, enabled)
2167 .await
2168 }
2169 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2170 self.handle_supervisor_health_probe(frame, module_id).await
2171 }
2172 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2173 ClientControlRequest::SupervisorRoutes { module_id } => {
2174 self.handle_supervisor_routes(frame, module_id)
2175 }
2176 ClientControlRequest::SupervisorProvenance { module_id } => {
2177 self.handle_supervisor_provenance(frame, module_id).await
2178 }
2179 ClientControlRequest::SupervisorStderrTail {
2180 module_id,
2181 max_lines,
2182 max_bytes,
2183 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2184 ClientControlRequest::SupervisorTerminals { module_id } => {
2185 self.handle_supervisor_terminals(frame, module_id).await
2186 }
2187 }
2188 }
2189
2190 fn handle_module_control_request(
2191 &self,
2192 connection_id: ConnectionId,
2193 frame: Frame,
2194 request: ModuleControlRequestFromModule,
2195 ) -> Result<Vec<Frame>, RouterError> {
2196 match request {
2197 ModuleControlRequestFromModule::CatalogUpdate {
2198 provides,
2199 capabilities,
2200 ready,
2201 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2202 ModuleControlRequestFromModule::LiveRoots {} => {
2203 let registered = self
2204 .registry
2205 .get_module_by_connection(connection_id)
2206 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2207 let Some(registration) = registered else {
2208 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2209 };
2210 let response = self
2211 .forwarding
2212 .live_roots(®istration.manifest.module_id)
2213 .map_err(RouterError::Forwarding)?;
2214 Ok(vec![control_response_body_frame(
2215 &frame,
2216 &response,
2217 "ModuleControlResponseToModule::LiveRoots",
2218 )?])
2219 }
2220 }
2221 }
2222
2223 fn handle_catalog_update(
2224 &self,
2225 connection_id: ConnectionId,
2226 frame: Frame,
2227 provides: Vec<ProviderRole>,
2228 capabilities: Option<CapabilityDeclarations>,
2229 ready: Option<bool>,
2230 ) -> Result<Vec<Frame>, RouterError> {
2231 self.refresh_capability_requirements();
2232 let Some(registration) = self
2233 .registry
2234 .get_module_by_connection(connection_id)
2235 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2236 else {
2237 return Ok(vec![control_error_frame(
2238 &frame,
2239 "not_registered",
2240 "catalog.update requires an active module registration owned by this connection",
2241 )?]);
2242 };
2243
2244 if let Some(message) =
2245 catalog_update_frozen_field_message(®istration.manifest, &provides)
2246 {
2247 return Ok(vec![control_error_frame(
2248 &frame,
2249 "catalog_update_frozen_field",
2250 message,
2251 )?]);
2252 }
2253
2254 let mut candidate = registration.manifest.clone();
2255 candidate.provides = provides.clone();
2256 candidate.capabilities = capabilities
2257 .clone()
2258 .or_else(|| registration.manifest.capabilities.clone());
2259 if let Err(err) = candidate.validate_capability_grammar() {
2260 return Ok(vec![control_error_frame(
2261 &frame,
2262 "invalid_capability_grammar",
2263 err.to_string(),
2264 )?]);
2265 }
2266
2267 let updated = self
2268 .registry
2269 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
2270 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2271 if updated.is_none() {
2272 return Ok(vec![control_error_frame(
2273 &frame,
2274 "not_registered",
2275 "catalog.update requires an active module registration owned by this connection",
2276 )?]);
2277 }
2278 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
2279 log_duplicate_claim_events(
2280 self.capability_evaluator
2281 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
2282 );
2283 }
2284 if capability_census_trigger(
2285 registration.manifest.capabilities.as_ref(),
2286 updated
2287 .as_ref()
2288 .and_then(|entry| entry.manifest.capabilities.as_ref()),
2289 ) {
2290 self.enforce_capability_denies();
2291 }
2292 self.refresh_capability_requirements();
2293
2294 let response = ModuleControlResponseToModule::CatalogUpdate {};
2295 control_response_body_frame(
2296 &frame,
2297 &response,
2298 "ModuleControlResponseToModule::CatalogUpdate",
2299 )
2300 .map(|frame| vec![frame])
2301 }
2302
2303 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
2304 self.refresh_capability_requirements();
2305 let mut counters = self.counters.snapshot();
2311 if let (Ok((connections_with_routes, max)), Some(obj)) = (
2312 self.forwarding.client_route_concentration(),
2313 counters.as_object_mut(),
2314 ) {
2315 obj.insert(
2316 "client_connections_with_routes".into(),
2317 connections_with_routes.into(),
2318 );
2319 obj.insert("max_routes_on_one_connection".into(), max.into());
2320 }
2321 if let (Some(open_breakers), Some(obj)) = (
2327 self.route_bind_breakers.open_snapshot(),
2328 counters.as_object_mut(),
2329 ) {
2330 obj.insert("route_bind_breakers_open".into(), open_breakers);
2331 }
2332 let response = ClientControlResponse::ServerDescribe {
2333 protocol_ver: PROTOCOL_VERSION,
2334 subc_ops: subc_ops(),
2335 capabilities: self.subc_capabilities.as_ref().to_vec(),
2336 connected_clients: self.connected_clients.count(),
2337 counters: Some(counters),
2338 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
2339 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
2340 capability_requirements: self.capability_requirement_statuses(),
2341 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2342 };
2343 Ok(vec![control_response_body_frame(
2344 &frame,
2345 &response,
2346 "ClientControlResponse::ServerDescribe",
2347 )?])
2348 }
2349
2350 fn handle_catalog_list(
2351 &self,
2352 frame: Frame,
2353 module_id: Option<String>,
2354 ) -> Result<Vec<Frame>, RouterError> {
2355 let (generation, modules) = self.registry.list_modules().map_err(|err| {
2356 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
2357 })?;
2358 let entries = modules
2359 .into_iter()
2360 .filter(|registration| {
2361 module_id
2362 .as_deref()
2363 .map(|wanted| registration.manifest.module_id == wanted)
2364 .unwrap_or(true)
2365 })
2366 .map(|registration| {
2367 let not_ready = self.not_ready_reason(®istration);
2368 let roles = registration.manifest.provides;
2369 CatalogEntry {
2370 module_id: registration.manifest.module_id,
2371 ready: not_ready.is_none(),
2372 not_ready,
2373 module_version: Some(registration.manifest.module_version),
2374 roles,
2375 control_ops: registration.control_ops,
2376 capabilities: registration.manifest.capabilities,
2377 self_signals: registration.manifest.self_signals,
2378 }
2379 })
2380 .collect();
2381 let response = ClientControlResponse::CatalogList {
2382 generation,
2383 modules: entries,
2384 subc_ops: subc_ops(),
2385 };
2386 Ok(vec![control_response_body_frame(
2387 &frame,
2388 &response,
2389 "ClientControlResponse::CatalogList",
2390 )?])
2391 }
2392
2393 fn route_open_principal(
2394 &self,
2395 frame: &Frame,
2396 consumer_identity: Option<ConsumerIdentity>,
2397 ) -> Result<Result<Principal, Frame>, RouterError> {
2398 let Some(consumer_identity) = consumer_identity else {
2399 return Ok(Ok(Principal::Direct));
2400 };
2401
2402 if self.supervisor.spawned_consumer_authorized(
2403 &consumer_identity.module_id,
2404 &consumer_identity.launch_nonce,
2405 ) {
2406 return Ok(Ok(Principal::Reserved {
2407 module_id: consumer_identity.module_id,
2408 }));
2409 }
2410
2411 Ok(Err(control_error_frame(
2412 frame,
2413 "bad_consumer_identity",
2414 format!(
2415 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
2416 consumer_identity.module_id
2417 ),
2418 )?))
2419 }
2420
2421 fn route_open_refusal_frame(
2428 &self,
2429 ctx: &RouteCtx,
2430 frame: &Frame,
2431 module_id: &str,
2432 reason: &'static str,
2433 code: &'static str,
2434 message: impl Into<String>,
2435 ) -> Result<Frame, RouterError> {
2436 self.observe_route_open_refusal(ctx, module_id, reason, code);
2437 control_error_frame(frame, code, message.into())
2438 }
2439
2440 fn route_open_breaker_refusal_frame(
2458 &self,
2459 ctx: &RouteCtx,
2460 frame: &Frame,
2461 module_id: &str,
2462 consecutive_timeouts: u32,
2463 retry_in: Duration,
2464 probe_in_flight: bool,
2465 ) -> Result<Frame, RouterError> {
2466 self.counters
2467 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
2468 debug!(
2469 target: "control",
2470 code = "module_timeout",
2471 module_id = ?module_id,
2472 connection_id = ctx.connection_id.get(),
2473 consecutive_timeouts,
2474 retry_in_ms = retry_in.as_millis() as u64,
2475 probe_in_flight,
2476 "route.open refused by open bind-relay breaker"
2477 );
2478 let detail = if probe_in_flight {
2479 "one probe bind is already in flight; retry once it settles".to_string()
2480 } else {
2481 format!("not relaying for another {retry_in:?}")
2482 };
2483 control_error_frame(
2484 frame,
2485 "module_timeout",
2486 format!(
2487 "module_id '{module_id}' failed {consecutive_timeouts} consecutive route.bind \
2488 relays; {detail}"
2489 ),
2490 )
2491 }
2492
2493 fn observe_route_open_refusal(
2506 &self,
2507 ctx: &RouteCtx,
2508 module_id: &str,
2509 reason: &'static str,
2510 code: &'static str,
2511 ) {
2512 self.counters.increment_route_open_refused(code);
2513 info!(
2514 target: "control",
2515 code,
2516 reason,
2517 module_id = ?module_id,
2518 connection_id = ctx.connection_id.get(),
2519 "route.open refused"
2520 );
2521 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
2522 self.route_outages.record_not_serving(module_id, reason);
2523 }
2524 }
2525
2526 fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
2571 self.route_outages.record_accepted(module_id);
2572 self.counters.increment_route_open_accepted(principal);
2573 info!(
2574 target: "control",
2575 principal,
2576 module_id,
2577 connection_id = ctx.connection_id.get(),
2578 "route.open accepted"
2579 );
2580 }
2581
2582 fn supervised_absent_route_open_refusal_frame(
2583 &self,
2584 ctx: &RouteCtx,
2585 frame: &Frame,
2586 module_id: &str,
2587 code: &'static str,
2588 status: &crate::supervise::ModuleStatus,
2589 ) -> Result<Frame, RouterError> {
2590 self.counters.increment_route_open_refused(code);
2591 info!(
2592 target: "control",
2593 code,
2594 reason = "supervised_not_registered",
2595 module_id = ?module_id,
2596 connection_id = ctx.connection_id.get(),
2597 state = %status.state,
2598 enabled = status.enabled,
2599 live = status.live,
2600 "route.open refused"
2601 );
2602 self.route_outages
2606 .record_not_serving(module_id, "supervised_not_registered");
2607 control_error_frame(
2608 frame,
2609 code,
2610 format!(
2611 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
2612 status.state, status.enabled, status.live
2613 ),
2614 )
2615 }
2616
2617 async fn handle_route_open(
2618 &self,
2619 ctx: &RouteCtx,
2620 frame: Frame,
2621 request: RouteOpenRequest,
2622 ) -> Result<Vec<Frame>, RouterError> {
2623 let RouteOpenRequest {
2624 target,
2625 mut identity,
2626 consumer_identity,
2627 consumer_capabilities,
2628 admission_facts,
2629 } = request;
2630 let target_module_id = target_module_id(&target).to_string();
2631 debug!(
2632 connection_id = ctx.connection_id.get(),
2633 corr = frame.header.corr,
2634 module_id = %target_module_id,
2635 "handling route.open"
2636 );
2637
2638 let Some(registration) = self
2657 .registry
2658 .get_module(&target_module_id)
2659 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2660 else {
2661 if let Some((status, warming)) =
2662 self.supervisor_status(&target_module_id, frame.header.corr)?
2663 {
2664 if status.protocol == ModuleProtocol::None {
2671 return Ok(vec![self.route_open_refusal_frame(
2672 ctx,
2673 &frame,
2674 &target_module_id,
2675 "protocol_none",
2676 error_codes::MODULE_NO_PROTOCOL,
2677 format!(
2678 "module_id '{target_module_id}' is declared protocol: none; \
2679 it speaks no subc wire and serves no routes"
2680 ),
2681 )?]);
2682 }
2683 let code = if warming {
2684 "module_warming"
2685 } else {
2686 "target_unavailable"
2687 };
2688 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
2689 ctx,
2690 &frame,
2691 &target_module_id,
2692 code,
2693 &status,
2694 )?]);
2695 }
2696 if let Some(removed_ago_ms) =
2697 self.supervisor.removal_tombstone_age_ms(&target_module_id)
2698 {
2699 return Ok(vec![self.route_open_refusal_frame(
2700 ctx,
2701 &frame,
2702 &target_module_id,
2703 "removed",
2704 error_codes::MODULE_REMOVED,
2705 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
2706 )?]);
2707 }
2708 return Ok(vec![self.route_open_refusal_frame(
2709 ctx,
2710 &frame,
2711 &target_module_id,
2712 "not_registered",
2713 error_codes::UNKNOWN_MODULE,
2714 format!("module_id '{target_module_id}' is not registered"),
2715 )?]);
2716 };
2717
2718 if !registration.ready {
2722 self.counters
2723 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
2724 info!(
2725 target: "control",
2726 code = error_codes::MODULE_WARMING,
2727 module_id = ?target_module_id,
2728 connection_id = ctx.connection_id.get(),
2729 reason = "declared_not_ready",
2730 "route.open refused"
2731 );
2732 self.route_outages
2735 .record_not_serving(&target_module_id, "declared_not_ready");
2736 return Ok(vec![control_error_body_frame(
2737 &frame,
2738 ErrorBody {
2739 code: error_codes::MODULE_WARMING.to_string(),
2740 message: format!(
2741 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
2742 ),
2743 detail: Some(serde_json::json!({
2744 "reason": "declared_not_ready"
2745 })),
2746 },
2747 )?]);
2748 }
2749
2750 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
2769 self.counters
2770 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
2771 info!(
2772 target: "control",
2773 code = error_codes::MODULE_WARMING,
2774 module_id = ?target_module_id,
2775 connection_id = ctx.connection_id.get(),
2776 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
2777 capability = %capability,
2778 "route.open refused"
2779 );
2780 return Ok(vec![control_error_body_frame(
2781 &frame,
2782 ErrorBody {
2783 code: error_codes::MODULE_WARMING.to_string(),
2784 message: format!(
2785 "module_id '{target_module_id}' requires capability '{capability}', \
2786 which no registered module provides; retry"
2787 ),
2788 detail: Some(serde_json::json!({
2789 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
2790 "capability": capability,
2791 })),
2792 },
2793 )?]);
2794 }
2795
2796 if !target_has_required_role(&target, ®istration.manifest.provides) {
2797 return Ok(vec![self.route_open_refusal_frame(
2798 ctx,
2799 &frame,
2800 &target_module_id,
2801 "role_not_provided",
2802 "target_unavailable",
2803 format!("module_id '{target_module_id}' does not provide the requested target"),
2804 )?]);
2805 }
2806
2807 if registration.state != ChannelState::Active {
2808 return Ok(vec![self.route_open_refusal_frame(
2809 ctx,
2810 &frame,
2811 &target_module_id,
2812 "registration_not_active",
2813 "target_unavailable",
2814 format!("module_id '{target_module_id}' is not active"),
2815 )?]);
2816 }
2817
2818 if self
2819 .forwarding
2820 .module_is_draining(&target_module_id)
2821 .map_err(RouterError::Forwarding)?
2822 {
2823 return Ok(vec![self.route_open_refusal_frame(
2824 ctx,
2825 &frame,
2826 &target_module_id,
2827 "reloading",
2828 "module_reloading",
2829 format!("module_id '{target_module_id}' is reloading"),
2830 )?]);
2831 }
2832
2833 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
2834 process_liveness.process_live(&target_module_id) == Some(false)
2835 }) {
2836 if process_liveness.process_replacing(&target_module_id) {
2843 return Ok(vec![self.route_open_refusal_frame(
2844 ctx,
2845 &frame,
2846 &target_module_id,
2847 "reloading",
2848 "module_reloading",
2849 format!("module_id '{target_module_id}' is reloading"),
2850 )?]);
2851 }
2852 return Ok(vec![self.route_open_refusal_frame(
2853 ctx,
2854 &frame,
2855 &target_module_id,
2856 "supervisor_not_live",
2857 "target_unavailable",
2858 format!("module_id '{target_module_id}' is not live"),
2859 )?]);
2860 }
2861
2862 if !self
2863 .forwarding
2864 .has_live_module_connection(&target_module_id)
2865 .map_err(RouterError::Forwarding)?
2866 {
2867 return Ok(vec![self.route_open_refusal_frame(
2868 ctx,
2869 &frame,
2870 &target_module_id,
2871 "no_forwarding_connection",
2872 "target_unavailable",
2873 format!("module_id '{target_module_id}' has no live forwarding connection"),
2874 )?]);
2875 }
2876
2877 if let Some(error) =
2878 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
2879 {
2880 self.observe_route_open_refusal(
2881 ctx,
2882 &target_module_id,
2883 "op_not_allowed",
2884 "op_not_allowed",
2885 );
2886 return Ok(vec![error]);
2887 }
2888
2889 let principal = match self.route_open_principal(&frame, consumer_identity)? {
2890 Ok(principal) => principal,
2891 Err(error) => {
2892 self.observe_route_open_refusal(
2893 ctx,
2894 &target_module_id,
2895 "bad_consumer_identity",
2896 "bad_consumer_identity",
2897 );
2898 return Ok(vec![error]);
2899 }
2900 };
2901
2902 if let Principal::Reserved {
2906 module_id: opening_module_id,
2907 } = &principal
2908 {
2909 if let Some(opening_registration) = self
2910 .registry
2911 .get_module(opening_module_id)
2912 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2913 {
2914 if let Some(capability) =
2915 denied_capability(&opening_registration.manifest, ®istration.manifest)
2916 {
2917 warn!(
2918 opening_module_id,
2919 target_module_id,
2920 capability,
2921 "refusing route.open because an attested capability deny edge matches"
2922 );
2923 return Ok(vec![self.route_open_refusal_frame(
2924 ctx,
2925 &frame,
2926 &target_module_id,
2927 "capability_deny_edge",
2928 "capability_forbidden",
2929 format!(
2930 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
2931 ),
2932 )?]);
2933 }
2934 }
2935 }
2936
2937 if admission_facts.is_some() {
2938 let carrier_matches = matches!(
2939 &principal,
2940 Principal::Reserved { module_id }
2941 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
2942 );
2943 if !carrier_matches {
2944 return Ok(vec![self.route_open_refusal_frame(
2945 ctx,
2946 &frame,
2947 &target_module_id,
2948 "admission_facts_carrier_not_permitted",
2949 "admission_facts_not_permitted",
2950 "admission facts may only be carried by the configured reserved module",
2951 )?]);
2952 }
2953
2954 let target_allowed = self
2955 .admission_facts_targets
2956 .as_ref()
2957 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
2958 if !target_allowed {
2959 return Ok(vec![self.route_open_refusal_frame(
2960 ctx,
2961 &frame,
2962 &target_module_id,
2963 "admission_facts_target_not_listed",
2964 "admission_facts_target_not_allowed",
2965 format!(
2966 "admission facts are not permitted for target module_id '{target_module_id}'"
2967 ),
2968 )?]);
2969 }
2970
2971 }
2975
2976 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
2998 Ok(project_root) => project_root,
2999 Err(err) => {
3000 return Ok(vec![control_error_frame(
3001 &frame,
3002 "invalid_project_root",
3003 err.to_string(),
3004 )?])
3005 }
3006 };
3007 identity.project_root = project_root.as_path().to_path_buf();
3008
3009 let _concurrency_guard = match self
3020 .route_bind_concurrency
3021 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
3022 {
3023 Ok(guard) => guard,
3024 Err(in_flight) => {
3025 return Ok(vec![self.route_open_target_capacity_refusal(
3026 ctx,
3027 &frame,
3028 &target_module_id,
3029 in_flight,
3030 )?]);
3031 }
3032 };
3033
3034 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
3037 RouteBindAdmission::Admitted { guard, probe } => {
3038 if probe {
3039 info!(
3040 module_id = %target_module_id,
3041 connection_id = ctx.connection_id.get(),
3042 "route.bind breaker half-open: admitting one probe"
3043 );
3044 }
3045 guard
3046 }
3047 RouteBindAdmission::Refused {
3048 consecutive_timeouts,
3049 retry_in,
3050 probe_in_flight,
3051 } => {
3052 return Ok(vec![self.route_open_breaker_refusal_frame(
3053 ctx,
3054 &frame,
3055 &target_module_id,
3056 consecutive_timeouts,
3057 retry_in,
3058 probe_in_flight,
3059 )?]);
3060 }
3061 };
3062
3063 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
3068 let relay_deadline = Instant::now() + route_bind_relay_timeout;
3069 let pending = match self
3070 .forwarding
3071 .begin_route_bind_relay_for(
3072 ctx.connection_id,
3073 ctx.egress.clone(),
3074 response_version(&frame),
3075 frame.header.corr,
3076 &target_module_id,
3077 principal.clone(),
3078 Some(project_root),
3079 relay_deadline,
3080 )
3081 .await
3082 {
3083 Ok(pending) => pending,
3084 Err(err) => {
3085 return Ok(vec![self.route_open_refusal_frame(
3086 ctx,
3087 &frame,
3088 &target_module_id,
3089 "relay_reservation_failed",
3090 forwarding_error_code(&err),
3091 err.to_string(),
3092 )?])
3093 }
3094 };
3095 let crate::forwarding::PendingRouteBindRelay {
3096 endpoint,
3097 module_sink,
3098 negotiated_ver,
3099 client_channel,
3100 client_epoch,
3101 module_channel,
3102 module_epoch,
3103 corr: relay_corr,
3104 receiver,
3105 } = pending;
3106 let mut reservation =
3107 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
3108
3109 debug!(
3110 connection_id = ctx.connection_id.get(),
3111 client_channel,
3112 client_epoch,
3113 module_channel,
3114 module_epoch,
3115 "reserved route handle pair"
3116 );
3117 let principal_label = match &principal {
3120 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
3121 Principal::Direct => "direct".to_string(),
3122 other => format!("{other:?}"),
3123 };
3124 let relay = ModuleControlRequest::RouteBind {
3125 route_channel: module_channel,
3126 epoch: module_epoch,
3127 target,
3128 identity,
3129 principal: Some(principal),
3130 consumer_capabilities,
3131 admission_facts,
3132 };
3133 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
3134 RouterError::backend(
3135 0,
3136 frame.header.corr,
3137 format!("failed to encode route.bind request: {err}"),
3138 )
3139 })?;
3140 let relay_frame = Frame::build_with_version(
3141 negotiated_ver,
3142 FrameType::Request,
3143 control_flags(),
3144 0,
3145 0,
3146 relay_corr,
3147 relay_body,
3148 )
3149 .map_err(RouterError::FrameBuild)?;
3150
3151 if let Err(err) = module_sink.send(relay_frame).await {
3152 reservation.release_and_disarm();
3153 return Ok(vec![self.route_open_refusal_frame(
3154 ctx,
3155 &frame,
3156 &target_module_id,
3157 "relay_send_failed",
3158 "target_unavailable",
3159 err.to_string(),
3160 )?]);
3161 }
3162
3163 if !self
3164 .forwarding
3165 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
3166 .map_err(RouterError::Forwarding)?
3167 {
3168 self.send_abandoned_route_bind_goodbye(
3169 &module_sink,
3170 negotiated_ver,
3171 module_channel,
3172 module_epoch,
3173 );
3174 }
3175
3176 match timeout_at(relay_deadline, receiver).await {
3177 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
3178 reservation.disarm();
3179 if breaker.record_accepted() {
3180 info!(
3181 module_id = %target_module_id,
3182 "route.bind breaker closed: the probe was accepted"
3183 );
3184 }
3185 self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
3186 Ok(Vec::new())
3187 }
3188 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
3189 reservation.release_and_disarm();
3190 breaker.record_inconclusive();
3194 self.counters
3195 .increment_route_open_refused("module_rejected");
3196 info!(
3197 target: "control",
3198 code = "module_rejected",
3199 module_code = ?body.code,
3200 module_id = ?target_module_id,
3201 connection_id = ctx.connection_id.get(),
3202 "route.open refused"
3203 );
3204 Ok(vec![control_error_body_frame(&frame, body)?])
3205 }
3206 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
3207 reservation.release_and_disarm();
3208 breaker.record_inconclusive();
3209 tracing::warn!(
3216 module_id = %target_module_id,
3217 "route.bind relay abandoned: {message}"
3218 );
3219 Ok(vec![self.route_open_refusal_frame(
3220 ctx,
3221 &frame,
3222 &target_module_id,
3223 "relay_abandoned",
3224 "target_unavailable",
3225 message,
3226 )?])
3227 }
3228 Ok(Err(_)) => {
3229 reservation.release_and_disarm();
3230 breaker.record_inconclusive();
3231 Ok(vec![self.route_open_refusal_frame(
3232 ctx,
3233 &frame,
3234 &target_module_id,
3235 "relay_waiter_canceled",
3236 "target_unavailable",
3237 "route.bind relay waiter was canceled before the module responded",
3238 )?])
3239 }
3240 Err(_) => {
3241 reservation.release_and_disarm();
3242 if let Some(opened) = breaker.record_timeout(
3246 self.route_bind_breaker_threshold,
3247 self.route_bind_breaker_cooldown,
3248 ) {
3249 warn!(
3250 module_id = %target_module_id,
3251 consecutive_timeouts = opened.consecutive_timeouts,
3252 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
3253 reopened_after_probe = opened.reopened_after_probe,
3254 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
3255 );
3256 }
3257 tracing::warn!(
3265 module_id = %target_module_id,
3266 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
3267 "route.bind relay timed out: module did not ack within budget"
3268 );
3269 Ok(vec![self.route_open_refusal_frame(
3270 ctx,
3271 &frame,
3272 &target_module_id,
3273 "relay_timed_out",
3274 "module_timeout",
3275 format!(
3276 "module_id '{target_module_id}' did not answer route.bind within {:?}",
3277 route_bind_relay_timeout
3278 ),
3279 )?])
3280 }
3281 }
3282 }
3283
3284 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3285 let response = ClientControlResponse::SupervisorSpawnSnapshot {
3286 snapshot: self.supervisor.spawn_snapshot(),
3287 };
3288 Ok(vec![control_response_body_frame(
3289 &frame,
3290 &response,
3291 "ClientControlResponse::SupervisorSpawnSnapshot",
3292 )?])
3293 }
3294
3295 fn handle_supervisor_spawn_subscribe(
3296 &self,
3297 ctx: &RouteCtx,
3298 frame: Frame,
3299 since: Option<SpawnCursor>,
3300 ) -> Result<Vec<Frame>, RouterError> {
3301 match self.supervisor.subscribe_spawns(
3302 ctx.connection_id,
3303 frame.header.corr,
3304 response_version(&frame),
3305 since,
3306 ctx.egress.clone(),
3307 ) {
3308 Ok(()) => Ok(Vec::new()),
3309 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
3310 Ok(vec![control_error_body_frame(
3311 &frame,
3312 ErrorBody {
3313 code: "spawn_cursor_incarnation_mismatch".to_string(),
3314 message: "spawn cursor belongs to a different daemon incarnation"
3315 .to_string(),
3316 detail: Some(serde_json::json!({
3317 "current_daemon_incarnation": current
3318 })),
3319 },
3320 )?])
3321 }
3322 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
3323 &frame,
3324 ErrorBody {
3325 code: "spawn_cursor_too_old".to_string(),
3326 message: "spawn cursor predates the retained event ring".to_string(),
3327 detail: Some(serde_json::json!({
3328 "oldest_retained_cursor": oldest
3329 })),
3330 },
3331 )?]),
3332 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
3333 0,
3334 frame.header.corr,
3335 format!("failed to open supervisor spawn subscription: {error}"),
3336 )),
3337 }
3338 }
3339
3340 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3341 let generation = self
3342 .registry
3343 .generation()
3344 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3345 let mut modules = Vec::new();
3346 for module in self.supervisor.list() {
3347 let status = module.status_for_control("list").map_err(|err| {
3348 RouterError::backend(
3349 0,
3350 frame.header.corr,
3351 format!("failed to read supervisor status: {err}"),
3352 )
3353 })?;
3354 let (configured, _) = module.configuration().map_err(|err| {
3355 RouterError::backend(
3356 0,
3357 frame.header.corr,
3358 format!("failed to read module configuration: {err}"),
3359 )
3360 })?;
3361 let image = module.running_image_agreement().await;
3363 let resources = Some(module.child_resource_usage());
3366 let pending_reload = Some(reload_verdict(
3367 &configured.program,
3368 status.spawned_from.as_deref(),
3369 image,
3370 ));
3371 modules.push(SupervisorEntry {
3372 module_id: status.module_id,
3373 state: status.state.to_string(),
3374 enabled: status.enabled,
3375 live: status.live,
3376 protocol: status.protocol,
3377 health: status.health.status,
3378 pending_reload,
3379 last_probe_ms: status.health.last_probe_ms,
3380 last_exit_code: status.last_exit.as_ref().and_then(|e| e.code),
3381 last_exit_signal: status.last_exit.as_ref().and_then(|e| e.signal),
3382 last_exit_ms: status.last_exit.as_ref().map(|e| e.at_ms),
3383 last_exit_kind: status.last_exit.as_ref().map(|e| e.kind.into()),
3384 restart_count: Some(status.restart_count),
3385 max_restarts: Some(status.max_restarts),
3386 lifetime_restarts: Some(status.lifetime_restarts),
3387 spawn_generation: Some(status.spawn_generation),
3388 restart_window_secs: Some(status.restart_window.as_secs()),
3389 drain_timeout_ms: Some(status.drain_timeout.as_millis() as u64),
3390 restart_backoff_ms: Some(status.restart_backoff.as_millis() as u64),
3391 restart_max_backoff_ms: Some(status.restart_max_backoff.as_millis() as u64),
3392 resources,
3393 });
3394 }
3395 let response = ClientControlResponse::SupervisorList {
3396 generation,
3397 modules,
3398 };
3399 Ok(vec![control_response_body_frame(
3400 &frame,
3401 &response,
3402 "ClientControlResponse::SupervisorList",
3403 )?])
3404 }
3405
3406 fn handle_supervisor_stderr_tail(
3407 &self,
3408 frame: Frame,
3409 module_id: String,
3410 max_lines: Option<u32>,
3411 max_bytes: Option<u32>,
3412 ) -> Result<Vec<Frame>, RouterError> {
3413 let Some(module) = self.supervisor.get(&module_id) else {
3414 return Ok(vec![control_error_frame(
3415 &frame,
3416 "unknown_module",
3417 format!("module_id '{module_id}' is not supervised"),
3418 )?]);
3419 };
3420
3421 let snapshot = module.stderr_tail(
3422 max_lines.map(|value| value as usize),
3423 max_bytes.map(|value| value as usize),
3424 );
3425
3426 let response = ClientControlResponse::SupervisorStderrTail {
3427 module_id,
3428 tail: StderrTail {
3429 capture: match snapshot.capture {
3430 CaptureState::Captured => StderrCaptureState::Captured,
3431 CaptureState::Incomplete { reason } => {
3432 StderrCaptureState::Incomplete { reason }
3433 }
3434 CaptureState::NotCaptured { reason } => {
3435 StderrCaptureState::NotCaptured { reason }
3436 }
3437 },
3438 entries: snapshot
3439 .entries
3440 .into_iter()
3441 .map(|entry| match entry {
3442 TailEntry::Line {
3443 text,
3444 truncated,
3445 at_ms,
3446 } => StderrTailEntry::Line {
3447 text,
3448 truncated,
3449 at_ms,
3450 },
3451 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
3452 })
3453 .collect(),
3454 dropped_lines: snapshot.dropped_lines,
3455 },
3456 };
3457 Ok(vec![control_response_body_frame(
3458 &frame,
3459 &response,
3460 "ClientControlResponse::SupervisorStderrTail",
3461 )?])
3462 }
3463
3464 async fn handle_supervisor_terminals(
3465 &self,
3466 frame: Frame,
3467 module_id: String,
3468 ) -> Result<Vec<Frame>, RouterError> {
3469 let Some(module) = self.supervisor.get(&module_id) else {
3470 return Ok(vec![control_error_frame(
3471 &frame,
3472 "unknown_module",
3473 format!("module_id '{module_id}' is not supervised"),
3474 )?]);
3475 };
3476
3477 let terminals = module
3480 .read_durable_terminal_history()
3481 .await
3482 .map_err(|error| {
3483 RouterError::backend(
3484 0,
3485 frame.header.corr,
3486 format!("failed to read terminal history: {error}"),
3487 )
3488 })?;
3489 let response = ClientControlResponse::SupervisorTerminals {
3490 module_id,
3491 terminals,
3492 };
3493 Ok(vec![control_response_body_frame(
3494 &frame,
3495 &response,
3496 "ClientControlResponse::SupervisorTerminals",
3497 )?])
3498 }
3499
3500 fn handle_supervisor_routes(
3501 &self,
3502 frame: Frame,
3503 module_id: Option<String>,
3504 ) -> Result<Vec<Frame>, RouterError> {
3505 let modules = self
3506 .forwarding
3507 .route_census(module_id.as_deref())
3508 .map_err(RouterError::Forwarding)?
3509 .into_iter()
3510 .map(|(module_id, routes)| SupervisorRouteModule {
3511 module_id,
3512 routes: routes
3513 .into_iter()
3514 .map(|route| SupervisorRoute {
3515 consumer: match route.principal {
3516 Principal::Reserved { module_id } => {
3517 SupervisorRouteConsumer::Reserved { module_id }
3518 }
3519 Principal::Direct | Principal::Unverified => {
3520 SupervisorRouteConsumer::Direct {
3521 connection_id: route.goodbye_target.connection_id.get(),
3522 }
3523 }
3524 },
3525 age_ms: Instant::now()
3526 .saturating_duration_since(route.bound_at)
3527 .as_millis()
3528 .try_into()
3529 .unwrap_or(u64::MAX),
3530 draining: route.draining,
3531 drain_reason: route.drain_reason,
3532 })
3533 .collect(),
3534 })
3535 .collect();
3536 let response = ClientControlResponse::SupervisorRoutes { modules };
3537 Ok(vec![control_response_body_frame(
3538 &frame,
3539 &response,
3540 "ClientControlResponse::SupervisorRoutes",
3541 )?])
3542 }
3543
3544 async fn handle_supervisor_provenance(
3545 &self,
3546 frame: Frame,
3547 module_id: Option<String>,
3548 ) -> Result<Vec<Frame>, RouterError> {
3549 let mut selected = if let Some(module_id) = module_id {
3550 let Some(module) = self.supervisor.get(&module_id) else {
3551 return Ok(vec![control_error_frame(
3552 &frame,
3553 "unknown_module",
3554 format!("module_id '{module_id}' is not supervised"),
3555 )?]);
3556 };
3557 vec![module]
3558 } else {
3559 self.supervisor.list()
3560 };
3561
3562 let mut modules = Vec::with_capacity(selected.len());
3563 for module in selected.drain(..) {
3564 let status = module.status().map_err(|err| {
3565 RouterError::backend(
3566 0,
3567 frame.header.corr,
3568 format!("failed to read supervisor status: {err}"),
3569 )
3570 })?;
3571 let module_declared = self
3572 .registry
3573 .get_module(&status.module_id)
3574 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3575 .and_then(|registration| registration.manifest.provenance)
3576 .map(|build| ModuleDeclaredProvenance::Reported { build })
3577 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
3578 #[cfg(test)]
3579 let running_image = match &self.provenance_probe_override {
3580 Some(result) => result.clone(),
3581 None => module.running_image_agreement().await,
3582 };
3583 #[cfg(not(test))]
3584 let running_image = module.running_image_agreement().await;
3585 modules.push(SupervisorModuleProvenance {
3586 module_id: status.module_id,
3587 module_declared,
3588 daemon_observed: SupervisorObservedProcess {
3589 pid: status.pid,
3590 spawned_at_ms: status.spawned_at_ms,
3591 spawned_from: status.spawned_from,
3592 running_image,
3593 },
3594 });
3595 }
3596 let daemon = SupervisorDaemonProvenance {
3597 daemon_build: self.daemon_provenance.build.clone(),
3598 daemon_observed: DaemonObservedProcess {
3599 pid: self.daemon_provenance.pid,
3600 started_at_ms: self
3601 .daemon_provenance
3602 .start_clock
3603 .map(|clock| clock.started_at_ms())
3604 .or(self.daemon_provenance.started_at_ms),
3605 running_image: self
3606 .daemon_provenance
3607 .probe
3608 .observe(
3609 self.daemon_provenance.pid,
3610 self.daemon_provenance.executable_path.as_deref(),
3611 self.daemon_provenance.executable_identity,
3612 self.daemon_provenance.process_start_time,
3613 )
3614 .await,
3615 },
3616 };
3617 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
3618 Ok(vec![control_response_body_frame(
3619 &frame,
3620 &response,
3621 "ClientControlResponse::SupervisorProvenance",
3622 )?])
3623 }
3624
3625 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3626 self.refresh_capability_requirements();
3627 let generation = self
3628 .registry
3629 .generation()
3630 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3631 let modules = self
3632 .supervisor
3633 .list()
3634 .into_iter()
3635 .map(|module| {
3636 let status = module.status_for_control("health").map_err(|err| {
3637 RouterError::backend(
3638 0,
3639 frame.header.corr,
3640 format!("failed to read supervisor health: {err}"),
3641 )
3642 })?;
3643 let module_id = status.module_id;
3644 let capability_detail = self
3645 .capability_evaluator
3646 .required_problem_detail(&module_id);
3647 Ok(SupervisorHealthEntry {
3648 module_id,
3649 status: status.health.status,
3650 detail: append_capability_problem_detail(
3651 status.health.detail,
3652 capability_detail,
3653 ),
3654 metrics: status.health.metrics,
3655 consecutive_failures: status.health.consecutive_failures,
3656 late_answer_count: status.health.late_answer_count,
3657 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
3658 last_action: status.health.last_action,
3659 last_action_ms: status.health.last_action_ms,
3660 last_probe_ms: status.health.last_probe_ms,
3661 })
3662 })
3663 .collect::<Result<Vec<_>, RouterError>>()?;
3664 let response = ClientControlResponse::SupervisorHealth {
3665 generation,
3666 modules,
3667 };
3668 Ok(vec![control_response_body_frame(
3669 &frame,
3670 &response,
3671 "ClientControlResponse::SupervisorHealth",
3672 )?])
3673 }
3674
3675 async fn handle_supervisor_restart(
3676 &self,
3677 frame: Frame,
3678 module_id: String,
3679 drain_timeout_ms: Option<u64>,
3680 ) -> Result<Vec<Frame>, RouterError> {
3681 let operation_lock = self.supervisor.operation_lock();
3682 let _operation_guard = operation_lock.lock().await;
3683 let Some(module) = self.supervisor.get(&module_id) else {
3684 return Ok(vec![control_error_frame(
3685 &frame,
3686 "unknown_module",
3687 format!("module_id '{module_id}' is not supervised"),
3688 )?]);
3689 };
3690
3691 self.route_outages.mark_operator_action(&module_id);
3692 if let Err(err) = module.restart(drain_timeout_ms).await {
3693 self.route_outages
3694 .operator_action_ended_unrefused(&module_id);
3695 let (code, message) = match err {
3696 crate::supervise::SuperviseError::Disabled { .. } => {
3697 ("module_disabled", err.to_string())
3698 }
3699 crate::supervise::SuperviseError::SwapInProgress { .. } => {
3700 ("swap_in_progress", err.to_string())
3701 }
3702 _ => (
3703 "target_unavailable",
3704 format!("failed to restart module_id '{module_id}': {err}"),
3705 ),
3706 };
3707 return Ok(vec![control_error_frame(&frame, code, message)?]);
3708 }
3709
3710 let response = ClientControlResponse::SupervisorAck {
3711 module_id,
3712 applied: true,
3713 };
3714 Ok(vec![control_response_body_frame(
3715 &frame,
3716 &response,
3717 "ClientControlResponse::SupervisorAck",
3718 )?])
3719 }
3720
3721 async fn handle_supervisor_swap(
3725 &self,
3726 frame: Frame,
3727 module_id: String,
3728 ready_timeout_ms: Option<u64>,
3729 ) -> Result<Vec<Frame>, RouterError> {
3730 let module = {
3737 let operation_lock = self.supervisor.operation_lock();
3738 let _operation_guard = operation_lock.lock().await;
3739 self.supervisor.get(&module_id)
3740 };
3741 let Some(module) = module else {
3742 return Ok(vec![control_error_frame(
3743 &frame,
3744 "unknown_module",
3745 format!("module_id '{module_id}' is not supervised"),
3746 )?]);
3747 };
3748
3749 self.route_outages.mark_operator_action(&module_id);
3750 if let Err(err) = module
3751 .swap(ready_timeout_ms.map(Duration::from_millis))
3752 .await
3753 {
3754 self.route_outages
3755 .operator_action_ended_unrefused(&module_id);
3756 use crate::supervise::SuperviseError;
3757 let message = err.to_string();
3758 let error = match err {
3759 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
3760 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
3761 code: "swap_refused".to_string(),
3762 message,
3763 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
3764 },
3765 SuperviseError::SwapFailed {
3766 arm,
3767 candidate_exit,
3768 ..
3769 } => ErrorBody {
3770 code: "swap_failed".to_string(),
3771 message,
3772 detail: Some(serde_json::json!({
3773 "arm": arm.as_str(),
3774 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
3775 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
3776 })),
3777 },
3778 _ => ErrorBody::new(
3779 "target_unavailable",
3780 format!("failed to swap module_id '{module_id}': {message}"),
3781 ),
3782 };
3783 return Ok(vec![control_error_body_frame(&frame, error)?]);
3784 }
3785 self.route_outages
3789 .operator_action_ended_unrefused(&module_id);
3790
3791 let response = ClientControlResponse::SupervisorAck {
3792 module_id,
3793 applied: true,
3794 };
3795 Ok(vec![control_response_body_frame(
3796 &frame,
3797 &response,
3798 "ClientControlResponse::SupervisorAck",
3799 )?])
3800 }
3801
3802 async fn handle_supervisor_reload(
3803 &self,
3804 frame: Frame,
3805 module_id: String,
3806 ) -> Result<Vec<Frame>, RouterError> {
3807 let operation_lock = self.supervisor.operation_lock();
3808 let _operation_guard = operation_lock.lock().await;
3809 let Some(module) = self.supervisor.get(&module_id) else {
3810 return Ok(vec![control_error_frame(
3811 &frame,
3812 "unknown_module",
3813 format!("module_id '{module_id}' is not supervised"),
3814 )?]);
3815 };
3816
3817 self.route_outages.mark_operator_action(&module_id);
3818 if let Err(err) = module.reload().await {
3819 self.route_outages
3820 .operator_action_ended_unrefused(&module_id);
3821 let (code, message) = match err {
3822 crate::supervise::SuperviseError::Disabled { .. } => {
3823 ("module_disabled", err.to_string())
3824 }
3825 crate::supervise::SuperviseError::SwapInProgress { .. } => {
3826 ("swap_in_progress", err.to_string())
3827 }
3828 _ => (
3829 "reload_failed",
3830 format!("failed to reload module_id '{module_id}': {err}"),
3831 ),
3832 };
3833 return Ok(vec![control_error_frame(&frame, code, message)?]);
3834 }
3835
3836 let response = ClientControlResponse::SupervisorAck {
3837 module_id,
3838 applied: true,
3839 };
3840 Ok(vec![control_response_body_frame(
3841 &frame,
3842 &response,
3843 "ClientControlResponse::SupervisorAck",
3844 )?])
3845 }
3846
3847 async fn handle_supervisor_rescan(
3848 &self,
3849 frame: Frame,
3850 preview: bool,
3851 ) -> Result<Vec<Frame>, RouterError> {
3852 let Some(context) = self.rescan.clone() else {
3853 return Ok(vec![control_error_frame(
3854 &frame,
3855 "rescan_unavailable",
3856 "the daemon was not started with a reloadable config path".to_string(),
3857 )?]);
3858 };
3859
3860 let operation_lock = self.supervisor.operation_lock();
3861 let _operation_guard = operation_lock.lock().await;
3862 let loaded = match crate::daemon_config::load(&context.config_path) {
3863 Ok(config) => config,
3864 Err(err) => {
3865 return Ok(vec![control_error_frame(
3866 &frame,
3867 "invalid_daemon_config",
3868 format!("supervisor rescan rejected daemon config: {err}"),
3869 )?])
3870 }
3871 };
3872 let Some(config) = loaded else {
3880 return Ok(vec![control_error_frame(
3881 &frame,
3882 "invalid_daemon_config",
3883 format!(
3884 "daemon config not found at {}; refusing to rescan (an absent config would \
3885 retire every supervised module)",
3886 context.config_path.display()
3887 ),
3888 )?]);
3889 };
3890 let (
3891 configured_port,
3892 storage_config,
3893 admission_facts_carrier_module_id,
3894 admission_facts_targets,
3895 modules,
3896 reserved_capabilities,
3897 ) = (
3898 config.port,
3899 config.storage,
3900 config.admission_facts_carrier_module_id,
3901 config.admission_facts_targets,
3902 config.modules,
3903 config.reserved_capabilities,
3904 );
3905
3906 let mut restart_required = Vec::new();
3915 for section in RestartRequiredSection::ALL {
3916 let changed = match section {
3917 RestartRequiredSection::Port => configured_port != context.configured_port,
3918 RestartRequiredSection::Storage => storage_config != context.storage_config,
3919 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
3920 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
3921 }
3922 RestartRequiredSection::AdmissionFactsTargets => {
3923 admission_facts_targets != context.admission_facts_targets
3924 }
3925 };
3926 if changed {
3927 restart_required.push(section.label().to_string());
3928 }
3929 }
3930 if !restart_required.is_empty() {
3931 warn!(
3932 config_path = %context.config_path.display(),
3933 sections = %restart_required.join(", "),
3934 "daemon config changed outside the modules section; restart the daemon to apply those changes"
3935 );
3936 }
3937
3938 for configured in &modules {
3939 if let Err(err) = validate_spec(&configured.module_spec()) {
3940 return Ok(vec![control_error_frame(
3941 &frame,
3942 "invalid_daemon_config",
3943 format!("supervisor rescan rejected daemon config: {err}"),
3944 )?]);
3945 }
3946 }
3947
3948 let configured_capabilities = modules
3949 .iter()
3950 .map(|module| (module.module_id.clone(), module.enabled))
3951 .collect::<Vec<_>>();
3952 let preview_capability_warnings = if preview {
3953 let (_, registrations) = self.runtime_capability_snapshot()?;
3954 let current_modules = self
3955 .supervisor
3956 .list()
3957 .into_iter()
3958 .map(|module| module.module_id().to_string())
3959 .collect::<BTreeSet<_>>();
3960 let resulting_modules = configured_capabilities.clone();
3961 let removed = current_modules
3962 .into_iter()
3963 .filter(|module_id| {
3964 !resulting_modules
3965 .iter()
3966 .any(|(configured_id, _)| configured_id == module_id)
3967 })
3968 .collect::<Vec<_>>();
3969 self.capability_evaluator.preview_removal_warnings(
3970 resulting_modules,
3971 &removed,
3972 ®istrations,
3973 )
3974 } else {
3975 Vec::new()
3976 };
3977 let result = match self
3978 .reconcile_supervised_modules(&context.supervisor, modules, preview)
3979 .await
3980 {
3981 Ok(result) => result,
3982 Err(message) => {
3983 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
3984 }
3985 };
3986 if !preview {
3987 self.capability_evaluator
3988 .configure(configured_capabilities, reserved_capabilities);
3989 self.capability_evaluator.wake_deadline_loop();
3990 self.refresh_capability_requirements();
3991 }
3992 let mut result = result;
3993 result.restart_required = restart_required;
3994 result.capability_warnings = preview_capability_warnings;
3995 let response = ClientControlResponse::SupervisorRescan { result };
3996 Ok(vec![control_response_body_frame(
3997 &frame,
3998 &response,
3999 "ClientControlResponse::SupervisorRescan",
4000 )?])
4001 }
4002
4003 async fn handle_supervisor_release_reserved(
4004 &self,
4005 frame: Frame,
4006 module_id: String,
4007 ) -> Result<Vec<Frame>, RouterError> {
4008 let Some(context) = self.rescan.clone() else {
4009 return Ok(vec![control_error_frame(
4010 &frame,
4011 "release_unavailable",
4012 "reserved-id release requires a daemon started with a reloadable config path",
4013 )?]);
4014 };
4015 let operation_lock = self.supervisor.operation_lock();
4016 let _operation_guard = operation_lock.lock().await;
4017 let loaded = match crate::daemon_config::load(&context.config_path) {
4018 Ok(Some(config)) => config,
4019 Ok(None) => {
4020 return Ok(vec![control_error_frame(
4021 &frame,
4022 "invalid_daemon_config",
4023 format!(
4024 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
4025 context.config_path.display()
4026 ),
4027 )?])
4028 }
4029 Err(err) => {
4030 return Ok(vec![control_error_frame(
4031 &frame,
4032 "invalid_daemon_config",
4033 format!("unable to verify reserved-id release against daemon config: {err}"),
4034 )?])
4035 }
4036 };
4037 if loaded
4038 .modules
4039 .iter()
4040 .any(|configured| configured.module_id == module_id)
4041 {
4042 return Ok(vec![control_error_frame(
4043 &frame,
4044 "reserved_module_configured",
4045 format!(
4046 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
4047 ),
4048 )?]);
4049 }
4050 if !self.supervisor.release_retained_reserved_gate(&module_id) {
4051 return Ok(vec![control_error_frame(
4052 &frame,
4053 "reserved_gate_not_retained",
4054 format!(
4055 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
4056 ),
4057 )?]);
4058 }
4059
4060 let response = ClientControlResponse::SupervisorAck {
4061 module_id,
4062 applied: true,
4063 };
4064 Ok(vec![control_response_body_frame(
4065 &frame,
4066 &response,
4067 "ClientControlResponse::SupervisorAck",
4068 )?])
4069 }
4070
4071 async fn reconcile_supervised_modules(
4080 &self,
4081 supervisor: &Supervisor,
4082 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
4083 preview: bool,
4084 ) -> Result<SupervisorRescanResult, String> {
4085 let mut current = BTreeMap::new();
4086 for module in self.supervisor.list() {
4087 let (spec, health) = module.configuration().map_err(|err| {
4088 format!(
4089 "failed to read configuration for module_id '{}': {err}",
4090 module.module_id()
4091 )
4092 })?;
4093 let enabled = module
4094 .status()
4095 .map_err(|err| {
4096 format!(
4097 "failed to read status for module_id '{}': {err}",
4098 module.module_id()
4099 )
4100 })?
4101 .enabled;
4102 current.insert(
4103 module.module_id().to_string(),
4104 (module, spec, health, enabled),
4105 );
4106 }
4107 let configured = configured_modules
4108 .into_iter()
4109 .map(|module| (module.module_id.clone(), module))
4110 .collect::<BTreeMap<_, _>>();
4111
4112 let added = configured
4113 .keys()
4114 .filter(|module_id| !current.contains_key(*module_id))
4115 .cloned()
4116 .collect::<Vec<_>>();
4117 let removed = current
4118 .keys()
4119 .filter(|module_id| !configured.contains_key(*module_id))
4120 .cloned()
4121 .collect::<Vec<_>>();
4122 let mut changed_pending_reload = Vec::new();
4123 let mut configuration_changes = BTreeSet::new();
4124 let mut enabled_changes = BTreeSet::new();
4125 let mut unchanged = 0_u32;
4126
4127 for (module_id, configured_module) in &configured {
4128 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
4129 else {
4130 continue;
4131 };
4132 let configuration_changed = *current_spec != configured_module.module_spec()
4133 || *current_health != configured_module.health;
4134 let enabled_changed = *current_enabled != configured_module.enabled;
4135 if configuration_changed {
4136 configuration_changes.insert(module_id.clone());
4137 changed_pending_reload.push(module_id.clone());
4138 }
4139 if enabled_changed {
4140 enabled_changes.insert(module_id.clone());
4141 }
4142 if !configuration_changed && !enabled_changed {
4143 unchanged = unchanged.saturating_add(1);
4144 }
4145 }
4146
4147 if preview {
4153 return Ok(SupervisorRescanResult {
4154 added,
4155 removed,
4156 changed_pending_reload,
4157 enabled_changes: enabled_changes.iter().cloned().collect(),
4158 unchanged,
4159 preview: true,
4160 restart_required: Vec::new(),
4164 capability_warnings: Vec::new(),
4165 });
4166 }
4167
4168 for module_id in &removed {
4169 let module = ¤t
4170 .get(module_id)
4171 .expect("removed module came from current supervisor state")
4172 .0;
4173 module.retire().await.map_err(|err| {
4174 format!("failed to retire module_id '{module_id}' during rescan: {err}")
4175 })?;
4176 self.supervisor.record_rescan_removal(module_id);
4210 self.supervisor.retire(module_id);
4211 self.route_outages.forget(module_id);
4212 }
4213
4214 for module_id in configured.keys() {
4215 let Some((module, _, _, _)) = current.get(module_id) else {
4216 continue;
4217 };
4218 let configured_module = configured
4219 .get(module_id)
4220 .expect("configured module id came from configured map");
4221 if configuration_changes.contains(module_id) {
4222 module
4223 .update_configuration(
4224 configured_module.module_spec(),
4225 configured_module.health,
4226 configured_module.drain_timeout_ms,
4227 )
4228 .await
4229 .map_err(|err| {
4230 format!(
4231 "failed to update module_id '{module_id}' configuration during rescan: {err}"
4232 )
4233 })?;
4234 }
4235 if enabled_changes.contains(module_id) {
4236 self.route_outages.mark_operator_action(module_id);
4239 module
4240 .set_enabled(configured_module.enabled)
4241 .await
4242 .map_err(|err| {
4243 self.route_outages.operator_action_ended_unrefused(module_id);
4244 format!(
4245 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
4246 configured_module.enabled
4247 )
4248 })?;
4249 }
4250 }
4251
4252 for module_id in &added {
4253 let configured_module = configured
4254 .get(module_id)
4255 .expect("added module id came from configured map");
4256 supervisor
4257 .supervise_configured_with_health(
4258 configured_module.module_spec(),
4259 configured_module.enabled,
4260 configured_module.health,
4261 configured_module.drain_timeout_ms,
4262 configured_module.restart,
4263 )
4264 .map_err(|err| {
4265 format!("failed to add module_id '{module_id}' during rescan: {err}")
4266 })?;
4267 }
4268
4269 Ok(SupervisorRescanResult {
4270 added,
4271 removed,
4272 changed_pending_reload,
4273 enabled_changes: enabled_changes.iter().cloned().collect(),
4274 unchanged,
4275 preview: false,
4276 restart_required: Vec::new(),
4279 capability_warnings: Vec::new(),
4280 })
4281 }
4282
4283 async fn handle_supervisor_set_enabled(
4284 &self,
4285 frame: Frame,
4286 module_id: String,
4287 enabled: bool,
4288 ) -> Result<Vec<Frame>, RouterError> {
4289 let operation_lock = self.supervisor.operation_lock();
4290 let _operation_guard = operation_lock.lock().await;
4291 let Some(module) = self.supervisor.get(&module_id) else {
4292 return Ok(vec![control_error_frame(
4293 &frame,
4294 "unknown_module",
4295 format!("module_id '{module_id}' is not supervised"),
4296 )?]);
4297 };
4298
4299 self.route_outages.mark_operator_action(&module_id);
4302 let applied = match module.set_enabled(enabled).await {
4303 Ok(applied) => applied,
4304 Err(err) => {
4305 self.route_outages
4306 .operator_action_ended_unrefused(&module_id);
4307 return Ok(vec![control_error_frame(
4308 &frame,
4309 "target_unavailable",
4310 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
4311 )?]);
4312 }
4313 };
4314 if !applied {
4315 self.route_outages
4318 .operator_action_ended_unrefused(&module_id);
4319 }
4320
4321 self.capability_evaluator.wake_deadline_loop();
4322 self.refresh_capability_requirements();
4323 let response = ClientControlResponse::SupervisorAck { module_id, applied };
4324 Ok(vec![control_response_body_frame(
4325 &frame,
4326 &response,
4327 "ClientControlResponse::SupervisorAck",
4328 )?])
4329 }
4330
4331 async fn handle_supervisor_health_probe(
4332 &self,
4333 frame: Frame,
4334 module_id: String,
4335 ) -> Result<Vec<Frame>, RouterError> {
4336 self.refresh_capability_requirements();
4337 let Some(registration) = self
4338 .registry
4339 .get_module(&module_id)
4340 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4341 else {
4342 return Ok(vec![control_error_frame(
4343 &frame,
4344 "unknown_module",
4345 format!("module_id '{module_id}' is not registered"),
4346 )?]);
4347 };
4348
4349 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
4362 {
4363 return Ok(vec![control_error_frame(
4364 &frame,
4365 "health_not_advertised",
4366 format!("module_id '{module_id}' did not advertise health.check"),
4367 )?]);
4368 }
4369
4370 let deadline = Instant::now() + self.health_probe_timeout;
4371 let pending = match self.forwarding.begin_module_control_rpc_for(
4372 &module_id,
4373 MODULE_CONTROL_OP_HEALTH_CHECK,
4374 deadline,
4375 ) {
4376 Ok(pending) => pending,
4377 Err(err) => {
4378 return Ok(vec![control_error_frame(
4379 &frame,
4380 forwarding_error_code(&err),
4381 err.to_string(),
4382 )?])
4383 }
4384 };
4385
4386 let PendingModuleControlRpc {
4387 endpoint,
4388 module_sink,
4389 negotiated_ver,
4390 corr: probe_corr,
4391 receiver,
4392 } = pending;
4393 let mut guard =
4394 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
4395 let probe_body =
4396 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
4397 RouterError::backend(
4398 0,
4399 frame.header.corr,
4400 format!("failed to encode health.check request: {err}"),
4401 )
4402 })?;
4403 let probe_frame = Frame::build_with_version(
4404 negotiated_ver,
4405 FrameType::Request,
4406 control_flags(),
4407 0,
4408 0,
4409 probe_corr,
4410 probe_body,
4411 )
4412 .map_err(RouterError::FrameBuild)?;
4413
4414 if let Err(err) = module_sink.send(probe_frame).await {
4415 return Ok(vec![control_error_frame(
4416 &frame,
4417 "target_unavailable",
4418 err.to_string(),
4419 )?]);
4420 }
4421
4422 match timeout_at(deadline, receiver).await {
4423 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
4424 guard.disarm();
4425 let Some(report) = response.health_report() else {
4426 return Ok(vec![control_error_frame(
4427 &frame,
4428 "invalid_control_body",
4429 "health.check RPC returned a non-health response",
4430 )?]);
4431 };
4432 let HealthReport {
4437 status,
4438 detail,
4439 metrics,
4440 } = report;
4441 let capability_detail = self
4442 .capability_evaluator
4443 .required_problem_detail(&module_id);
4444 let response = ClientControlResponse::SupervisorHealthProbe {
4445 module_id,
4446 status,
4447 detail: append_capability_problem_detail(detail, capability_detail),
4448 metrics,
4449 };
4450 Ok(vec![control_response_body_frame(
4451 &frame,
4452 &response,
4453 "ClientControlResponse::SupervisorHealthProbe",
4454 )?])
4455 }
4456 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
4457 guard.disarm();
4458 Ok(vec![control_error_body_frame(&frame, body)?])
4459 }
4460 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
4461 guard.disarm();
4462 Ok(vec![control_error_frame(
4463 &frame,
4464 "target_unavailable",
4465 message,
4466 )?])
4467 }
4468 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
4469 guard.disarm();
4470 Ok(vec![control_error_frame(
4471 &frame,
4472 "invalid_control_body",
4473 message,
4474 )?])
4475 }
4476 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
4477 guard.disarm();
4478 Ok(vec![control_error_frame(
4479 &frame,
4480 "invalid_control_body",
4481 format!("expected module-control op '{expected}', got '{actual}'"),
4482 )?])
4483 }
4484 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
4485 guard.disarm();
4486 Ok(vec![control_error_frame(
4487 &frame,
4488 "module_timeout",
4489 format!(
4490 "module_id '{module_id}' answered health.check after {:?}",
4491 self.health_probe_timeout
4492 ),
4493 )?])
4494 }
4495 Ok(Err(_)) => Ok(vec![control_error_frame(
4496 &frame,
4497 "target_unavailable",
4498 "health.check waiter was canceled before the module responded",
4499 )?]),
4500 Err(_) => Ok(vec![control_error_frame(
4501 &frame,
4502 "module_timeout",
4503 format!(
4504 "module_id '{module_id}' did not answer health.check within {:?}",
4505 self.health_probe_timeout
4506 ),
4507 )?]),
4508 }
4509 }
4510
4511 fn supervisor_status(
4512 &self,
4513 module_id: &str,
4514 corr: u64,
4515 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
4516 self.supervisor
4517 .get(module_id)
4518 .map(|module| {
4519 let warming = module.is_warming_for_control("status").map_err(|err| {
4520 RouterError::backend(
4521 0,
4522 corr,
4523 format!(
4524 "failed to read supervisor warming state for module_id '{module_id}': {err}"
4525 ),
4526 )
4527 })?;
4528 module.status_for_control("status").map_err(|err| {
4529 RouterError::backend(
4530 0,
4531 corr,
4532 format!(
4533 "failed to read supervisor status for module_id '{module_id}': {err}"
4534 ),
4535 )
4536 }).map(|status| (status, warming))
4537 })
4538 .transpose()
4539 }
4540
4541 fn guard_module_control_op(
4542 &self,
4543 frame: &Frame,
4544 module_id: &str,
4545 op: &str,
4546 ) -> Result<Option<Frame>, RouterError> {
4547 if self.module_grants_op(module_id, op, frame.header.corr)? {
4548 return Ok(None);
4549 }
4550
4551 Ok(Some(control_error_frame(
4552 frame,
4553 "op_not_allowed",
4554 format!("module_id '{module_id}' did not grant control op '{op}'"),
4555 )?))
4556 }
4557
4558 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
4559 let Some(registration) = self
4560 .registry
4561 .get_module(module_id)
4562 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
4563 else {
4564 return Ok(false);
4565 };
4566 Ok(module_registration_grants_op(®istration.control_ops, op))
4567 }
4568
4569 fn handle_status_update(
4570 &self,
4571 endpoint: ModuleEndpointId,
4572 frame: Frame,
4573 ) -> Result<Vec<Frame>, RouterError> {
4574 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
4575 Ok(update) => update,
4576 Err(err) => {
4577 if is_known_module_push_op(&frame.body) {
4582 return Ok(vec![control_error_frame(
4583 &frame,
4584 "invalid_control_body",
4585 format!("malformed module control push body: {err}"),
4586 )?]);
4587 }
4588 return Ok(Vec::new());
4589 }
4590 };
4591
4592 match update {
4593 ModuleControlPush::RouteStatus {
4594 route_channel,
4595 route_epoch,
4596 status,
4597 } => {
4598 self.forwarding
4599 .cache_status(endpoint, route_channel, route_epoch, status)
4600 .map_err(RouterError::Forwarding)?;
4601 }
4602 }
4603 Ok(Vec::new())
4604 }
4605
4606 fn handle_route_poll(
4607 &self,
4608 ctx: &RouteCtx,
4609 frame: Frame,
4610 route_channel: u16,
4611 route_epoch: u32,
4612 kind: PollKind,
4613 ) -> Result<Vec<Frame>, RouterError> {
4614 let snapshot = self
4615 .forwarding
4616 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
4617 .map_err(RouterError::Forwarding)?;
4618 let response = match (kind, snapshot) {
4619 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
4620 ClientControlResponse::RoutePoll {
4621 route_channel,
4622 route_epoch,
4623 status,
4624 live: None,
4625 }
4626 }
4627 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
4628 route_channel,
4629 route_epoch,
4630 status: None,
4631 live: None,
4632 },
4633 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
4634 let live = self
4660 .process_liveness
4661 .as_ref()
4662 .and_then(|source| source.process_live(&module_id))
4663 .unwrap_or(true);
4664 ClientControlResponse::RoutePoll {
4665 route_channel,
4666 route_epoch,
4667 status: None,
4668 live: Some(live),
4669 }
4670 }
4671 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
4672 route_channel,
4673 route_epoch,
4674 status: None,
4675 live: Some(false),
4676 },
4677 };
4678
4679 Ok(vec![control_response_body_frame(
4680 &frame,
4681 &response,
4682 "ClientControlResponse::RoutePoll",
4683 )?])
4684 }
4685
4686 pub(crate) fn observe_module_control_completion(
4687 &self,
4688 completion: ModuleControlRpcCompletion,
4689 ) -> bool {
4690 match completion {
4691 ModuleControlRpcCompletion::Unknown => false,
4692 ModuleControlRpcCompletion::Settled => true,
4693 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
4694 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
4695 info!(
4696 module_id = %module_id,
4697 latency_ms,
4698 "late health.check answer proves the module is alive"
4699 );
4700 match self
4701 .supervisor
4702 .record_late_health_answer(&module_id, latency_ms)
4703 {
4704 Ok(true) => {}
4705 Ok(false) => debug!(
4706 module_id = %module_id,
4707 latency_ms,
4708 "late health.check answer has no active supervisor snapshot"
4709 ),
4710 Err(err) => warn!(
4711 module_id = %module_id,
4712 latency_ms,
4713 error = %err,
4714 "failed to record late health.check answer"
4715 ),
4716 }
4717 true
4718 }
4719 }
4720 }
4721
4722 fn refuse_to_end_module_connection_for_a_client(
4743 &self,
4744 module_connection_id: ConnectionId,
4745 corr: u64,
4746 err: ForwardingError,
4747 ) -> Result<(), RouterError> {
4748 if let ForwardingError::ConnectionClosing { connection_id } = err {
4749 if connection_id != module_connection_id {
4750 warn!(
4751 module_connection_id = module_connection_id.get(),
4752 client_connection_id = connection_id.get(),
4753 corr,
4754 "dropping a route.bind response for a closing client; the module connection keeps serving"
4755 );
4756 return Ok(());
4757 }
4758 }
4759 Err(RouterError::Forwarding(err))
4760 }
4761
4762 fn handle_module_relay_response(
4763 &self,
4764 connection_id: ConnectionId,
4765 frame: Frame,
4766 ) -> Result<Vec<Frame>, RouterError> {
4767 let mut secondary_error = None;
4768 let outcome = match frame.header.ty {
4769 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
4770 Ok(probe) if probe.op == "route.bind" => {
4771 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
4772 Ok(ModuleControlResponse::RouteBindAck {}) => {
4773 RouteBindRelayOutcome::Accepted
4774 }
4775 Ok(other) => {
4776 let message =
4777 format!("route.bind response carried unexpected body: {other:?}");
4778 secondary_error = Some(control_error_frame(
4779 &frame,
4780 "invalid_control_body",
4781 message.clone(),
4782 )?);
4783 RouteBindRelayOutcome::ModuleGone(message)
4784 }
4785 Err(err) => {
4786 let message = format!("malformed route.bind response body: {err}");
4787 secondary_error = Some(control_error_frame(
4788 &frame,
4789 "invalid_control_body",
4790 message.clone(),
4791 )?);
4792 RouteBindRelayOutcome::ModuleGone(message)
4793 }
4794 }
4795 }
4796 Ok(probe) => {
4797 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
4798 {
4799 Ok(response) => ModuleControlRpcOutcome::Response(response),
4800 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
4801 "malformed {} response body: {err}",
4802 probe.op
4803 )),
4804 };
4805 let completion = self
4806 .forwarding
4807 .complete_module_control_rpc(
4808 connection_id,
4809 frame.header.corr,
4810 Some(&probe.op),
4811 outcome,
4812 )
4813 .map_err(RouterError::Forwarding)?;
4814 if !self.observe_module_control_completion(completion) {
4815 debug!(
4816 connection_id = connection_id.get(),
4817 corr = frame.header.corr,
4818 op = %probe.op,
4819 "dropping late or unknown module-control RPC response"
4820 );
4821 }
4822 return Ok(Vec::new());
4823 }
4824 Err(err) => {
4825 if let Some(expected_op) = self
4826 .forwarding
4827 .pending_module_control_op(connection_id, frame.header.corr)
4828 .map_err(RouterError::Forwarding)?
4829 {
4830 let completion = self
4831 .forwarding
4832 .complete_module_control_rpc(
4833 connection_id,
4834 frame.header.corr,
4835 None,
4836 ModuleControlRpcOutcome::MalformedResponse(format!(
4837 "malformed {expected_op} response body: {err}"
4838 )),
4839 )
4840 .map_err(RouterError::Forwarding)?;
4841 if !self.observe_module_control_completion(completion) {
4842 debug!(
4843 connection_id = connection_id.get(),
4844 corr = frame.header.corr,
4845 "dropping late malformed module-control RPC response"
4846 );
4847 }
4848 return Ok(Vec::new());
4849 }
4850 let message = format!("malformed route.bind response body: {err}");
4851 secondary_error = Some(control_error_frame(
4852 &frame,
4853 "invalid_control_body",
4854 message.clone(),
4855 )?);
4856 RouteBindRelayOutcome::ModuleGone(message)
4857 }
4858 },
4859 FrameType::Error => {
4860 if self
4861 .forwarding
4862 .pending_module_control_op(connection_id, frame.header.corr)
4863 .map_err(RouterError::Forwarding)?
4864 .is_some()
4865 {
4866 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
4867 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
4868 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
4869 "malformed module-control ERROR body: {err}"
4870 )),
4871 };
4872 let completion = self
4873 .forwarding
4874 .complete_module_control_rpc(
4875 connection_id,
4876 frame.header.corr,
4877 None,
4878 outcome,
4879 )
4880 .map_err(RouterError::Forwarding)?;
4881 if !self.observe_module_control_completion(completion) {
4882 debug!(
4883 connection_id = connection_id.get(),
4884 corr = frame.header.corr,
4885 "dropping late or unknown module-control RPC error"
4886 );
4887 }
4888 return Ok(Vec::new());
4889 }
4890 match serde_json::from_slice::<ErrorBody>(&frame.body) {
4891 Ok(body) => RouteBindRelayOutcome::Rejected(body),
4892 Err(err) => {
4893 let message = format!("malformed route.bind ERROR body: {err}");
4894 secondary_error = Some(control_error_frame(
4895 &frame,
4896 "invalid_control_body",
4897 message.clone(),
4898 )?);
4899 RouteBindRelayOutcome::ModuleGone(message)
4900 }
4901 }
4902 }
4903 ty => {
4904 return Ok(vec![control_error_frame(
4905 &frame,
4906 "unsupported_control_frame",
4907 format!("unsupported module channel-0 frame {ty:?}"),
4908 )?])
4909 }
4910 };
4911
4912 let settled =
4913 self.forwarding
4914 .complete_pending_relay(connection_id, frame.header.corr, outcome);
4915 let completion = match settled {
4916 Ok(completion) => completion,
4917 Err(err) => {
4918 self.refuse_to_end_module_connection_for_a_client(
4919 connection_id,
4920 frame.header.corr,
4921 err,
4922 )?;
4923 return Ok(secondary_error.into_iter().collect());
4924 }
4925 };
4926 if let Some(target) = completion.abandoned.as_ref() {
4927 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
4928 }
4929 if !completion.settled {
4930 debug!(
4931 connection_id = connection_id.get(),
4932 corr = frame.header.corr,
4933 frame_type = ?frame.header.ty,
4934 "dropping late or unknown route.bind relay response"
4935 );
4936 }
4937 Ok(secondary_error.into_iter().collect())
4938 }
4939
4940 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
4941 debug!(connection_id = connection_id.get(), "handling GOODBYE");
4942 let registrations = self
4943 .deregister_connection(connection_id)
4944 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
4945 let released_routes = self
4946 .forwarding
4947 .cleanup_connection(connection_id)
4948 .map_err(RouterError::Forwarding)?;
4949 self.emit_route_goodbyes(released_routes);
4950 if !registrations.is_empty() {
4952 crate::supervise::notify_registration_release();
4953 }
4954 Ok(Vec::new())
4955 }
4956}
4957
4958impl Default for ControlHandler {
4959 fn default() -> Self {
4960 Self::new(Arc::new(Registry::default()))
4961 }
4962}
4963
4964impl crate::supervise::SwapPromotionObserver for ControlHandler {
4965 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
4966 self.apply_registration_capabilities(registration);
4967 }
4968}
4969
4970fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
4971 CapabilityRequirementStatus {
4972 consumer: status.consumer,
4973 capability: status.capability,
4974 need: match status.need {
4975 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
4976 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
4977 },
4978 verdict: status.verdict.as_str().to_string(),
4979 episode_seq: status.episode_seq,
4980 config_satisfiable: status.config_satisfiable,
4981 runtime_available: status.runtime_available,
4982 detail: status.detail,
4983 }
4984}
4985
4986fn append_capability_problem_detail(
4987 detail: Option<String>,
4988 capability_detail: Option<String>,
4989) -> Option<String> {
4990 match (detail, capability_detail) {
4991 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
4992 (Some(detail), None) => Some(detail),
4993 (None, Some(capability_detail)) => Some(capability_detail),
4994 (None, None) => None,
4995 }
4996}
4997
4998fn subc_ops() -> Vec<String> {
4999 SUBC_CONTROL_OPS
5000 .iter()
5001 .map(|op| (*op).to_string())
5002 .collect()
5003}
5004
5005fn module_subc_ops() -> Vec<String> {
5006 SUBC_CONTROL_OPS
5007 .iter()
5008 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
5009 .map(|op| (*op).to_string())
5010 .collect()
5011}
5012
5013#[cfg(test)]
5014fn module_baseline_control_ops() -> Vec<String> {
5015 MODULE_BASELINE_CONTROL_OPS
5016 .iter()
5017 .map(|op| (*op).to_string())
5018 .collect()
5019}
5020
5021fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
5022 let mut seen = HashSet::new();
5023 let mut effective = Vec::new();
5024 for op in MODULE_BASELINE_CONTROL_OPS {
5025 if seen.insert((*op).to_string()) {
5026 effective.push((*op).to_string());
5027 }
5028 }
5029 for op in declared.unwrap_or_default() {
5030 if seen.insert(op.clone()) {
5031 effective.push(op);
5032 }
5033 }
5034 effective
5035}
5036
5037fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
5038 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
5039}
5040
5041fn target_module_id(target: &RouteTarget) -> &str {
5042 match target {
5043 RouteTarget::ToolProvider { module_id }
5044 | RouteTarget::ManagementSurface { module_id }
5045 | RouteTarget::InternalService { module_id, .. } => module_id,
5046 }
5047}
5048
5049fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
5050 roles.iter().any(|role| match (target, role) {
5051 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
5052 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
5053 (
5054 RouteTarget::InternalService { service_id, .. },
5055 ProviderRole::InternalService {
5056 service_id: provided,
5057 ..
5058 },
5059 ) => service_id == provided,
5060 _ => false,
5061 })
5062}
5063
5064fn is_routable_role(role: &ProviderRole) -> bool {
5065 matches!(
5066 role,
5067 ProviderRole::ToolProvider { .. }
5068 | ProviderRole::ManagementSurface { .. }
5069 | ProviderRole::InternalService { .. }
5070 )
5071}
5072
5073#[derive(Debug, Clone, Copy, PartialEq, Eq)]
5074enum ControlRequestBodyError {
5075 UnknownOp,
5076 InvalidBody,
5077}
5078
5079#[derive(Debug, Deserialize)]
5080struct ControlOpProbe {
5081 op: String,
5082}
5083
5084const MODULE_PUSH_OPS: &[&str] = &["route.status"];
5087
5088fn is_known_module_push_op(body: &[u8]) -> bool {
5089 serde_json::from_slice::<ControlOpProbe>(body)
5090 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
5091 .unwrap_or(false)
5092}
5093
5094fn is_known_module_request_op(body: &[u8]) -> bool {
5095 serde_json::from_slice::<ControlOpProbe>(body)
5096 .map(|probe| MODULE_TO_SUBC_CONTROL_OPS.contains(&probe.op.as_str()))
5097 .unwrap_or(false)
5098}
5099
5100fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
5101 debug!(
5102 op = %op,
5103 connection_id = connection_id.get(),
5104 corr,
5105 "control dispatch"
5106 );
5107}
5108
5109fn log_slow_control_dispatch(
5110 dispatch_started_at: Option<StdInstant>,
5111 op: &'static str,
5112 connection_id: ConnectionId,
5113 corr: u64,
5114) {
5115 let Some(dispatch_started_at) = dispatch_started_at else {
5116 return;
5117 };
5118 let elapsed = dispatch_started_at.elapsed();
5119 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
5120 warn!(
5121 op = %op,
5122 connection_id = connection_id.get(),
5123 corr,
5124 elapsed_ms = elapsed.as_millis() as u64,
5125 "slow control dispatch"
5126 );
5127 }
5128}
5129
5130fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
5131 match request {
5132 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
5133 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
5134 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
5135 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
5136 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
5137 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
5138 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
5139 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
5140 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
5141 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
5142 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
5143 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
5144 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
5145 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
5146 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
5147 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
5148 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
5149 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
5150 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
5151 }
5152}
5153
5154fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
5155 match request {
5156 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
5157 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
5158 }
5159}
5160
5161fn parse_client_control_request(
5162 body: &[u8],
5163) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
5164 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
5165 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5166 Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
5167 ControlRequestBodyError::InvalidBody
5168 }
5169 Ok(_) => ControlRequestBodyError::UnknownOp,
5170 Err(_) => ControlRequestBodyError::InvalidBody,
5171 };
5172 (err, classification)
5173 })
5174}
5175
5176fn parse_module_control_request_from_module(
5177 body: &[u8],
5178) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
5179 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
5180 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5181 Ok(probe) if MODULE_TO_SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
5182 ControlRequestBodyError::InvalidBody
5183 }
5184 Ok(_) => ControlRequestBodyError::UnknownOp,
5185 Err(_) => ControlRequestBodyError::InvalidBody,
5186 };
5187 (err, classification)
5188 })
5189}
5190
5191#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
5192enum ProviderRoleKind {
5193 ToolProvider,
5194 PipelineStage,
5195 ManagementSurface,
5196 InternalService,
5197}
5198
5199fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
5200 match role {
5201 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
5202 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
5203 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
5204 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
5205 }
5206}
5207
5208fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
5209 roles.iter().map(provider_role_kind).collect()
5210}
5211
5212fn capability_census_trigger(
5216 old: Option<&CapabilityDeclarations>,
5217 new: Option<&CapabilityDeclarations>,
5218) -> bool {
5219 let old_provides = old
5220 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
5221 .unwrap_or_default();
5222 let old_denies = old
5223 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
5224 .unwrap_or_default();
5225 let new = new.cloned().unwrap_or(CapabilityDeclarations {
5226 provides: Vec::new(),
5227 requires: Vec::new(),
5228 must_never_reach: Vec::new(),
5229 });
5230
5231 new.provides
5232 .iter()
5233 .any(|capability| !old_provides.contains(capability))
5234 || new
5235 .must_never_reach
5236 .iter()
5237 .any(|capability| !old_denies.contains(capability))
5238}
5239
5240fn denied_capability<'a>(
5243 opening_manifest: &'a ModuleManifest,
5244 target_manifest: &ModuleManifest,
5245) -> Option<&'a str> {
5246 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
5247 let target_capabilities = target_manifest.capabilities.as_ref()?;
5248 opening_capabilities
5249 .must_never_reach
5250 .iter()
5251 .find(|denied| {
5252 target_capabilities
5253 .provides
5254 .iter()
5255 .any(|provided| provided == *denied)
5256 })
5257 .map(String::as_str)
5258}
5259
5260fn catalog_update_frozen_field_message(
5261 registered: &ModuleManifest,
5262 provides: &[ProviderRole],
5263) -> Option<String> {
5264 let old_has_provides = !registered.provides.is_empty();
5265 let new_has_provides = !provides.is_empty();
5266 if old_has_provides != new_has_provides {
5267 return Some(format!(
5268 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
5269 registered.module_id
5270 ));
5271 }
5272
5273 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
5274 return Some(format!(
5275 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
5276 registered.module_id
5277 ));
5278 }
5279
5280 let registered_concurrency = manifest_concurrency(registered);
5281 let mut candidate = registered.clone();
5282 candidate.provides = provides.to_vec();
5283 let candidate_concurrency = manifest_concurrency(&candidate);
5284 if candidate_concurrency != registered_concurrency {
5285 return Some(format!(
5286 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
5287 registered.module_id, registered_concurrency, candidate_concurrency
5288 ));
5289 }
5290
5291 None
5294}
5295
5296fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
5297 manifest.provides.iter().any(is_routable_role)
5298}
5299
5300fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
5306 manifest
5307 .provides
5308 .iter()
5309 .find_map(|provider| match provider {
5310 ProviderRole::ToolProvider { concurrency, .. }
5311 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
5312 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
5313 })
5314 .unwrap_or(Concurrency::ModuleManaged)
5315}
5316
5317fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
5324 let has_management_surface = manifest
5325 .provides
5326 .iter()
5327 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
5328 if !has_management_surface {
5329 return false;
5330 }
5331 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
5332 return false;
5333 };
5334 let Some(provides) = raw
5335 .get("manifest")
5336 .and_then(|manifest| manifest.get("provides"))
5337 .and_then(serde_json::Value::as_array)
5338 else {
5339 return false;
5340 };
5341 provides.iter().any(|role| {
5346 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
5347 && role.get("concurrency").is_none()
5348 })
5349}
5350
5351fn negotiate_version(peer_version: u8) -> Result<u8, String> {
5352 if peer_version != PROTOCOL_VERSION {
5353 return Err(format!(
5354 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
5355 ));
5356 }
5357 Ok(PROTOCOL_VERSION)
5358}
5359
5360fn pong(frame: &Frame) -> Result<Frame, RouterError> {
5361 Frame::build_with_version(
5362 response_version(frame),
5363 FrameType::Pong,
5364 frame.header.flags,
5365 0,
5366 0,
5367 frame.header.corr,
5368 Vec::new(),
5369 )
5370 .map_err(RouterError::FrameBuild)
5371}
5372
5373fn control_error_frame(
5374 frame: &Frame,
5375 code: &'static str,
5376 message: impl Into<String>,
5377) -> Result<Frame, RouterError> {
5378 control_error_body_frame(
5379 frame,
5380 ErrorBody {
5381 code: code.to_string(),
5382 message: message.into(),
5383 detail: None,
5384 },
5385 )
5386}
5387
5388fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
5389 let body = serde_json::to_vec(&error).map_err(|err| {
5390 RouterError::backend(
5391 0,
5392 frame.header.corr,
5393 format!("failed to encode control ERROR: {err}"),
5394 )
5395 })?;
5396
5397 Frame::build_with_version(
5398 response_version(frame),
5399 FrameType::Error,
5400 control_flags(),
5401 0,
5402 0,
5403 frame.header.corr,
5404 body,
5405 )
5406 .map_err(RouterError::FrameBuild)
5407}
5408
5409fn control_response_body_frame<T: Serialize>(
5410 frame: &Frame,
5411 reply: &T,
5412 label: &'static str,
5413) -> Result<Frame, RouterError> {
5414 let body = serde_json::to_vec(reply).map_err(|err| {
5415 RouterError::backend(
5416 0,
5417 frame.header.corr,
5418 format!("failed to encode {label}: {err}"),
5419 )
5420 })?;
5421
5422 Frame::build_with_version(
5423 response_version(frame),
5424 FrameType::Response,
5425 control_flags(),
5426 0,
5427 0,
5428 frame.header.corr,
5429 body,
5430 )
5431 .map_err(RouterError::FrameBuild)
5432}
5433
5434fn forwarding_error_code(err: &ForwardingError) -> &'static str {
5464 match err {
5465 ForwardingError::NoModuleConnection => "target_unavailable",
5466 ForwardingError::ModuleReloading { .. } => "module_reloading",
5467 ForwardingError::ClientRouteChannelExhausted { .. }
5468 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
5469 ForwardingError::StaleModuleEndpoint
5470 | ForwardingError::UnknownReservation { .. }
5471 | ForwardingError::ConnectionClosing { .. }
5472 | ForwardingError::ClientEgressClosed { .. }
5473 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
5474 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
5477 ForwardingError::RelayCorrelationExhausted
5478 | ForwardingError::RouteOpenBuild(_)
5479 | ForwardingError::Poisoned => "forwarding_error",
5480 }
5481}
5482
5483fn response_version(frame: &Frame) -> u8 {
5484 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
5485 frame.header.ver
5486 } else {
5487 PROTOCOL_VERSION
5488 }
5489}
5490
5491fn control_flags() -> Flags {
5492 Flags::new(false, Priority::Passive, false)
5493}
5494
5495fn send_goodbye_target_best_effort(
5500 counters: &DaemonCounters,
5501 target: &GoodbyeTarget,
5502 context: &'static str,
5503) {
5504 let Ok(frame) = Frame::build_with_version(
5505 target.negotiated_ver,
5506 FrameType::Goodbye,
5507 control_flags(),
5508 target.channel,
5509 target.epoch,
5510 0,
5511 Vec::new(),
5512 ) else {
5513 return;
5514 };
5515 crate::forwarding::send_module_route_goodbye(
5516 counters,
5517 &target.sink,
5518 frame,
5519 target.module_id.as_deref(),
5520 context,
5521 );
5522}
5523
5524pub(crate) fn send_route_control_pushes(
5525 forwarding: &ForwardingTable,
5526 routes: Vec<EndpointRoute>,
5527 push: ClientControlPush,
5528) {
5529 let body = match serde_json::to_vec(&push) {
5530 Ok(body) => body,
5531 Err(err) => {
5532 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
5533 return;
5534 }
5535 };
5536 let mut targets = Vec::new();
5537 for route in routes {
5538 let target = route.goodbye_target;
5539 if let Some(existing) = targets
5540 .iter()
5541 .find(|existing: &&GoodbyeTarget| existing.connection_id == target.connection_id)
5542 {
5543 debug_assert_eq!(
5544 existing.negotiated_ver, target.negotiated_ver,
5545 "one connection cannot negotiate multiple frame versions"
5546 );
5547 continue;
5548 }
5549 targets.push(target);
5550 }
5551 for target in targets {
5552 let frame = match Frame::build_with_version(
5553 target.negotiated_ver,
5554 FrameType::Push,
5555 control_flags(),
5556 0,
5557 0,
5558 0,
5559 body.clone(),
5560 ) {
5561 Ok(frame) => frame,
5562 Err(err) => {
5563 warn!(
5564 route_channel = target.channel,
5565 error = %err,
5566 "failed to build route lifecycle control PUSH frame"
5567 );
5568 continue;
5569 }
5570 };
5571 if let Err(err) = target.sink.try_send(frame) {
5572 if target.close_on_delivery_failure() {
5573 warn!(
5574 target_connection_id = target.connection_id.get(),
5575 route_channel = target.channel,
5576 error = %err,
5577 "route lifecycle control PUSH was not delivered to client; closing target connection"
5578 );
5579 let _ = forwarding.escalate_client_delivery_failure(
5580 target.connection_id,
5581 target.channel,
5582 target.epoch,
5583 CloseReason::new(
5584 "route_lifecycle_push_delivery_failed",
5585 format!(
5586 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
5587 target.channel
5588 ),
5589 ),
5590 crate::forwarding::UndeliveredFrame {
5591 module_id: target.module_id.as_deref(),
5592 sink: &target.sink,
5593 },
5594 );
5595 }
5596 }
5597 }
5598}
5599
5600#[cfg(test)]
5601mod tests {
5602 use std::{
5603 collections::BTreeMap,
5604 fmt,
5605 path::PathBuf,
5606 sync::{Arc, Mutex},
5607 time::Duration,
5608 };
5609 use subc_test_support::TestTempDir;
5610
5611 use serde_json::{json, Value};
5612 use subc_protocol::{
5613 manifest::{
5614 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
5615 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
5616 },
5617 session::HealthStatus,
5618 FrameType,
5619 };
5620
5621 use super::*;
5622 use crate::{
5623 forwarding::{DataRoute, DataRouteState},
5624 registry::ChannelState,
5625 router::FrameSink,
5626 stderr_tail::DEFAULT_MAX_LINE_BYTES,
5627 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
5628 RouteCtx, Router,
5629 };
5630 use tokio::{
5631 sync::mpsc,
5632 time::{sleep, Instant},
5633 };
5634 use tracing::{
5635 field::{Field, Visit},
5636 Event, Subscriber,
5637 };
5638 use tracing_subscriber::{layer::Context, prelude::*, Layer};
5639
5640 fn fake_aft_stub_path() -> PathBuf {
5659 let mut path = std::env::current_exe().expect("current_exe available in tests");
5660 path.pop(); path.pop(); path.push(if cfg!(windows) {
5663 "fake-aft-stub.exe"
5664 } else {
5665 "fake-aft-stub"
5666 });
5667 assert!(
5668 path.exists(),
5669 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
5670 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
5671 path.display()
5672 );
5673 path
5674 }
5675
5676 fn client_retries(code: &str) -> bool {
5681 subc_protocol::error_codes::is_retryable_route_open(code)
5682 }
5683
5684 #[test]
5695 fn retryability_of_forwarding_codes_matches_the_failure() {
5696 let transient = [
5699 ForwardingError::NoModuleConnection,
5700 ForwardingError::ModuleReloading {
5701 module_id: "m".into(),
5702 },
5703 ForwardingError::StaleModuleEndpoint,
5704 ForwardingError::UnknownReservation {
5705 client_channel: 1,
5706 module_channel: 1,
5707 },
5708 ForwardingError::ConnectionClosing {
5709 connection_id: ConnectionId::new(1),
5710 },
5711 ForwardingError::ClientEgressClosed {
5712 connection_id: ConnectionId::new(1),
5713 },
5714 ForwardingError::ModuleEgressUnavailable {
5715 connection_id: ConnectionId::new(1),
5716 },
5717 ];
5718 for err in transient {
5719 let code = forwarding_error_code(&err);
5720 assert!(
5721 client_retries(code),
5722 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
5723 );
5724 }
5725
5726 let permanent = [
5731 ForwardingError::ClientRouteChannelExhausted {
5732 connection_id: ConnectionId::new(1),
5733 },
5734 ForwardingError::ModuleRouteChannelExhausted {
5735 endpoint: ModuleEndpointId {
5736 connection_id: ConnectionId::new(1),
5737 generation: 1,
5738 },
5739 },
5740 ForwardingError::RelayCorrelationExhausted,
5741 ForwardingError::RouteOpenBuild("x".into()),
5742 ForwardingError::Poisoned,
5743 ];
5744 for err in permanent {
5745 let code = forwarding_error_code(&err);
5746 assert!(
5747 !client_retries(code),
5748 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
5749 );
5750 }
5751 }
5752
5753 #[tokio::test]
5781 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
5782 let handler = ControlHandler::default();
5783 let frame =
5784 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
5785
5786 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
5790 assert_eq!(
5791 stamped,
5792 Principal::Direct,
5793 "a caller that proved nothing must not be stamped as a supervised module"
5794 );
5795
5796 let forged = handler
5801 .route_open_principal(
5802 &frame,
5803 Some(ConsumerIdentity {
5804 module_id: "aft".to_string(),
5805 launch_nonce: "not-a-real-nonce".to_string(),
5806 }),
5807 )
5808 .unwrap();
5809 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
5810 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
5811 }
5812
5813 #[test]
5830 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
5831 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
5832 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
5833 let ClientControlRequest::RouteOpen {
5834 consumer_identity, ..
5835 } = parsed
5836 else {
5837 panic!("route.open body must parse as RouteOpen");
5838 };
5839 assert_eq!(
5840 consumer_identity,
5841 Some(ConsumerIdentity {
5842 module_id: "aft".to_string(),
5843 launch_nonce: "n".to_string(),
5844 }),
5845 "the wire field name must reach the value route_open_principal reads"
5846 );
5847 }
5848
5849 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
5850 ModuleManifest::builder(module_id, "0.1.0")
5851 .protocol_ver(protocol_ver)
5852 .provides(vec![ProviderRole::ToolProvider {
5853 tools: vec![Tool {
5854 name: "read".to_string(),
5855 description: None,
5856 execution_mode: ExecutionMode::Pure,
5857 schema: json!({"type": "object"}),
5858 }],
5859 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
5860 concurrency: Concurrency::ModuleManaged,
5861 emits_push: true,
5862 sub_supervises: true,
5863 }])
5864 .build()
5865 }
5866
5867 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
5868 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
5869 }
5870
5871 fn hello_frame_with_control_ops(
5872 module_id: &str,
5873 protocol_ver: u8,
5874 corr: u64,
5875 control_ops: Option<Vec<String>>,
5876 ) -> Frame {
5877 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
5878 }
5879
5880 fn hello_frame_with_nonce(
5881 module_id: &str,
5882 protocol_ver: u8,
5883 corr: u64,
5884 launch_nonce: Option<&str>,
5885 ) -> Frame {
5886 hello_frame_full(
5887 module_id,
5888 protocol_ver,
5889 corr,
5890 None,
5891 launch_nonce.map(ToOwned::to_owned),
5892 )
5893 }
5894
5895 fn hello_frame_full(
5896 module_id: &str,
5897 protocol_ver: u8,
5898 corr: u64,
5899 control_ops: Option<Vec<String>>,
5900 launch_nonce: Option<String>,
5901 ) -> Frame {
5902 let body = serde_json::to_vec(&ModuleHelloBody {
5903 manifest: manifest(module_id, protocol_ver),
5904 protocol_ver,
5905 control_ops,
5906 launch_nonce,
5907 })
5908 .unwrap();
5909 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
5910 }
5911
5912 fn non_routable_hello_frame_with_control_ops(
5913 module_id: &str,
5914 corr: u64,
5915 control_ops: Option<Vec<String>>,
5916 ) -> Frame {
5917 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
5918 manifest.provides.clear();
5919 let body = serde_json::to_vec(&ModuleHelloBody {
5920 manifest,
5921 protocol_ver: PROTOCOL_VERSION,
5922 control_ops,
5923 launch_nonce: None,
5924 })
5925 .unwrap();
5926 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
5927 }
5928
5929 fn capability_grammar_hello_frame(
5930 capabilities: Value,
5931 runtime_computed: Option<Value>,
5932 corr: u64,
5933 ) -> Frame {
5934 let mut body = serde_json::to_value(ModuleHelloBody {
5935 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
5936 protocol_ver: PROTOCOL_VERSION,
5937 control_ops: None,
5938 launch_nonce: None,
5939 })
5940 .expect("HELLO body serializes");
5941 body["manifest"]["capabilities"] = capabilities;
5942 if let Some(runtime_computed) = runtime_computed {
5943 body["runtime_computed"] = runtime_computed;
5944 }
5945 Frame::build(
5946 FrameType::Hello,
5947 control_flags(),
5948 0,
5949 0,
5950 corr,
5951 serde_json::to_vec(&body).expect("HELLO body reserializes"),
5952 )
5953 .expect("HELLO frame builds")
5954 }
5955
5956 fn channel_request(channel: u16, corr: u64) -> Frame {
5957 Frame::build(
5958 FrameType::Request,
5959 Flags::new(true, Priority::Interactive, false),
5960 channel,
5961 0,
5962 corr,
5963 b"opaque".to_vec(),
5964 )
5965 .unwrap()
5966 }
5967
5968 fn route_ctx(
5969 connection_id: ConnectionId,
5970 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
5971 let (tx, rx) = mpsc::channel(8);
5972 (
5973 RouteCtx {
5974 connection_id,
5975 egress: FrameSink::new(tx),
5976 },
5977 rx,
5978 )
5979 }
5980
5981 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
5982 serde_json::from_slice(&frame.body).unwrap()
5983 }
5984
5985 async fn hello_via_sink(
5990 handler: &ControlHandler,
5991 ctx: &RouteCtx,
5992 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
5993 hello: Frame,
5994 ) -> Frame {
5995 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
5996 assert!(
5997 replies.is_empty(),
5998 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
5999 );
6000 let ack = rx
6001 .try_recv()
6002 .expect("HELLO_ACK is queued on the module sink")
6003 .frame;
6004 assert_eq!(ack.header.ty, FrameType::HelloAck);
6005 ack
6006 }
6007
6008 fn parse_error(frame: &Frame) -> Value {
6009 serde_json::from_slice(&frame.body).unwrap()
6010 }
6011
6012 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
6013 serde_json::from_slice(&frame.body).unwrap()
6014 }
6015
6016 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
6017 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
6018 route_channel,
6019 route_epoch: 0,
6020 kind,
6021 })
6022 .unwrap();
6023 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6024 }
6025
6026 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
6027 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
6028 module_id: module_id.to_string(),
6029 })
6030 .unwrap();
6031 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6032 }
6033
6034 fn route_open_frame(corr: u64, module_id: &str, project_root: TestTempDir) -> Frame {
6035 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
6036 }
6037
6038 fn route_open_frame_with_consumer_capabilities(
6039 corr: u64,
6040 module_id: &str,
6041 project_root: TestTempDir,
6042 consumer_capabilities: Option<Vec<String>>,
6043 ) -> Frame {
6044 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6045 target: RouteTarget::ToolProvider {
6046 module_id: module_id.to_string(),
6047 },
6048 identity: BindIdentity::new(
6049 project_root.path().to_path_buf(),
6050 "unit".to_string(),
6051 "session".to_string(),
6052 ),
6053 consumer_identity: None,
6054 consumer_capabilities,
6055 admission_facts: None,
6056 })
6057 .unwrap();
6058 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6059 }
6060
6061 fn route_open_frame_with_admission_facts(
6062 corr: u64,
6063 module_id: &str,
6064 project_root: TestTempDir,
6065 consumer_identity: Option<subc_control::ConsumerIdentity>,
6066 facts: Option<Value>,
6067 ) -> Frame {
6068 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6069 target: RouteTarget::ToolProvider {
6070 module_id: module_id.to_string(),
6071 },
6072 identity: BindIdentity::new(
6073 project_root.path().to_path_buf(),
6074 "unit".to_string(),
6075 format!("session-{corr}"),
6076 ),
6077 consumer_identity,
6078 consumer_capabilities: None,
6079 admission_facts: facts,
6080 })
6081 .unwrap();
6082 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6083 }
6084
6085 #[derive(Clone, Default)]
6086 struct EventCapture {
6087 events: Arc<Mutex<Vec<CapturedEvent>>>,
6088 }
6089
6090 #[derive(Clone, Debug)]
6091 struct CapturedEvent {
6092 target: String,
6093 level: tracing::Level,
6094 fields: BTreeMap<String, String>,
6095 }
6096
6097 impl EventCapture {
6098 fn events(&self) -> Vec<CapturedEvent> {
6099 self.events.lock().unwrap().clone()
6100 }
6101 }
6102
6103 impl<S> Layer<S> for EventCapture
6104 where
6105 S: Subscriber,
6106 {
6107 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
6108 let mut visitor = EventFieldVisitor::default();
6109 event.record(&mut visitor);
6110 self.events.lock().unwrap().push(CapturedEvent {
6111 target: event.metadata().target().to_string(),
6112 level: *event.metadata().level(),
6113 fields: visitor.fields,
6114 });
6115 }
6116 }
6117
6118 #[derive(Default)]
6119 struct EventFieldVisitor {
6120 fields: BTreeMap<String, String>,
6121 }
6122
6123 impl Visit for EventFieldVisitor {
6124 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
6125 self.fields
6126 .insert(field.name().to_string(), format!("{value:?}"));
6127 }
6128 }
6129
6130 fn health_response(corr: u64, status: HealthStatus) -> Frame {
6131 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
6132 status,
6133 detail: Some("warming".to_string()),
6134 metrics: Some(json!({"queue_depth": 3})),
6135 })
6136 .unwrap();
6137 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
6138 }
6139
6140 fn route_bind_ack(corr: u64) -> Frame {
6141 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
6142 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
6143 }
6144
6145 fn unique_project_root(label: &str) -> TestTempDir {
6146 TestTempDir::new(label)
6147 }
6148
6149 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
6150 match parse_route_poll(frame) {
6151 ClientControlResponse::RoutePoll {
6152 status: None,
6153 live: Some(live),
6154 ..
6155 } => assert_eq!(live, expected_live),
6156 other => panic!("unexpected route.poll response: {other:?}"),
6157 }
6158 }
6159
6160 fn bind_liveness_route(
6161 registry: &Registry,
6162 forwarding: &ForwardingTable,
6163 module_id: &str,
6164 ) -> (RouteCtx, u16, u32) {
6165 let module_connection = ConnectionId::new(101);
6166 let client_connection = ConnectionId::new(202);
6167 let registration = registry
6168 .register_with_control_ops(
6169 manifest(module_id, PROTOCOL_VERSION),
6170 PROTOCOL_VERSION,
6171 module_connection,
6172 module_baseline_control_ops(),
6173 )
6174 .unwrap();
6175 let (module_tx, _module_rx) = mpsc::channel(8);
6176 let endpoint = forwarding
6177 .register_module_connection(
6178 module_connection,
6179 module_id.to_string(),
6180 PROTOCOL_VERSION,
6181 manifest_concurrency(®istration.manifest),
6182 FrameSink::new(module_tx),
6183 )
6184 .unwrap();
6185 let (client_ctx, _client_rx) = route_ctx(client_connection);
6186 let pending = forwarding
6187 .begin_route_bind_relay_for_test(
6188 client_connection,
6189 client_ctx.egress.clone(),
6190 1,
6191 module_id,
6192 )
6193 .unwrap();
6194 assert_eq!(pending.endpoint, endpoint);
6195 let route_channel = pending.client_channel;
6196 let route_epoch = pending.client_epoch;
6197 forwarding
6198 .complete_pending_relay(
6199 module_connection,
6200 pending.corr,
6201 RouteBindRelayOutcome::Accepted,
6202 )
6203 .unwrap();
6204 (client_ctx, route_channel, route_epoch)
6205 }
6206
6207 struct FakeProcessLiveness {
6208 live: Option<bool>,
6209 }
6210
6211 impl ModuleProcessLiveness for FakeProcessLiveness {
6212 fn process_live(&self, _module_id: &str) -> Option<bool> {
6213 self.live
6214 }
6215 }
6216
6217 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6218 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
6219 {
6220 let registry = Arc::new(Registry::default());
6221 let supervisor_handle = SupervisorHandle::new();
6222 let supervisor = Supervisor::new(
6223 Arc::clone(®istry),
6224 RestartPolicy::new(1, Duration::from_millis(10)),
6225 )
6226 .with_handle(supervisor_handle.clone());
6227 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
6228 let module = supervisor
6229 .spawn(ModuleSpec {
6230 module_id: "stderr-tail-wire".to_string(),
6231 program: fake_aft_stub_path(),
6232 args: Vec::new(),
6233 env: vec![
6234 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
6235 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
6236 ],
6237 reserved: false,
6238 reserved_prefixes: Vec::new(),
6239 protocol: ModuleProtocol::Subc,
6240 overlap: Default::default(),
6241 })
6242 .unwrap();
6243
6244 let deadline = Instant::now() + Duration::from_secs(5);
6245 loop {
6246 let tail = module.stderr_tail(None, None);
6247 if tail
6248 .entries
6249 .iter()
6250 .any(|entry| matches!(entry, TailEntry::ProcessStart))
6251 && tail.entries.iter().any(|entry| {
6252 matches!(
6253 entry,
6254 TailEntry::Line {
6255 truncated: true,
6256 ..
6257 }
6258 )
6259 })
6260 {
6261 break;
6262 }
6263 assert!(
6264 Instant::now() < deadline,
6265 "module did not produce a truncated line and restart boundary: {tail:?}"
6266 );
6267 sleep(Duration::from_millis(10)).await;
6268 }
6269
6270 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
6271 let request = ClientControlRequest::SupervisorStderrTail {
6272 module_id: "stderr-tail-wire".to_string(),
6273 max_lines: None,
6274 max_bytes: None,
6275 };
6276 let frame = Frame::build(
6277 FrameType::Request,
6278 control_flags(),
6279 0,
6280 0,
6281 1,
6282 serde_json::to_vec(&request).unwrap(),
6283 )
6284 .unwrap();
6285 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6286 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
6287 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
6288 serde_json::from_slice(&responses[0].body).unwrap()
6289 else {
6290 panic!("expected supervisor.stderr_tail response");
6291 };
6292
6293 assert!(
6294 tail.entries
6295 .iter()
6296 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
6297 "the control response lost the restart boundary"
6298 );
6299 let Some(StderrTailEntry::Line {
6300 text,
6301 truncated,
6302 at_ms,
6303 }) = tail.entries.iter().find(|entry| {
6304 matches!(
6305 entry,
6306 StderrTailEntry::Line {
6307 truncated: true,
6308 ..
6309 }
6310 )
6311 })
6312 else {
6313 panic!("the control response lost the truncated line");
6314 };
6315 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
6316 assert!(*truncated);
6317 assert!(
6318 at_ms.is_some(),
6319 "the control response lost the line's capture time"
6320 );
6321 }
6322
6323 #[tokio::test(flavor = "current_thread")]
6328 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
6329 let dir = TestTempDir::new("terminals-off-worker");
6330 let journal_path = dir.join("terminals.jsonl");
6331 let registry = Arc::new(Registry::default());
6332 let supervisor_handle = SupervisorHandle::new();
6333 let supervisor =
6334 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
6335 .with_handle(supervisor_handle.clone())
6336 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
6337 let module = supervisor
6338 .spawn(ModuleSpec {
6339 module_id: "terminal-off-worker".to_string(),
6340 program: fake_aft_stub_path(),
6341 args: Vec::new(),
6342 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
6343 reserved: false,
6344 reserved_prefixes: Vec::new(),
6345 protocol: ModuleProtocol::Subc,
6346 overlap: Default::default(),
6347 })
6348 .unwrap();
6349 let deadline = Instant::now() + Duration::from_secs(5);
6350 while module.terminal_history().entries.len() != 2 {
6351 assert!(Instant::now() < deadline, "module did not record two exits");
6352 sleep(Duration::from_millis(10)).await;
6353 }
6354
6355 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
6356 let handler =
6357 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
6358 let frame = Frame::build(
6359 FrameType::Request,
6360 control_flags(),
6361 0,
6362 0,
6363 1,
6364 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
6365 module_id: "terminal-off-worker".to_string(),
6366 })
6367 .unwrap(),
6368 )
6369 .unwrap();
6370 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6371 let spawned_at = std::time::Instant::now();
6372 let read = tokio::spawn({
6373 let handler = Arc::clone(&handler);
6374 async move { handler.handle_control_frame(&ctx, frame).await }
6375 });
6376 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
6379 .await
6380 .unwrap()
6381 .expect("the history read reached its pause");
6382 let elapsed = spawned_at.elapsed();
6383 assert!(
6384 elapsed < Duration::from_secs(2) && !read.is_finished(),
6385 "this task could not run while the history read was paused \
6386 (resumed after {elapsed:?}, read finished: {})",
6387 read.is_finished()
6388 );
6389
6390 drop(release);
6391 let responses = read.await.unwrap().unwrap();
6392 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
6393 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
6394 panic!("expected supervisor.terminals response");
6395 };
6396 assert_eq!(terminals.entries.len(), 2);
6397 assert_eq!(terminals.journal_skipped_lines, 0);
6398 assert_eq!(terminals.journal_read_errors, 0);
6399 }
6400
6401 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6402 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
6403 let registry = Arc::new(Registry::default());
6404 let supervisor_handle = SupervisorHandle::new();
6405 let supervisor =
6406 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
6407 .with_handle(supervisor_handle.clone());
6408 let module = supervisor
6409 .spawn(ModuleSpec {
6410 module_id: "terminal-golden".to_string(),
6411 program: fake_aft_stub_path(),
6412 args: Vec::new(),
6413 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
6414 reserved: false,
6415 reserved_prefixes: Vec::new(),
6416 protocol: ModuleProtocol::Subc,
6417 overlap: Default::default(),
6418 })
6419 .unwrap();
6420
6421 let deadline = Instant::now() + Duration::from_secs(5);
6422 while module.terminal_history().entries.len() != 2 {
6423 assert!(
6424 Instant::now() < deadline,
6425 "module did not retain two terminal exits: {:?}",
6426 module.terminal_history()
6427 );
6428 sleep(Duration::from_millis(10)).await;
6429 }
6430
6431 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
6432 let request = ClientControlRequest::SupervisorTerminals {
6433 module_id: "terminal-golden".to_string(),
6434 };
6435 let frame = Frame::build(
6436 FrameType::Request,
6437 control_flags(),
6438 0,
6439 0,
6440 1,
6441 serde_json::to_vec(&request).unwrap(),
6442 )
6443 .unwrap();
6444 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6445 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
6446 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
6447 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
6448 panic!("expected supervisor.terminals response");
6449 };
6450 assert_eq!(terminals.entries.len(), 2);
6451 assert_eq!(terminals.dropped, 0);
6452
6453 let mut rendered = serde_json::to_value(response).unwrap();
6454 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
6457 for (index, entry) in rendered["entries"]
6458 .as_array_mut()
6459 .expect("terminal response entries array")
6460 .iter_mut()
6461 .enumerate()
6462 {
6463 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
6464 }
6465
6466 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
6467 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
6468 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
6469 if std::env::var_os("UPDATE_GOLDEN").is_some() {
6470 std::fs::write(&golden_path, &serialized).unwrap();
6471 }
6472 let expected: Value =
6473 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
6474 assert_eq!(rendered, expected);
6475 }
6476
6477 #[test]
6478 fn hello_registers_manifest_and_returns_ack() {
6479 let registry = Arc::new(Registry::default());
6480 let handler = ControlHandler::new(Arc::clone(®istry));
6481 let conn = ConnectionId::new(1);
6482
6483 let responses = handler
6484 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
6485 .unwrap();
6486
6487 assert_eq!(responses.len(), 1);
6488 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
6489 assert_eq!(responses[0].header.channel, 0);
6490 assert_eq!(responses[0].header.corr, 7);
6491 let ack = parse_ack(&responses[0]);
6492 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
6493 assert!(ack
6494 .subc_capabilities
6495 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
6496 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
6497 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
6498 assert!(ack
6499 .subc_ops
6500 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
6501 assert!(ack
6502 .subc_ops
6503 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
6504
6505 let registration = registry.get_module("aft").unwrap().unwrap();
6506 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
6507 assert_eq!(registration.state, ChannelState::Active);
6508 assert_eq!(registration.connection_id, conn);
6509 assert_eq!(registration.control_ops, module_baseline_control_ops());
6510 }
6511
6512 #[test]
6513 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
6514 let invalid_identifiers = [
6515 ("case_change", "credentials-Provider/v1"),
6516 ("leading_zero", "credentials-provider/v01"),
6517 ("trailing_hyphen", "credentials-provider-/v1"),
6518 ("consecutive_hyphens", "credentials--provider/v1"),
6519 ("uppercase", "Credentials-provider/v1"),
6520 ("missing_v", "credentials-provider/1"),
6521 ("whitespace", "credentials provider/v1"),
6522 ("zero_version", "credentials-provider/v0"),
6523 ("out_of_range_version", "credentials-provider/v4294967296"),
6524 (
6525 "overlength_name",
6526 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
6527 ),
6528 ];
6529 let mut cases = invalid_identifiers
6530 .into_iter()
6531 .map(|(name, identifier)| {
6532 (
6533 format!("identifier_{name}"),
6534 "capabilities.provides[0]".to_string(),
6535 identifier.to_string(),
6536 json!({ "provides": [identifier] }),
6537 None,
6538 )
6539 })
6540 .collect::<Vec<_>>();
6541 cases.extend([
6542 (
6543 "unknown_need".to_string(),
6544 "capabilities.requires[0].need".to_string(),
6545 "deferred".to_string(),
6546 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
6547 None,
6548 ),
6549 (
6550 "duplicate_provides".to_string(),
6551 "capabilities.provides[1]".to_string(),
6552 "credentials-provider/v1".to_string(),
6553 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
6554 None,
6555 ),
6556 (
6557 "duplicate_must_never_reach".to_string(),
6558 "capabilities.must_never_reach[1]".to_string(),
6559 "credentials-provider/v1".to_string(),
6560 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
6561 None,
6562 ),
6563 (
6564 "duplicate_requires_same_need".to_string(),
6565 "capabilities.requires[1]".to_string(),
6566 "credentials-provider/v1".to_string(),
6567 json!({ "requires": [
6568 { "capability": "credentials-provider/v1", "need": "required" },
6569 { "capability": "credentials-provider/v1", "need": "required" }
6570 ] }),
6571 None,
6572 ),
6573 (
6574 "duplicate_requires_conflicting_need".to_string(),
6575 "capabilities.requires[1]".to_string(),
6576 "credentials-provider/v1".to_string(),
6577 json!({ "requires": [
6578 { "capability": "credentials-provider/v1", "need": "required" },
6579 { "capability": "credentials-provider/v1", "need": "optional" }
6580 ] }),
6581 None,
6582 ),
6583 (
6584 "capabilities_root_pointer".to_string(),
6585 "runtime_computed[0]".to_string(),
6586 "/capabilities".to_string(),
6587 json!({}),
6588 Some(json!(["/capabilities"])),
6589 ),
6590 (
6591 "capabilities_descendant_pointer".to_string(),
6592 "runtime_computed[0]".to_string(),
6593 "/capabilities/provides".to_string(),
6594 json!({}),
6595 Some(json!(["/capabilities/provides"])),
6596 ),
6597 (
6598 "malformed_pointer_without_leading_slash".to_string(),
6599 "runtime_computed[0]".to_string(),
6600 "capabilities".to_string(),
6601 json!({}),
6602 Some(json!(["capabilities"])),
6603 ),
6604 (
6605 "malformed_pointer_escape".to_string(),
6606 "runtime_computed[0]".to_string(),
6607 "/roles/~2/tools".to_string(),
6608 json!({}),
6609 Some(json!(["/roles/~2/tools"])),
6610 ),
6611 (
6612 "unknown_capabilities_field".to_string(),
6613 "capabilities.future".to_string(),
6614 "<array>".to_string(),
6615 json!({ "future": [] }),
6616 None,
6617 ),
6618 ]);
6619
6620 for (index, (name, field, value, capabilities, runtime_computed)) in
6621 cases.into_iter().enumerate()
6622 {
6623 let registry = Arc::new(Registry::default());
6624 let handler = ControlHandler::new(Arc::clone(®istry));
6625 let response = handler
6626 .handle_control(
6627 ConnectionId::new((index + 1) as u64),
6628 capability_grammar_hello_frame(
6629 capabilities,
6630 runtime_computed,
6631 index as u64 + 1,
6632 ),
6633 )
6634 .expect("invalid HELLO returns a refusal");
6635
6636 assert_eq!(response.len(), 1, "{name} must emit one refusal");
6637 let error = parse_error(&response[0]);
6638 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
6639 let message = error["message"]
6640 .as_str()
6641 .expect("error message is a string");
6642 assert!(
6643 message.contains(&field),
6644 "{name}: field missing from {message}"
6645 );
6646 assert!(
6647 message.contains(&value),
6648 "{name}: value missing from {message}"
6649 );
6650 assert_eq!(
6651 registry
6652 .active_registration_count()
6653 .expect("registry reads"),
6654 0,
6655 "{name}: refused HELLO must not create a catalog entry"
6656 );
6657 }
6658 }
6659
6660 #[test]
6661 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
6662 let registry = Arc::new(Registry::default());
6663 let handler = ControlHandler::new(Arc::clone(®istry));
6664 let capabilities = json!({
6665 "provides": ["credentials-provider/v1"],
6666 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
6667 "must_never_reach": ["federation-transport/v1"]
6668 });
6669 let response = handler
6670 .handle_control(
6671 ConnectionId::new(99),
6672 capability_grammar_hello_frame(
6673 capabilities.clone(),
6674 Some(json!(["/roles/0/tools"])),
6675 99,
6676 ),
6677 )
6678 .expect("valid HELLO registers");
6679 assert_eq!(response[0].header.ty, FrameType::HelloAck);
6680
6681 let request = Frame::build(
6682 FrameType::Request,
6683 control_flags(),
6684 0,
6685 0,
6686 100,
6687 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
6688 .expect("catalog request serializes"),
6689 )
6690 .expect("catalog request frame builds");
6691 let response = handler
6692 .handle_catalog_list(request, None)
6693 .expect("catalog list succeeds");
6694 let ClientControlResponse::CatalogList { modules, .. } =
6695 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
6696 else {
6697 panic!("catalog request must return catalog.list");
6698 };
6699 assert_eq!(modules.len(), 1);
6700 assert_eq!(
6701 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
6702 capabilities
6703 );
6704 }
6705
6706 #[test]
6707 fn catalog_list_mirrors_management_operation_description() {
6708 let registry = Arc::new(Registry::default());
6709 let handler = ControlHandler::new(Arc::clone(®istry));
6710 let description = "List managed records and return their identifiers and metadata.";
6711 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
6712 manifest.provides = vec![ProviderRole::ManagementSurface {
6713 operations: vec![ManagementOperation {
6714 name: "records.list".to_string(),
6715 kind: ManagementOperationKind::Query,
6716 description: Some(description.to_string()),
6717 }],
6718 config_schema: json!({"type": "object"}),
6719 observability: vec![ObservabilitySurface {
6720 name: "records.stats".to_string(),
6721 kind: ObservabilityKind::Snapshot,
6722 }],
6723 identity_scope: vec![IdentityScope::Project],
6724 concurrency: Concurrency::ModuleManaged,
6725 }];
6726 registry
6727 .register_with_control_ops(
6728 manifest,
6729 PROTOCOL_VERSION,
6730 ConnectionId::new(99),
6731 Vec::new(),
6732 )
6733 .expect("described management manifest registers");
6734
6735 let request = Frame::build(
6736 FrameType::Request,
6737 control_flags(),
6738 0,
6739 0,
6740 100,
6741 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
6742 .expect("catalog request serializes"),
6743 )
6744 .expect("catalog request frame builds");
6745 let response = handler
6746 .handle_catalog_list(request, None)
6747 .expect("catalog list succeeds");
6748 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
6749 assert_eq!(
6750 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
6751 "catalog.list must preserve the declared operation description verbatim"
6752 );
6753 }
6754
6755 #[test]
6756 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
6757 let registry = Arc::new(Registry::default());
6758 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
6759 [("vault".to_string(), true), ("squatter".to_string(), true)],
6760 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
6761 );
6762 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
6763 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
6764 provides: vec!["credentials-provider/v1".to_string()],
6765 requires: Vec::new(),
6766 must_never_reach: Vec::new(),
6767 });
6768 let frame = Frame::build(
6769 FrameType::Hello,
6770 control_flags(),
6771 0,
6772 0,
6773 77,
6774 serde_json::to_vec(&ModuleHelloBody {
6775 manifest: squatter,
6776 protocol_ver: PROTOCOL_VERSION,
6777 control_ops: None,
6778 launch_nonce: None,
6779 })
6780 .expect("HELLO serializes"),
6781 )
6782 .expect("HELLO frame builds");
6783 let response = handler
6784 .handle_control(ConnectionId::new(77), frame)
6785 .expect("reserved claim receives a typed refusal");
6786 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
6787 assert_eq!(
6788 registry
6789 .active_registration_count()
6790 .expect("registry reads"),
6791 0,
6792 "a reserved capability refusal must not leave a catalog entry"
6793 );
6794 }
6795
6796 #[test]
6797 fn server_describe_surfaces_required_capability_verdict_fields() {
6798 let registry = Arc::new(Registry::default());
6799 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
6800 [
6801 ("consumer".to_string(), true),
6802 ("provider".to_string(), false),
6803 ],
6804 BTreeMap::new(),
6805 );
6806 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
6807 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
6808 provides: Vec::new(),
6809 requires: vec![subc_protocol::manifest::CapabilityRequirement {
6810 capability: "credentials-provider/v1".to_string(),
6811 need: subc_protocol::manifest::CapabilityNeed::Required,
6812 }],
6813 must_never_reach: Vec::new(),
6814 });
6815 let hello = Frame::build(
6816 FrameType::Hello,
6817 control_flags(),
6818 0,
6819 0,
6820 78,
6821 serde_json::to_vec(&ModuleHelloBody {
6822 manifest: consumer,
6823 protocol_ver: PROTOCOL_VERSION,
6824 control_ops: None,
6825 launch_nonce: None,
6826 })
6827 .expect("HELLO serializes"),
6828 )
6829 .expect("HELLO frame builds");
6830 handler
6831 .handle_control(ConnectionId::new(78), hello)
6832 .expect("consumer registers");
6833 let describe = Frame::build(
6834 FrameType::Request,
6835 control_flags(),
6836 0,
6837 0,
6838 79,
6839 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
6840 .expect("request serializes"),
6841 )
6842 .expect("describe frame builds");
6843 let response = handler
6844 .handle_server_describe(describe)
6845 .expect("server.describe succeeds");
6846 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
6847 let requirement = &rendered["capability_requirements"][0];
6848 assert_eq!(requirement["consumer"], "consumer");
6849 assert_eq!(requirement["verdict"], "never_provided");
6850 assert_eq!(requirement["episode_seq"], 1);
6851 assert_eq!(requirement["config_satisfiable"], false);
6852 assert_eq!(requirement["runtime_available"], false);
6853 assert!(requirement["detail"]
6854 .as_str()
6855 .expect("detail string")
6856 .contains("credentials-provider/v1"));
6857 }
6858
6859 #[test]
6860 fn catalog_list_omits_capabilities_for_legacy_manifest() {
6861 let registry = Arc::new(Registry::default());
6862 let handler = ControlHandler::new(Arc::clone(®istry));
6863 let hello = handler
6864 .handle_control(
6865 ConnectionId::new(101),
6866 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
6867 )
6868 .expect("legacy HELLO registers");
6869 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
6870
6871 let request = Frame::build(
6872 FrameType::Request,
6873 control_flags(),
6874 0,
6875 0,
6876 102,
6877 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
6878 .expect("catalog request serializes"),
6879 )
6880 .expect("catalog request frame builds");
6881 let response = handler
6882 .handle_catalog_list(request, None)
6883 .expect("catalog list succeeds");
6884 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
6885 assert!(
6886 body["modules"][0].get("capabilities").is_none(),
6887 "legacy manifest must retain an absent capabilities field on catalog.list"
6888 );
6889 }
6890
6891 #[test]
6892 fn hello_ack_omits_storage_when_no_storage_config() {
6893 let registry = Arc::new(Registry::default());
6894 let handler = ControlHandler::new(Arc::clone(®istry));
6895 let responses = handler
6896 .handle_control(
6897 ConnectionId::new(1),
6898 hello_frame("aft", PROTOCOL_VERSION, 7),
6899 )
6900 .unwrap();
6901 let ack = parse_ack(&responses[0]);
6902 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
6903 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
6904 }
6905
6906 #[tokio::test]
6907 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
6908 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
6909 let registry = Arc::new(Registry::default());
6910 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
6911 let responses = handler
6912 .handle_control(
6913 ConnectionId::new(1),
6914 hello_frame("aft", PROTOCOL_VERSION, 7),
6915 )
6916 .unwrap();
6917 let ack = parse_ack(&responses[0]);
6918 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
6919
6920 let described = handler
6921 .handle_control_frame(
6922 &route_ctx(ConnectionId::new(2)).0,
6923 Frame::build(
6924 FrameType::Request,
6925 control_flags(),
6926 0,
6927 0,
6928 9,
6929 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
6930 )
6931 .unwrap(),
6932 )
6933 .await
6934 .unwrap();
6935 let ClientControlResponse::ServerDescribe { machine_id, .. } =
6936 serde_json::from_slice(&described[0].body).unwrap()
6937 else {
6938 panic!("server.describe answered with another shape");
6939 };
6940 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
6941 }
6942
6943 #[test]
6944 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
6945 let registry = Arc::new(Registry::default());
6948 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
6949 crate::daemon_config::StorageConfig::Sqlite {
6950 data_home: std::path::PathBuf::from("/data"),
6951 },
6952 ));
6953
6954 let responses = handler
6955 .handle_control(
6956 ConnectionId::new(1),
6957 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
6958 )
6959 .unwrap();
6960 let ack = parse_ack(&responses[0]);
6961 assert_eq!(
6962 ack.storage,
6963 Some(serde_json::json!({
6964 "module_id": "alfonso-routing",
6965 "storage_namespace": "default",
6966 "isolation": { "kind": "module" },
6967 "backend": {
6968 "backend": "sqlite",
6969 "path": "/data/cortexkit/alfonso-routing/store.db"
6970 }
6971 })),
6972 "the delivered descriptor is the module's own sqlite store path"
6973 );
6974 }
6975
6976 #[test]
6977 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
6978 let registry = Arc::new(Registry::default());
6979 let handler = ControlHandler::new(Arc::clone(®istry));
6980 let conn = ConnectionId::new(1);
6981 let responses = handler
6982 .handle_control(
6983 conn,
6984 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
6985 )
6986 .unwrap();
6987 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
6988 let registration = registry.get_module("aft").unwrap().unwrap();
6989 assert_eq!(registration.control_ops, module_baseline_control_ops());
6990
6991 let frame =
6992 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
6993 assert!(handler
6994 .guard_module_control_op(&frame, "aft", "route.bind")
6995 .unwrap()
6996 .is_none());
6997 let error = handler
6998 .guard_module_control_op(&frame, "aft", "test.synthetic")
6999 .unwrap()
7000 .expect("synthetic ungranted op should be rejected");
7001 assert_eq!(error.header.ty, FrameType::Error);
7002 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
7003 }
7004
7005 #[test]
7006 fn hello_control_ops_some_adds_optional_grants() {
7007 let registry = Arc::new(Registry::default());
7008 let handler = ControlHandler::new(Arc::clone(®istry));
7009 handler
7010 .handle_control(
7011 ConnectionId::new(1),
7012 hello_frame_with_control_ops(
7013 "aft",
7014 PROTOCOL_VERSION,
7015 7,
7016 Some(vec![
7017 "future.synthetic".to_string(),
7018 "route.bind".to_string(),
7019 ]),
7020 ),
7021 )
7022 .unwrap();
7023 let registration = registry.get_module("aft").unwrap().unwrap();
7024 assert_eq!(
7025 registration.control_ops,
7026 vec![
7027 "route.bind".to_string(),
7028 "route.status".to_string(),
7029 "future.synthetic".to_string(),
7030 ]
7031 );
7032 let frame =
7033 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
7034 assert!(handler
7035 .guard_module_control_op(&frame, "aft", "future.synthetic")
7036 .unwrap()
7037 .is_none());
7038 }
7039
7040 #[tokio::test]
7041 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
7042 let registry = Arc::new(Registry::default());
7043 let forwarding = Arc::new(ForwardingTable::default());
7044 let handler =
7045 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
7046 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
7047 hello_via_sink(
7048 &handler,
7049 &module_ctx,
7050 &mut module_rx,
7051 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
7052 )
7053 .await;
7054
7055 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
7056 let responses = handler
7057 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
7058 .await
7059 .unwrap();
7060 assert_eq!(responses.len(), 1);
7061 assert_eq!(responses[0].header.ty, FrameType::Error);
7062 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
7063 assert!(module_rx.try_recv().is_err());
7064 }
7065
7066 #[tokio::test]
7067 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
7068 let registry = Arc::new(Registry::default());
7069 let forwarding = Arc::new(ForwardingTable::default());
7070 let handler =
7071 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
7072 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
7073 hello_via_sink(
7074 &handler,
7075 &module_ctx,
7076 &mut module_rx,
7077 hello_frame_with_control_ops(
7078 "aft",
7079 PROTOCOL_VERSION,
7080 7,
7081 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
7082 ),
7083 )
7084 .await;
7085
7086 let project_root = unique_project_root("demux");
7087 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
7088 let route_handler = handler.clone();
7089 let route_task = tokio::spawn(async move {
7090 route_handler
7091 .handle_control_frame(
7092 &route_client_ctx,
7093 route_open_frame(100, "aft", project_root),
7094 )
7095 .await
7096 .unwrap()
7097 });
7098 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7099 .await
7100 .unwrap()
7101 .unwrap();
7102 assert!(matches!(
7103 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
7104 ModuleControlRequest::RouteBind { .. }
7105 ));
7106
7107 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
7108 let health_handler = handler.clone();
7109 let health_task = tokio::spawn(async move {
7110 health_handler
7111 .handle_control_frame(
7112 &health_client_ctx,
7113 supervisor_health_probe_frame(101, "aft"),
7114 )
7115 .await
7116 .unwrap()
7117 });
7118 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7119 .await
7120 .unwrap()
7121 .unwrap();
7122 assert_eq!(
7123 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
7124 ModuleControlRequest::HealthCheck {}
7125 );
7126
7127 handler
7128 .handle_control_frame(
7129 &module_ctx,
7130 health_response(health_frame.header.corr, HealthStatus::Degraded),
7131 )
7132 .await
7133 .unwrap();
7134 let health_response = health_task.await.unwrap();
7135 assert_eq!(health_response.len(), 1);
7136 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
7137 ClientControlResponse::SupervisorHealthProbe {
7138 module_id,
7139 status,
7140 detail,
7141 metrics,
7142 } => {
7143 assert_eq!(module_id, "aft");
7144 assert_eq!(status, HealthStatus::Degraded);
7145 assert_eq!(detail.as_deref(), Some("warming"));
7146 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
7147 }
7148 other => panic!("unexpected health response: {other:?}"),
7149 }
7150
7151 handler
7152 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
7153 .await
7154 .unwrap();
7155 let route_response = route_task.await.unwrap();
7156 assert!(route_response.is_empty());
7157 let published = route_client_rx.recv().await.unwrap();
7158 assert!(matches!(
7159 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
7160 ClientControlResponse::RouteOpen { .. }
7161 ));
7162 }
7163
7164 async fn relay_route_open(
7169 handler: &ControlHandler,
7170 client_connection: ConnectionId,
7171 client_egress: &FrameSink,
7172 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
7173 corr: u64,
7174 module_id: &str,
7175 project_root_label: &str,
7176 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
7177 let ctx = RouteCtx {
7178 connection_id: client_connection,
7179 egress: client_egress.clone(),
7180 };
7181 let handler = handler.clone();
7182 let project_root = unique_project_root(project_root_label);
7183 let module_id = module_id.to_string();
7184 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
7185 let task = tokio::spawn(async move {
7186 let _guard = tracing::dispatcher::set_default(&dispatch);
7187 handler
7188 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
7189 .await
7190 .unwrap()
7191 });
7192 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
7193 .await
7194 .expect("module receives the relayed route.bind")
7195 .expect("module egress is open");
7196 (task, bind.frame)
7197 }
7198
7199 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
7200 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
7201 ModuleControlRequest::RouteBind {
7202 route_channel,
7203 epoch,
7204 ..
7205 } => (route_channel, epoch),
7206 other => panic!("expected a route.bind request, got {other:?}"),
7207 }
7208 }
7209
7210 fn published_route(frame: &Frame) -> (u16, u32) {
7211 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
7212 ClientControlResponse::RouteOpen {
7213 route_channel,
7214 route_epoch,
7215 } => (route_channel, route_epoch),
7216 other => panic!("expected a route.open response, got {other:?}"),
7217 }
7218 }
7219
7220 #[tokio::test]
7240 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
7241 let registry = Arc::new(Registry::default());
7242 let forwarding = Arc::new(ForwardingTable::default());
7243 let handler =
7244 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
7245
7246 let module_connection = ConnectionId::new(30);
7247 let (module_ctx, mut module_rx) = route_ctx(module_connection);
7248 hello_via_sink(
7249 &handler,
7250 &module_ctx,
7251 &mut module_rx,
7252 hello_frame("aft", PROTOCOL_VERSION, 7),
7253 )
7254 .await;
7255
7256 let dying_client = ConnectionId::new(31);
7257 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
7258
7259 let (first_task, first_bind) = relay_route_open(
7262 &handler,
7263 dying_client,
7264 &dying_ctx.egress,
7265 &mut module_rx,
7266 100,
7267 "aft",
7268 "closing-first",
7269 )
7270 .await;
7271 handler
7272 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
7273 .await
7274 .unwrap();
7275 assert!(first_task.await.unwrap().is_empty());
7276 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
7277
7278 let (second_task, second_bind) = relay_route_open(
7280 &handler,
7281 dying_client,
7282 &dying_ctx.egress,
7283 &mut module_rx,
7284 101,
7285 "aft",
7286 "closing-second",
7287 )
7288 .await;
7289 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
7290
7291 assert!(forwarding
7294 .escalate_client_delivery_failure(
7295 dying_client,
7296 first_channel,
7297 first_epoch,
7298 CloseReason::new(
7299 "module_to_client_delivery_failed",
7300 "client egress refused a module frame",
7301 ),
7302 crate::forwarding::UndeliveredFrame {
7303 module_id: None,
7304 sink: &dying_ctx.egress,
7305 },
7306 )
7307 .unwrap());
7308 assert!(!dying_ctx.egress.is_closed());
7309
7310 let ack = handler
7312 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
7313 .await;
7314 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
7315 if module_loop_error.is_some() {
7316 handler.cleanup_connection(module_connection).unwrap();
7320 }
7321 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7325 .await
7326 .ok()
7327 .flatten();
7328
7329 assert!(
7331 registry
7332 .get_module_by_connection(module_connection)
7333 .unwrap()
7334 .is_some(),
7335 "one client's closing connection ended the shared module connection: \
7336 {module_loop_error:?}"
7337 );
7338 let cotenant = ConnectionId::new(32);
7340 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
7341 let (cotenant_task, cotenant_bind) = relay_route_open(
7342 &handler,
7343 cotenant,
7344 &cotenant_ctx.egress,
7345 &mut module_rx,
7346 102,
7347 "aft",
7348 "closing-cotenant",
7349 )
7350 .await;
7351 handler
7352 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
7353 .await
7354 .unwrap();
7355 assert!(cotenant_task.await.unwrap().is_empty());
7356 let (cotenant_channel, cotenant_epoch) =
7357 published_route(&cotenant_rx.recv().await.unwrap());
7358 assert!(matches!(
7359 forwarding
7360 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
7361 .unwrap(),
7362 DataRoute::Client(DataRouteState::Bound(_))
7363 ));
7364
7365 let goodbye = post_ack_module_frame
7368 .expect("module receives a GOODBYE for the abandoned route channel");
7369 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
7370 assert_eq!(goodbye.header.channel, abandoned_channel);
7371 assert_eq!(goodbye.header.epoch, abandoned_epoch);
7372
7373 assert!(dying_rx.try_recv().is_err());
7377 let second_response = second_task.await.unwrap();
7378 assert_eq!(second_response.len(), 1);
7379 assert_eq!(
7380 parse_error(&second_response[0])["code"],
7381 "target_unavailable"
7382 );
7383 }
7384
7385 #[test]
7392 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
7393 let handler = ControlHandler::default();
7394 let module_connection = ConnectionId::new(30);
7395 let client_connection = ConnectionId::new(31);
7396
7397 handler
7398 .refuse_to_end_module_connection_for_a_client(
7399 module_connection,
7400 77,
7401 ForwardingError::ConnectionClosing {
7402 connection_id: client_connection,
7403 },
7404 )
7405 .expect("a closing client must never end the module connection");
7406
7407 assert!(matches!(
7408 handler.refuse_to_end_module_connection_for_a_client(
7409 module_connection,
7410 78,
7411 ForwardingError::ConnectionClosing {
7412 connection_id: module_connection,
7413 },
7414 ),
7415 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
7416 connection_id
7417 })) if connection_id == module_connection
7418 ));
7419 assert!(matches!(
7420 handler.refuse_to_end_module_connection_for_a_client(
7421 module_connection,
7422 79,
7423 ForwardingError::Poisoned,
7424 ),
7425 Err(RouterError::Forwarding(ForwardingError::Poisoned))
7426 ));
7427 assert!(matches!(
7428 handler.refuse_to_end_module_connection_for_a_client(
7429 module_connection,
7430 80,
7431 ForwardingError::StaleModuleEndpoint,
7432 ),
7433 Err(RouterError::Forwarding(
7434 ForwardingError::StaleModuleEndpoint
7435 ))
7436 ));
7437 }
7438
7439 #[tokio::test]
7452 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
7453 let registry = Arc::new(Registry::default());
7454 let forwarding = Arc::new(ForwardingTable::default());
7455 let supervisor = SupervisorHandle::new();
7456 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7457 let handler =
7458 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
7459 .with_supervisor(supervisor);
7460
7461 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
7462 hello_via_sink(
7463 &handler,
7464 &target_ctx,
7465 &mut target_rx,
7466 hello_frame("target", PROTOCOL_VERSION, 1),
7467 )
7468 .await;
7469
7470 let wrong_nonce = handler
7474 .handle_control_frame(
7475 &route_ctx(ConnectionId::new(91)).0,
7476 route_open_frame_with_admission_facts(
7477 20,
7478 "target",
7479 unique_project_root("admission-facts"),
7480 Some(subc_control::ConsumerIdentity {
7481 module_id: "fed".to_string(),
7482 launch_nonce: "not-the-real-nonce".to_string(),
7483 }),
7484 None,
7485 ),
7486 )
7487 .await
7488 .unwrap();
7489 assert_eq!(
7490 parse_error(&wrong_nonce[0])["code"],
7491 "bad_consumer_identity",
7492 "a mismatched launch nonce must be refused, not stamped Reserved"
7493 );
7494
7495 let never_spawned = handler
7499 .handle_control_frame(
7500 &route_ctx(ConnectionId::new(92)).0,
7501 route_open_frame_with_admission_facts(
7502 21,
7503 "target",
7504 unique_project_root("admission-facts"),
7505 Some(subc_control::ConsumerIdentity {
7506 module_id: "never-spawned".to_string(),
7507 launch_nonce: "any-nonce".to_string(),
7508 }),
7509 None,
7510 ),
7511 )
7512 .await
7513 .unwrap();
7514 assert_eq!(
7515 parse_error(&never_spawned[0])["code"],
7516 "bad_consumer_identity",
7517 "an unspawned module_id must be refused rather than accepted for lack of a record"
7518 );
7519 }
7520
7521 #[tokio::test]
7544 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
7545 let registry = Arc::new(Registry::default());
7546 let forwarding = Arc::new(ForwardingTable::default());
7547 let supervisor = SupervisorHandle::new();
7548 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7549 let handler =
7550 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
7551 .with_supervisor(supervisor);
7552
7553 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
7554 hello_via_sink(
7555 &handler,
7556 &target_ctx,
7557 &mut target_rx,
7558 hello_frame("target", PROTOCOL_VERSION, 1),
7559 )
7560 .await;
7561
7562 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
7563 let route_handler = handler.clone();
7564 let route_task = tokio::spawn(async move {
7565 route_handler
7566 .handle_control_frame(
7567 &client_ctx,
7568 route_open_frame_with_admission_facts(
7569 30,
7570 "target",
7571 unique_project_root("admission-facts"),
7572 Some(subc_control::ConsumerIdentity {
7573 module_id: "fed".to_string(),
7574 launch_nonce: "fed-nonce".to_string(),
7575 }),
7576 None,
7577 ),
7578 )
7579 .await
7580 .unwrap()
7581 });
7582
7583 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7590 .await
7591 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
7592 .expect("module control channel closed before route.bind");
7593 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
7594 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
7595 panic!("expected route.bind")
7596 };
7597 assert_eq!(
7598 principal,
7599 Some(Principal::Reserved {
7600 module_id: "fed".to_string()
7601 }),
7602 "a correctly attested consumer must be stamped Reserved for its own id"
7603 );
7604
7605 handler
7606 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
7607 .await
7608 .unwrap();
7609 assert!(route_task.await.unwrap().is_empty());
7610 assert!(
7611 matches!(
7612 serde_json::from_slice::<ClientControlResponse>(
7613 &client_rx.recv().await.unwrap().body
7614 )
7615 .unwrap(),
7616 ClientControlResponse::RouteOpen { .. }
7617 ),
7618 "the route must actually open, not merely avoid an error"
7619 );
7620 }
7621
7622 #[tokio::test(start_paused = true)]
7623 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
7624 let registry = Arc::new(Registry::default());
7625 let forwarding = Arc::new(ForwardingTable::default());
7626 let supervisor = SupervisorHandle::new();
7627 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7628 let handler =
7629 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
7630 .with_supervisor(supervisor);
7631
7632 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
7633 hello_via_sink(
7634 &handler,
7635 &target_ctx,
7636 &mut target_rx,
7637 hello_frame("target", PROTOCOL_VERSION, 1),
7638 )
7639 .await;
7640
7641 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
7642 let direct_handler = handler.clone();
7643 let direct_open = tokio::spawn(async move {
7644 direct_handler
7645 .handle_control_frame(
7646 &direct_ctx,
7647 route_open_frame(2, "target", unique_project_root("route-census-direct")),
7648 )
7649 .await
7650 .unwrap()
7651 });
7652 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7653 .await
7654 .expect("no direct route.bind within 5s")
7655 .expect("target control channel closed before direct route.bind");
7656 handler
7657 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
7658 .await
7659 .unwrap();
7660 assert!(direct_open.await.unwrap().is_empty());
7661 let _ = direct_rx.recv().await.unwrap();
7662
7663 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
7664 let reserved_handler = handler.clone();
7665 let reserved_open = tokio::spawn(async move {
7666 reserved_handler
7667 .handle_control_frame(
7668 &reserved_ctx,
7669 route_open_frame_with_admission_facts(
7670 3,
7671 "target",
7672 unique_project_root("admission-facts"),
7673 Some(ConsumerIdentity {
7674 module_id: "fed".to_string(),
7675 launch_nonce: "fed-nonce".to_string(),
7676 }),
7677 None,
7678 ),
7679 )
7680 .await
7681 .unwrap()
7682 });
7683 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7684 .await
7685 .expect("no reserved route.bind within 5s")
7686 .expect("target control channel closed before reserved route.bind");
7687 handler
7688 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
7689 .await
7690 .unwrap();
7691 assert!(reserved_open.await.unwrap().is_empty());
7692 let _ = reserved_rx.recv().await.unwrap();
7693
7694 forwarding
7695 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
7696 .unwrap();
7697 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
7698 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
7699 module_id: Some("target".to_string()),
7700 })
7701 .unwrap();
7702 let census_frame =
7703 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
7704 let response = handler
7705 .handle_control_frame(&census_ctx, census_frame)
7706 .await
7707 .unwrap()
7708 .pop()
7709 .unwrap();
7710 let actual: Value = serde_json::from_slice(&response.body).unwrap();
7711 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
7712 assert!(matches!(
7713 decoded,
7714 ClientControlResponse::SupervisorRoutes { .. }
7715 ));
7716 let routes = actual["modules"][0]["routes"].as_array().unwrap();
7717 assert_eq!(routes.len(), 2);
7718 assert!(routes.iter().all(|route| route["draining"] == true));
7719 assert!(
7722 routes.iter().all(|route| route["drain_reason"] == "reload"),
7723 "draining routes must name the drain's reason: {routes:?}"
7724 );
7725 assert!(routes.iter().any(|route| {
7726 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
7727 }));
7728 assert!(routes.iter().any(|route| {
7729 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
7730 }));
7731
7732 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
7733 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
7734 if std::env::var_os("UPDATE_GOLDEN").is_some() {
7735 std::fs::write(
7736 &golden_path,
7737 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
7738 )
7739 .unwrap();
7740 }
7741 let expected: Value =
7742 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
7743 assert_eq!(actual, expected);
7744 }
7745
7746 async fn query_live_roots(
7747 handler: &ControlHandler,
7748 module_ctx: &RouteCtx,
7749 ) -> ModuleControlResponseToModule {
7750 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
7751 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
7752 let response = handler
7753 .handle_control_frame(module_ctx, frame)
7754 .await
7755 .unwrap()
7756 .pop()
7757 .unwrap();
7758 serde_json::from_slice(&response.body).unwrap()
7759 }
7760
7761 #[tokio::test(start_paused = true)]
7762 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
7763 let registry = Arc::new(Registry::default());
7764 let forwarding = Arc::new(ForwardingTable::default());
7765 let handler = ControlHandler::with_forwarding(registry, forwarding);
7766 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
7767 hello_via_sink(
7768 &handler,
7769 &target_ctx,
7770 &mut target_rx,
7771 hello_frame("target", PROTOCOL_VERSION, 1),
7772 )
7773 .await;
7774 let root = unique_project_root("live-roots-known");
7775 let path = ProjectRootId::from_path_allowing_missing(root.path())
7776 .unwrap()
7777 .as_path()
7778 .to_path_buf();
7779 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
7780 let open_handler = handler.clone();
7781 let opened = tokio::spawn(async move {
7782 open_handler
7783 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
7784 .await
7785 .unwrap()
7786 });
7787 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7788 .await
7789 .unwrap()
7790 .unwrap();
7791 handler
7792 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
7793 .await
7794 .unwrap();
7795 assert!(opened.await.unwrap().is_empty());
7796 let _ = client_rx.recv().await.unwrap();
7797
7798 let root = unique_project_root("live-roots-pending");
7799 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
7800 .unwrap()
7801 .as_path()
7802 .to_path_buf();
7803 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
7804 let open_handler = handler.clone();
7805 let pending = tokio::spawn(async move {
7806 open_handler
7807 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
7808 .await
7809 .unwrap()
7810 });
7811 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7812 .await
7813 .unwrap()
7814 .unwrap();
7815 let actual = query_live_roots(&handler, &target_ctx).await;
7816 let ModuleControlResponseToModule::LiveRoots {
7817 roots,
7818 unknown_root_bindings,
7819 total_bindings,
7820 } = actual
7821 else {
7822 panic!("expected live roots")
7823 };
7824 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
7825 assert_eq!(unknown_root_bindings, 0);
7826 assert_eq!(
7827 roots.len(),
7828 2,
7829 "root-known arm must retain each canonical root"
7830 );
7831 assert_eq!(
7832 total_bindings,
7833 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
7834 );
7835 let counts = roots
7836 .iter()
7837 .map(|root| (root.project_root.clone(), root.bound, root.pending))
7838 .collect::<Vec<_>>();
7839 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
7840 expected.sort_by(|a, b| a.0.cmp(&b.0));
7841 assert_eq!(
7842 counts, expected,
7843 "roots must sort by path and count pending separately"
7844 );
7845 handler
7846 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
7847 .await
7848 .unwrap();
7849 assert!(pending.await.unwrap().is_empty());
7850 }
7851
7852 #[tokio::test(start_paused = true)]
7853 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
7854 let forwarding = Arc::new(ForwardingTable::default());
7855 let handler =
7856 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
7857 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
7858 hello_via_sink(
7859 &handler,
7860 &target_ctx,
7861 &mut target_rx,
7862 hello_frame("target", PROTOCOL_VERSION, 1),
7863 )
7864 .await;
7865 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
7866 let pending = forwarding
7867 .begin_route_bind_relay_for_test(
7868 client_ctx.connection_id,
7869 client_ctx.egress.clone(),
7870 2,
7871 "target",
7872 )
7873 .unwrap();
7874 forwarding
7875 .complete_pending_relay(
7876 target_ctx.connection_id,
7877 pending.corr,
7878 RouteBindRelayOutcome::Accepted,
7879 )
7880 .unwrap();
7881 let actual = query_live_roots(&handler, &target_ctx).await;
7882 let ModuleControlResponseToModule::LiveRoots {
7883 roots,
7884 unknown_root_bindings,
7885 total_bindings,
7886 } = actual
7887 else {
7888 panic!("expected live roots")
7889 };
7890 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
7891 assert_eq!(
7892 unknown_root_bindings, 1,
7893 "unknown-root arm must not read as no bindings"
7894 );
7895 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
7896 assert_eq!(
7897 total_bindings,
7898 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
7899 );
7900 }
7901
7902 #[tokio::test(start_paused = true)]
7910 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
7911 let forwarding = Arc::new(ForwardingTable::default());
7912 let handler =
7913 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
7914 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
7915 let replies = handler
7916 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
7917 .await
7918 .unwrap();
7919 let queued_by_hello = module_rx.len();
7920
7921 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
7922 let open_handler = handler.clone();
7923 let open = tokio::spawn(async move {
7924 open_handler
7925 .handle_control_frame(
7926 &client_ctx,
7927 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
7928 )
7929 .await
7930 .unwrap()
7931 });
7932 let mut spins = 0;
7934 while module_rx.len() == queued_by_hello {
7935 spins += 1;
7936 assert!(spins < 10_000, "route.open never queued a route.bind");
7937 tokio::task::yield_now().await;
7938 }
7939
7940 for reply in replies {
7942 module_ctx.egress.send(reply).await.unwrap();
7943 }
7944
7945 let first = module_rx.recv().await.unwrap().frame;
7946 assert_eq!(
7947 first.header.ty,
7948 FrameType::HelloAck,
7949 "the first frame a registering module reads must be its HELLO_ACK"
7950 );
7951 assert_eq!(first.header.corr, 7);
7952 let second = module_rx.recv().await.unwrap().frame;
7953 assert_eq!(second.header.ty, FrameType::Request);
7954 assert!(
7955 matches!(
7956 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
7957 ModuleControlRequest::RouteBind { .. }
7958 ),
7959 "the route.bind follows the ack"
7960 );
7961 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
7962
7963 handler
7964 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
7965 .await
7966 .unwrap();
7967 assert!(open.await.unwrap().is_empty());
7968 let _ = client_rx.recv().await.unwrap();
7969 }
7970
7971 #[tokio::test(start_paused = true)]
7972 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
7973 let handler = ControlHandler::with_forwarding(
7974 Arc::new(Registry::default()),
7975 Arc::new(ForwardingTable::default()),
7976 );
7977 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
7978 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
7979 hello_via_sink(
7980 &handler,
7981 &first_ctx,
7982 &mut first_rx,
7983 hello_frame("first", PROTOCOL_VERSION, 1),
7984 )
7985 .await;
7986 hello_via_sink(
7987 &handler,
7988 &second_ctx,
7989 &mut second_rx,
7990 hello_frame("second", PROTOCOL_VERSION, 2),
7991 )
7992 .await;
7993 let root = unique_project_root("second-only");
7994 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
7995 let cloned = handler.clone();
7996 let open = tokio::spawn(async move {
7997 cloned
7998 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
7999 .await
8000 .unwrap()
8001 });
8002 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
8003 .await
8004 .unwrap()
8005 .unwrap();
8006 let first = query_live_roots(&handler, &first_ctx).await;
8007 let second = query_live_roots(&handler, &second_ctx).await;
8008 assert!(
8009 matches!(
8010 first,
8011 ModuleControlResponseToModule::LiveRoots {
8012 total_bindings: 0,
8013 ..
8014 }
8015 ),
8016 "cross-module scope must not expose another module's roots"
8017 );
8018 assert!(
8019 matches!(
8020 second,
8021 ModuleControlResponseToModule::LiveRoots {
8022 total_bindings: 1,
8023 ..
8024 }
8025 ),
8026 "second module must see its pending route"
8027 );
8028 handler
8029 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
8030 .await
8031 .unwrap();
8032 assert!(open.await.unwrap().is_empty());
8033 }
8034
8035 #[tokio::test(start_paused = true)]
8036 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
8037 let handler = ControlHandler::with_forwarding(
8038 Arc::new(Registry::default()),
8039 Arc::new(ForwardingTable::default()),
8040 );
8041 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
8042 hello_via_sink(
8043 &handler,
8044 &target_ctx,
8045 &mut target_rx,
8046 hello_frame("target", PROTOCOL_VERSION, 1),
8047 )
8048 .await;
8049 let actual = query_live_roots(&handler, &target_ctx).await;
8050 let ModuleControlResponseToModule::LiveRoots {
8051 roots,
8052 unknown_root_bindings,
8053 total_bindings,
8054 } = actual
8055 else {
8056 panic!("expected live roots")
8057 };
8058 assert!(roots.is_empty());
8059 assert_eq!(unknown_root_bindings, 0);
8060 assert_eq!(total_bindings, 0);
8061 assert_eq!(
8062 total_bindings,
8063 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
8064 );
8065 }
8066
8067 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
8074 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8075 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
8076 let text = std::fs::read_to_string(&path)
8077 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
8078 let vectors: Vec<(String, Value)> = text
8079 .lines()
8080 .filter(|line| !line.trim().is_empty())
8081 .map(|line| {
8082 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
8083 let id = entry["corpus_id"]
8084 .as_str()
8085 .expect("every vector carries a corpus_id")
8086 .to_string();
8087 (id, entry["package"].clone())
8088 })
8089 .collect();
8090 assert_eq!(
8094 vectors.len(),
8095 3,
8096 "vendored fed corpus changed size; re-sync from subc-federation"
8097 );
8098
8099 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
8114 let richest = vectors
8115 .iter()
8116 .filter_map(|(_, package)| package.as_object())
8117 .map(|object| {
8118 object
8119 .keys()
8120 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
8121 .count()
8122 })
8123 .max()
8124 .unwrap_or(0);
8125 assert!(
8126 richest >= 2,
8127 "vendored corpus no longer carries a package with unmodelled fields, \
8128 so the relay test can no longer distinguish a verbatim relay from a lossy one"
8129 );
8130
8131 vectors
8132 }
8133
8134 #[tokio::test]
8142 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
8143 for (corpus_id, package) in fed_admission_facts_vectors() {
8144 let registry = Arc::new(Registry::default());
8145 let forwarding = Arc::new(ForwardingTable::default());
8146 let supervisor = SupervisorHandle::new();
8147 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8148 let handler =
8149 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8150 .with_supervisor(supervisor)
8151 .with_admission_facts_config(
8152 Some("fed".to_string()),
8153 Some(vec!["target".to_string()]),
8154 );
8155
8156 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
8157 hello_via_sink(
8158 &handler,
8159 &target_ctx,
8160 &mut target_rx,
8161 hello_frame("target", PROTOCOL_VERSION, 1),
8162 )
8163 .await;
8164
8165 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
8166 let route_handler = handler.clone();
8167 let expected = package.clone();
8168 let route_task = tokio::spawn(async move {
8169 route_handler
8170 .handle_control_frame(
8171 &client_ctx,
8172 route_open_frame_with_admission_facts(
8173 20,
8174 "target",
8175 unique_project_root("admission-facts"),
8176 Some(subc_control::ConsumerIdentity {
8177 module_id: "fed".to_string(),
8178 launch_nonce: "fed-nonce".to_string(),
8179 }),
8180 Some(package),
8181 ),
8182 )
8183 .await
8184 .unwrap()
8185 });
8186
8187 let bind_frame = target_rx.recv().await.unwrap();
8188 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8189 let ModuleControlRequest::RouteBind {
8190 admission_facts, ..
8191 } = bind
8192 else {
8193 panic!("{corpus_id}: expected route.bind")
8194 };
8195 assert_eq!(
8196 admission_facts,
8197 Some(expected),
8198 "{corpus_id}: relay must not add, drop or reshape any field"
8199 );
8200
8201 handler
8202 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8203 .await
8204 .unwrap();
8205 route_task.await.unwrap();
8206 }
8207 }
8208
8209 #[tokio::test]
8210 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
8211 let registry = Arc::new(Registry::default());
8212 let forwarding = Arc::new(ForwardingTable::default());
8213 let supervisor = SupervisorHandle::new();
8214 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8215 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
8216 let handler =
8217 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8218 .with_supervisor(supervisor)
8219 .with_admission_facts_config(
8220 Some("fed".to_string()),
8221 Some(vec!["target".to_string()]),
8222 );
8223
8224 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
8225 hello_via_sink(
8226 &handler,
8227 &target_ctx,
8228 &mut target_rx,
8229 hello_frame("target", PROTOCOL_VERSION, 1),
8230 )
8231 .await;
8232 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
8233 hello_via_sink(
8234 &handler,
8235 &other_ctx,
8236 &mut other_rx,
8237 hello_frame("other", PROTOCOL_VERSION, 2),
8238 )
8239 .await;
8240
8241 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
8242 let expected_facts = facts.clone();
8243 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
8244 let route_handler = handler.clone();
8245 let route_task = tokio::spawn(async move {
8246 route_handler
8247 .handle_control_frame(
8248 &client_ctx,
8249 route_open_frame_with_admission_facts(
8250 10,
8251 "target",
8252 unique_project_root("admission-facts"),
8253 Some(subc_control::ConsumerIdentity {
8254 module_id: "fed".to_string(),
8255 launch_nonce: "fed-nonce".to_string(),
8256 }),
8257 Some(facts.clone()),
8258 ),
8259 )
8260 .await
8261 .unwrap()
8262 });
8263 let bind_frame = target_rx.recv().await.unwrap();
8264 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8265 let ModuleControlRequest::RouteBind {
8266 admission_facts, ..
8267 } = bind
8268 else {
8269 panic!("expected route.bind")
8270 };
8271 assert_eq!(admission_facts, Some(expected_facts));
8272 handler
8273 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8274 .await
8275 .unwrap();
8276 assert!(route_task.await.unwrap().is_empty());
8277 assert!(matches!(
8278 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
8279 .unwrap(),
8280 ClientControlResponse::RouteOpen { .. }
8281 ));
8282
8283 let direct = handler
8284 .handle_control_frame(
8285 &route_ctx(ConnectionId::new(73)).0,
8286 route_open_frame_with_admission_facts(
8287 11,
8288 "target",
8289 unique_project_root("admission-facts"),
8290 None,
8291 Some(json!({"x": 1})),
8292 ),
8293 )
8294 .await
8295 .unwrap();
8296 assert_eq!(
8297 parse_error(&direct[0])["code"],
8298 "admission_facts_not_permitted"
8299 );
8300
8301 let different_reserved = handler
8302 .handle_control_frame(
8303 &route_ctx(ConnectionId::new(77)).0,
8304 route_open_frame_with_admission_facts(
8305 15,
8306 "target",
8307 unique_project_root("admission-facts"),
8308 Some(subc_control::ConsumerIdentity {
8309 module_id: "other".to_string(),
8310 launch_nonce: "other-nonce".to_string(),
8311 }),
8312 Some(json!({"x": 1})),
8313 ),
8314 )
8315 .await
8316 .unwrap();
8317 assert_eq!(
8318 parse_error(&different_reserved[0])["code"],
8319 "admission_facts_not_permitted"
8320 );
8321
8322 let other_target = handler
8323 .handle_control_frame(
8324 &route_ctx(ConnectionId::new(74)).0,
8325 route_open_frame_with_admission_facts(
8326 12,
8327 "other",
8328 unique_project_root("admission-facts"),
8329 Some(subc_control::ConsumerIdentity {
8330 module_id: "fed".to_string(),
8331 launch_nonce: "fed-nonce".to_string(),
8332 }),
8333 Some(json!({"x": 1})),
8334 ),
8335 )
8336 .await
8337 .unwrap();
8338 assert_eq!(
8339 parse_error(&other_target[0])["code"],
8340 "admission_facts_target_not_allowed"
8341 );
8342
8343 let nonexistent = handler
8344 .handle_control_frame(
8345 &route_ctx(ConnectionId::new(75)).0,
8346 route_open_frame_with_admission_facts(
8347 13,
8348 "missing",
8349 unique_project_root("admission-facts"),
8350 None,
8351 Some(json!({"x": 1})),
8352 ),
8353 )
8354 .await
8355 .unwrap();
8356 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
8357
8358 let described = handler
8359 .handle_control_frame(
8360 &route_ctx(ConnectionId::new(76)).0,
8361 Frame::build(
8362 FrameType::Request,
8363 control_flags(),
8364 0,
8365 0,
8366 14,
8367 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
8368 )
8369 .unwrap(),
8370 )
8371 .await
8372 .unwrap();
8373 let ClientControlResponse::ServerDescribe { capabilities, .. } =
8374 serde_json::from_slice(&described[0].body).unwrap()
8375 else {
8376 panic!("expected server.describe response")
8377 };
8378 assert!(capabilities
8379 .iter()
8380 .any(|cap| cap == "admission_facts_relay_v1"));
8381 }
8382
8383 #[tokio::test]
8384 async fn admission_facts_without_configured_carrier_are_rejected() {
8385 let registry = Arc::new(Registry::default());
8386 let forwarding = Arc::new(ForwardingTable::default());
8387 let handler = ControlHandler::with_forwarding(registry, forwarding);
8388 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
8389 hello_via_sink(
8390 &handler,
8391 &target_ctx,
8392 &mut target_rx,
8393 hello_frame("target", PROTOCOL_VERSION, 1),
8394 )
8395 .await;
8396
8397 let responses = handler
8398 .handle_control_frame(
8399 &route_ctx(ConnectionId::new(79)).0,
8400 route_open_frame_with_admission_facts(
8401 16,
8402 "target",
8403 unique_project_root("admission-facts"),
8404 None,
8405 Some(json!({"x": 1})),
8406 ),
8407 )
8408 .await
8409 .unwrap();
8410 assert_eq!(
8411 parse_error(&responses[0])["code"],
8412 "admission_facts_not_permitted"
8413 );
8414 }
8415
8416 #[tokio::test]
8417 async fn route_open_relays_consumer_capabilities_verbatim() {
8418 let registry = Arc::new(Registry::default());
8419 let forwarding = Arc::new(ForwardingTable::default());
8420 let handler =
8421 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8422 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
8423 hello_via_sink(
8424 &handler,
8425 &module_ctx,
8426 &mut module_rx,
8427 hello_frame("aft", PROTOCOL_VERSION, 7),
8428 )
8429 .await;
8430
8431 let expected = vec!["elicitation".to_string(), "roots".to_string()];
8432 let expected_for_request = expected.clone();
8433 let project_root = unique_project_root("consumer-capabilities-present");
8434 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
8435 let route_handler = handler.clone();
8436 let route_task = tokio::spawn(async move {
8437 route_handler
8438 .handle_control_frame(
8439 &client_ctx,
8440 route_open_frame_with_consumer_capabilities(
8441 401,
8442 "aft",
8443 project_root,
8444 Some(expected_for_request),
8445 ),
8446 )
8447 .await
8448 .unwrap()
8449 });
8450 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8451 .await
8452 .unwrap()
8453 .unwrap();
8454 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8455 let ModuleControlRequest::RouteBind {
8456 consumer_capabilities,
8457 ..
8458 } = bind
8459 else {
8460 panic!("expected route.bind request, got {bind:?}");
8461 };
8462 assert_eq!(consumer_capabilities, Some(expected.clone()));
8463
8464 handler
8465 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8466 .await
8467 .unwrap();
8468 let route_response = route_task.await.unwrap();
8469 assert!(route_response.is_empty());
8470 let published = client_rx.recv().await.unwrap();
8471 assert!(matches!(
8472 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8473 ClientControlResponse::RouteOpen { .. }
8474 ));
8475 }
8476
8477 #[tokio::test]
8478 async fn route_open_without_consumer_capabilities_relays_none() {
8479 let registry = Arc::new(Registry::default());
8480 let forwarding = Arc::new(ForwardingTable::default());
8481 let handler =
8482 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8483 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
8484 hello_via_sink(
8485 &handler,
8486 &module_ctx,
8487 &mut module_rx,
8488 hello_frame("aft", PROTOCOL_VERSION, 7),
8489 )
8490 .await;
8491
8492 let project_root = unique_project_root("consumer-capabilities-absent");
8493 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
8494 let route_handler = handler.clone();
8495 let route_task = tokio::spawn(async move {
8496 route_handler
8497 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
8498 .await
8499 .unwrap()
8500 });
8501 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8502 .await
8503 .unwrap()
8504 .unwrap();
8505 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8506 let ModuleControlRequest::RouteBind {
8507 consumer_capabilities,
8508 ..
8509 } = bind
8510 else {
8511 panic!("expected route.bind request, got {bind:?}");
8512 };
8513 assert_eq!(consumer_capabilities, None);
8514
8515 handler
8516 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8517 .await
8518 .unwrap();
8519 let route_response = route_task.await.unwrap();
8520 assert!(route_response.is_empty());
8521 let published = client_rx.recv().await.unwrap();
8522 assert!(matches!(
8523 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8524 ClientControlResponse::RouteOpen { .. }
8525 ));
8526 }
8527
8528 #[tokio::test]
8529 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
8530 let registry = Arc::new(Registry::default());
8531 let forwarding = Arc::new(ForwardingTable::default());
8532 let handler =
8533 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8534 .with_health_probe_timeout(Duration::from_secs(5));
8535 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
8536 hello_via_sink(
8537 &handler,
8538 &module_ctx,
8539 &mut module_rx,
8540 non_routable_hello_frame_with_control_ops(
8541 "mcp",
8542 300,
8543 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
8544 ),
8545 )
8546 .await;
8547 assert!(registry
8548 .get_module("mcp")
8549 .unwrap()
8550 .unwrap()
8551 .manifest
8552 .provides
8553 .is_empty());
8554
8555 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
8556 let route_response = handler
8557 .handle_control_frame(
8558 &route_client_ctx,
8559 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
8560 )
8561 .await
8562 .unwrap();
8563 assert_eq!(route_response[0].header.ty, FrameType::Error);
8564 assert_eq!(
8565 parse_error(&route_response[0])["code"],
8566 "target_unavailable"
8567 );
8568 assert!(parse_error(&route_response[0])["message"]
8569 .as_str()
8570 .unwrap()
8571 .contains("does not provide the requested target"));
8572 assert!(module_rx.try_recv().is_err());
8573
8574 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
8575 let health_handler = handler.clone();
8576 let health_task = tokio::spawn(async move {
8577 health_handler
8578 .handle_control_frame(
8579 &health_client_ctx,
8580 supervisor_health_probe_frame(302, "mcp"),
8581 )
8582 .await
8583 .unwrap()
8584 });
8585 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8586 .await
8587 .unwrap()
8588 .unwrap();
8589 assert_eq!(
8590 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
8591 ModuleControlRequest::HealthCheck {}
8592 );
8593 handler
8594 .handle_control_frame(
8595 &module_ctx,
8596 health_response(health_frame.header.corr, HealthStatus::Ok),
8597 )
8598 .await
8599 .unwrap();
8600 let health_response = health_task.await.unwrap();
8601 assert_eq!(health_response[0].header.ty, FrameType::Response);
8602 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
8603 ClientControlResponse::SupervisorHealthProbe {
8604 module_id, status, ..
8605 } => {
8606 assert_eq!(module_id, "mcp");
8607 assert_eq!(status, HealthStatus::Ok);
8608 }
8609 other => panic!("unexpected health response: {other:?}"),
8610 }
8611
8612 forwarding
8617 .cleanup_connection(module_ctx.connection_id)
8618 .unwrap();
8619 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
8620 let cleanup_response = tokio::time::timeout(
8621 Duration::from_millis(200),
8622 handler.handle_control_frame(
8623 &cleanup_probe_ctx,
8624 supervisor_health_probe_frame(303, "mcp"),
8625 ),
8626 )
8627 .await
8628 .expect("probe should fail immediately when the control lane is gone")
8629 .unwrap();
8630 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
8631 assert_eq!(
8632 parse_error(&cleanup_response[0])["code"],
8633 "target_unavailable"
8634 );
8635 assert!(parse_error(&cleanup_response[0])["message"]
8636 .as_str()
8637 .unwrap()
8638 .contains("no module connection"));
8639
8640 handler
8641 .cleanup_connection(module_ctx.connection_id)
8642 .unwrap();
8643 }
8644
8645 #[tokio::test]
8646 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
8647 let registry = Arc::new(Registry::default());
8648 let supervisor_handle = SupervisorHandle::new();
8649 let supervisor =
8650 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
8651 .with_handle(supervisor_handle.clone())
8652 .with_connection_file_path(
8653 std::env::temp_dir()
8654 .join(format!("subc-route-open-warming-{}", std::process::id())),
8655 );
8656 let module = supervisor
8657 .supervise_configured(
8658 ModuleSpec {
8659 module_id: "warming".to_string(),
8660 program: fake_aft_stub_path(),
8661 args: Vec::new(),
8662 env: Vec::new(),
8663 reserved: false,
8664 reserved_prefixes: Vec::new(),
8665 protocol: ModuleProtocol::Subc,
8666 overlap: Default::default(),
8667 },
8668 true,
8669 )
8670 .unwrap();
8671 assert_eq!(module.state().unwrap(), ModuleState::Running);
8672
8673 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8674 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
8675 let response = handler
8676 .handle_control_frame(
8677 &ctx,
8678 route_open_frame(304, "warming", unique_project_root("warming")),
8679 )
8680 .await
8681 .unwrap();
8682 module.stop().await.unwrap();
8683
8684 assert_eq!(response[0].header.ty, FrameType::Error);
8685 let error = parse_error(&response[0]);
8686 assert_eq!(error["code"], "module_warming");
8687 assert!(error["message"]
8688 .as_str()
8689 .unwrap()
8690 .contains("state=running, enabled=true, live=false"));
8691 }
8692
8693 #[test]
8694 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
8695 let handler = ControlHandler::new(Arc::new(Registry::default()));
8696 let capture = EventCapture::default();
8697 let _subscriber =
8698 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8699 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
8700 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
8701 let pending = (0..limit).collect::<Vec<_>>();
8702 let response = handler
8703 .route_open_capacity_refusal(
8704 &ctx,
8705 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
8706 "busy",
8707 pending.len(),
8708 limit,
8709 )
8710 .unwrap();
8711 assert_eq!(parse_error(&response)["code"], "target_unavailable");
8712 let event = capture
8713 .events()
8714 .into_iter()
8715 .find(|event| {
8716 event.target == "control"
8717 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
8718 })
8719 .expect("connection admission refusal event");
8720 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
8721 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
8722 }
8723
8724 #[test]
8725 fn route_open_target_cap_logs_admission_reason_and_capacity() {
8726 let handler = ControlHandler::new(Arc::new(Registry::default()));
8727 let capture = EventCapture::default();
8728 let _subscriber =
8729 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8730 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
8731 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
8732 let guards = (0..limit)
8733 .map(|_| {
8734 handler
8735 .route_bind_concurrency
8736 .try_admit("busy", limit)
8737 .unwrap()
8738 })
8739 .collect::<Vec<_>>();
8740 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
8741 Err(in_flight) => in_flight,
8742 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
8743 };
8744 let response = handler
8745 .route_open_target_capacity_refusal(
8746 &ctx,
8747 &route_open_frame(397, "busy", unique_project_root("target-cap")),
8748 "busy",
8749 in_flight,
8750 )
8751 .unwrap();
8752 assert_eq!(parse_error(&response)["code"], "target_unavailable");
8753 let event = capture
8754 .events()
8755 .into_iter()
8756 .find(|event| {
8757 event.target == "control"
8758 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
8759 })
8760 .expect("target admission refusal event");
8761 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
8762 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
8763 drop(guards);
8764 }
8765
8766 #[tokio::test]
8771 async fn route_open_refusal_names_the_check_that_refused() {
8772 let handler = ControlHandler::new(Arc::new(Registry::default()));
8773 let capture = EventCapture::default();
8774 let _subscriber =
8775 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8776 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
8777 let response = handler
8778 .handle_control_frame(
8779 &ctx,
8780 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
8781 )
8782 .await
8783 .unwrap();
8784
8785 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
8786 let event = capture
8787 .events()
8788 .into_iter()
8789 .find(|event| {
8790 event.target == "control"
8791 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
8792 })
8793 .expect("route.open refusal event");
8794 assert_eq!(
8795 event.fields.get("reason"),
8796 Some(&"\"not_registered\"".to_string())
8797 );
8798 }
8799
8800 #[tokio::test]
8801 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
8802 let registry = Arc::new(Registry::default());
8803 let supervisor_handle = SupervisorHandle::new();
8804 let supervisor =
8805 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
8806 .with_handle(supervisor_handle.clone())
8807 .with_connection_file_path(std::env::temp_dir().join(format!(
8808 "subc-route-open-refusal-info-{}",
8809 std::process::id()
8810 )));
8811 let module = supervisor
8812 .supervise_configured(
8813 ModuleSpec {
8814 module_id: "warming".to_string(),
8815 program: fake_aft_stub_path(),
8816 args: Vec::new(),
8817 env: Vec::new(),
8818 reserved: false,
8819 reserved_prefixes: Vec::new(),
8820 protocol: ModuleProtocol::Subc,
8821 overlap: Default::default(),
8822 },
8823 true,
8824 )
8825 .unwrap();
8826 assert_eq!(module.state().unwrap(), ModuleState::Running);
8827
8828 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8829 assert!(handler
8830 .counters()
8831 .snapshot()
8832 .get("route_open_refused_by_code")
8833 .is_none());
8834 let capture = EventCapture::default();
8835 let _subscriber =
8836 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8837 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
8838 let response = handler
8839 .handle_control_frame(
8840 &ctx,
8841 route_open_frame(394, "warming", unique_project_root("refusal-info")),
8842 )
8843 .await
8844 .unwrap();
8845 module.stop().await.unwrap();
8846
8847 assert_eq!(parse_error(&response[0])["code"], "module_warming");
8848 let event = capture
8849 .events()
8850 .into_iter()
8851 .find(|event| {
8852 event.target == "control"
8853 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
8854 })
8855 .expect("route.open refusal event");
8856 assert_eq!(
8857 event.fields.get("module_id"),
8858 Some(&"\"warming\"".to_string())
8859 );
8860 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
8861 assert_eq!(
8862 event.fields.get("reason"),
8863 Some(&"\"supervised_not_registered\"".to_string())
8864 );
8865 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
8866 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
8867 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
8868 assert_eq!(
8869 handler.counters().snapshot()["route_open_refused_by_code"],
8870 json!({ "module_warming": 1 })
8871 );
8872 }
8873
8874 const OUTAGE_START: &str = "route.open refusing module: not serving";
8875 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
8876
8877 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
8878 capture
8879 .events()
8880 .into_iter()
8881 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
8882 .collect()
8883 }
8884
8885 fn supervise_stub(
8886 registry: &Arc<Registry>,
8887 module_id: &str,
8888 enabled: bool,
8889 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
8890 let supervisor_handle = SupervisorHandle::new();
8891 let supervisor =
8892 Supervisor::new(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
8893 .with_handle(supervisor_handle.clone())
8894 .with_connection_file_path(std::env::temp_dir().join(format!(
8895 "subc-route-outage-{module_id}-{}",
8896 std::process::id()
8897 )));
8898 let module = supervisor
8899 .supervise_configured(
8900 ModuleSpec {
8901 module_id: module_id.to_string(),
8902 program: fake_aft_stub_path(),
8903 args: Vec::new(),
8904 env: Vec::new(),
8905 reserved: false,
8906 reserved_prefixes: Vec::new(),
8907 protocol: ModuleProtocol::Subc,
8908 overlap: Default::default(),
8909 },
8910 enabled,
8911 )
8912 .unwrap();
8913 (supervisor_handle, module)
8914 }
8915
8916 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
8917 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
8918 module_id: module_id.to_string(),
8919 drain_timeout_ms: Some(50),
8920 })
8921 .unwrap();
8922 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8923 }
8924
8925 #[test]
8929 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
8930 let registry = Arc::new(Registry::default());
8931 let forwarding = Arc::new(ForwardingTable::default());
8932 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8933 let second = ControlHandler::with_forwarding(registry, forwarding);
8934 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
8935 }
8936
8937 #[tokio::test(flavor = "current_thread")]
8941 async fn route_open_unknown_module_refusals_add_no_outage_state() {
8942 let handler = ControlHandler::new(Arc::new(Registry::default()));
8943 let capture = EventCapture::default();
8944 let _subscriber =
8945 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8946 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
8947 for corr in 0..8 {
8948 let response = handler
8949 .handle_control_frame(
8950 &ctx,
8951 route_open_frame(
8952 380 + corr,
8953 &format!("nobody-{corr}"),
8954 unique_project_root("outage-unknown"),
8955 ),
8956 )
8957 .await
8958 .unwrap();
8959 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
8960 }
8961
8962 assert_eq!(handler.route_outages.tracked_module_count(), 0);
8963 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
8964 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
8965 }
8966
8967 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8972 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
8973 let registry = Arc::new(Registry::default());
8974 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
8975 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8976 let capture = EventCapture::default();
8977 let _subscriber =
8978 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8979 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
8980 handler.route_outages.record_accepted("outage-restart");
8983
8984 let response = handler
8985 .handle_control_frame(
8986 &ctx,
8987 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
8988 )
8989 .await
8990 .unwrap();
8991 assert_eq!(response[0].header.ty, FrameType::Error);
8992 let starts = outage_lines(&capture, OUTAGE_START);
8993 assert_eq!(starts.len(), 1, "{starts:?}");
8994 assert_eq!(starts[0].level, tracing::Level::WARN);
8995 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
8996 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
8997 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
8998 handler.route_outages.record_accepted("outage-restart");
8999 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
9000
9001 let restart = handler
9002 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
9003 .await
9004 .unwrap();
9005 assert_eq!(
9006 restart[0].header.ty,
9007 FrameType::Response,
9008 "{:?}",
9009 parse_error(&restart[0])
9010 );
9011 handler
9012 .handle_control_frame(
9013 &ctx,
9014 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
9015 )
9016 .await
9017 .unwrap();
9018 module.stop().await.unwrap();
9019
9020 let starts = outage_lines(&capture, OUTAGE_START);
9021 assert_eq!(starts.len(), 2, "{starts:?}");
9022 assert_eq!(starts[1].level, tracing::Level::INFO);
9023 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
9024 }
9025
9026 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9030 async fn failed_operator_restart_leaves_no_operator_mark() {
9031 let registry = Arc::new(Registry::default());
9032 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
9033 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9034 let capture = EventCapture::default();
9035 let _subscriber =
9036 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9037 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
9038 handler.route_outages.record_accepted("outage-disabled");
9039
9040 let restart = handler
9041 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
9042 .await
9043 .unwrap();
9044 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
9045 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
9046
9047 handler
9048 .handle_control_frame(
9049 &ctx,
9050 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
9051 )
9052 .await
9053 .unwrap();
9054 let starts = outage_lines(&capture, OUTAGE_START);
9055 assert_eq!(starts.len(), 1, "{starts:?}");
9056 assert_eq!(starts[0].level, tracing::Level::WARN);
9057 }
9058
9059 #[tokio::test(flavor = "current_thread")]
9060 async fn route_open_unknown_module_escapes_target_module_id() {
9061 let handler = ControlHandler::new(Arc::new(Registry::default()));
9062 let capture = EventCapture::default();
9063 let _subscriber =
9064 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9065 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
9066 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
9067 let response = handler
9068 .handle_control_frame(
9069 &ctx,
9070 route_open_frame(
9071 395,
9072 hostile_module_id,
9073 unique_project_root("hostile-target-module-id"),
9074 ),
9075 )
9076 .await
9077 .unwrap();
9078
9079 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
9080 let event = capture
9081 .events()
9082 .into_iter()
9083 .find(|event| {
9084 event.target == "control"
9085 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
9086 })
9087 .expect("route.open unknown-module refusal event");
9088 let logged = event.fields.get("module_id").expect("module_id field");
9089 assert!(!logged.bytes().any(|byte| byte < 0x20));
9090 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
9091 }
9092
9093 #[tokio::test(flavor = "current_thread")]
9094 async fn route_open_module_rejection_uses_daemon_counter_key() {
9095 let registry = Arc::new(Registry::default());
9096 let forwarding = Arc::new(ForwardingTable::default());
9097 let handler =
9098 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9099 let module_connection = ConnectionId::new(95);
9100 let (module_ctx, mut module_rx) = route_ctx(module_connection);
9101 hello_via_sink(
9102 &handler,
9103 &module_ctx,
9104 &mut module_rx,
9105 hello_frame("aft", PROTOCOL_VERSION, 395),
9106 )
9107 .await;
9108
9109 let client_connection = ConnectionId::new(96);
9110 let (client_ctx, _client_rx) = route_ctx(client_connection);
9111 let capture = EventCapture::default();
9112 let _subscriber =
9113 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9114 let (route_task, bind) = relay_route_open(
9115 &handler,
9116 client_connection,
9117 &client_ctx.egress,
9118 &mut module_rx,
9119 396,
9120 "aft",
9121 "hostile-module-code",
9122 )
9123 .await;
9124 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
9125 let rejection = Frame::build(
9126 FrameType::Error,
9127 control_flags(),
9128 0,
9129 0,
9130 bind.header.corr,
9131 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
9132 )
9133 .unwrap();
9134 handler
9135 .handle_control_frame(&module_ctx, rejection)
9136 .await
9137 .unwrap();
9138
9139 let response = route_task.await.unwrap();
9140 assert_eq!(parse_error(&response[0])["code"], hostile_code);
9141 let counters = handler.counters().snapshot();
9142 assert_eq!(
9143 counters["route_open_refused_by_code"],
9144 json!({ "module_rejected": 1 })
9145 );
9146 assert!(counters["route_open_refused_by_code"]
9147 .get(hostile_code)
9148 .is_none());
9149
9150 let event = capture
9151 .events()
9152 .into_iter()
9153 .find(|event| {
9154 event.target == "control"
9155 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
9156 })
9157 .expect("route.open module-rejection refusal event");
9158 let logged = event.fields.get("module_code").expect("module_code field");
9159 assert!(!logged.bytes().any(|byte| byte < 0x20));
9160 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
9161 }
9162
9163 #[tokio::test]
9164 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
9165 let registry = Arc::new(Registry::default());
9166 let supervisor_handle = SupervisorHandle::new();
9167 let missing_program = std::env::temp_dir().join(format!(
9168 "subc-route-open-missing-program-{}",
9169 std::process::id()
9170 ));
9171 let supervisor =
9172 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
9173 .with_handle(supervisor_handle.clone());
9174 let module = supervisor
9175 .supervise_configured(
9176 ModuleSpec {
9177 module_id: "failed".to_string(),
9178 program: missing_program,
9179 args: Vec::new(),
9180 env: Vec::new(),
9181 reserved: false,
9182 reserved_prefixes: Vec::new(),
9183 protocol: ModuleProtocol::Subc,
9184 overlap: Default::default(),
9185 },
9186 true,
9187 )
9188 .unwrap();
9189 assert_eq!(module.state().unwrap(), ModuleState::Failed);
9190
9191 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9192 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
9193 let response = handler
9194 .handle_control_frame(
9195 &ctx,
9196 route_open_frame(305, "failed", unique_project_root("failed")),
9197 )
9198 .await
9199 .unwrap();
9200
9201 assert_eq!(response[0].header.ty, FrameType::Error);
9202 let error = parse_error(&response[0]);
9203 assert_eq!(error["code"], "target_unavailable");
9204 assert!(error["message"]
9205 .as_str()
9206 .unwrap()
9207 .contains("state=failed, enabled=true, live=false"));
9208 }
9209
9210 #[tokio::test]
9211 async fn route_open_role_mismatch_remains_target_unavailable() {
9212 let registry = Arc::new(Registry::default());
9213 let handler = ControlHandler::new(Arc::clone(®istry));
9214 handler
9215 .handle_control(
9216 ConnectionId::new(41),
9217 non_routable_hello_frame_with_control_ops("health-only", 306, None),
9218 )
9219 .unwrap();
9220
9221 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
9222 let response = handler
9223 .handle_control_frame(
9224 &ctx,
9225 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
9226 )
9227 .await
9228 .unwrap();
9229
9230 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
9231 assert!(parse_error(&response[0])["message"]
9232 .as_str()
9233 .unwrap()
9234 .contains("does not provide the requested target"));
9235 }
9236
9237 #[tokio::test]
9238 async fn route_open_inactive_registration_remains_target_unavailable() {
9239 let registry = Arc::new(Registry::default());
9240 let handler = ControlHandler::new(Arc::clone(®istry));
9241 handler
9242 .handle_control(
9243 ConnectionId::new(43),
9244 hello_frame("inactive", PROTOCOL_VERSION, 308),
9245 )
9246 .unwrap();
9247 assert!(registry
9248 .set_module_state_for_test("inactive", ChannelState::Closed)
9249 .unwrap());
9250
9251 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
9252 let response = handler
9253 .handle_control_frame(
9254 &ctx,
9255 route_open_frame(309, "inactive", unique_project_root("inactive")),
9256 )
9257 .await
9258 .unwrap();
9259
9260 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
9261 assert!(parse_error(&response[0])["message"]
9262 .as_str()
9263 .unwrap()
9264 .contains("is not active"));
9265 }
9266
9267 #[tokio::test]
9268 async fn late_health_reply_is_recorded_through_the_module_response_path() {
9269 let registry = Arc::new(Registry::default());
9270 let forwarding = Arc::new(ForwardingTable::default());
9271 let supervisor_handle = SupervisorHandle::new();
9272 let supervisor = Supervisor::new(Arc::clone(®istry), crate::RestartPolicy::default())
9273 .with_forwarding(Arc::clone(&forwarding))
9274 .with_handle(supervisor_handle.clone());
9275 let module = supervisor
9276 .supervise_configured(
9277 crate::ModuleSpec {
9278 module_id: "late-health-response".to_string(),
9279 program: PathBuf::from("disabled-module"),
9280 args: Vec::new(),
9281 env: Vec::new(),
9282 reserved: false,
9283 reserved_prefixes: Vec::new(),
9284 protocol: ModuleProtocol::Subc,
9285 overlap: Default::default(),
9286 },
9287 false,
9288 )
9289 .unwrap();
9290 let handler =
9291 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9292 .with_supervisor(supervisor_handle);
9293 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
9294 handler
9295 .handle_control_frame(
9296 &module_ctx,
9297 hello_frame_with_control_ops(
9298 "late-health-response",
9299 PROTOCOL_VERSION,
9300 7,
9301 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9302 ),
9303 )
9304 .await
9305 .unwrap();
9306 let probe_started_at = Instant::now() - Duration::from_millis(80);
9307 let pending = forwarding
9308 .begin_health_probe_rpc_for(
9309 "late-health-response",
9310 MODULE_CONTROL_OP_HEALTH_CHECK,
9311 probe_started_at,
9312 Instant::now() - Duration::from_millis(1),
9313 )
9314 .unwrap();
9315 assert!(forwarding
9316 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
9317 .unwrap());
9318
9319 let responses = handler
9320 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
9321 .await
9322 .unwrap();
9323
9324 assert!(responses.is_empty());
9325 let health = module.status().unwrap().health;
9326 assert_eq!(health.late_answer_count, 1);
9327 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
9328 }
9329
9330 #[tokio::test]
9331 async fn health_probe_timeout_and_module_death_are_typed() {
9332 let registry = Arc::new(Registry::default());
9333 let forwarding = Arc::new(ForwardingTable::default());
9334 let handler =
9335 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9336 .with_health_probe_timeout(Duration::from_millis(50));
9337 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
9338 hello_via_sink(
9339 &handler,
9340 &module_ctx,
9341 &mut module_rx,
9342 hello_frame_with_control_ops(
9343 "aft",
9344 PROTOCOL_VERSION,
9345 7,
9346 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9347 ),
9348 )
9349 .await;
9350
9351 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
9352 let responses = handler
9353 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
9354 .await
9355 .unwrap();
9356 assert_eq!(responses[0].header.ty, FrameType::Error);
9357 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
9358 let _ = module_rx.try_recv();
9359
9360 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
9361 let health_handler = handler.clone();
9362 let death_task = tokio::spawn(async move {
9363 health_handler
9364 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
9365 .await
9366 .unwrap()
9367 });
9368 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9369 .await
9370 .unwrap()
9371 .unwrap();
9372 handler
9373 .cleanup_connection(module_ctx.connection_id)
9374 .unwrap();
9375 let responses = death_task.await.unwrap();
9376 assert_eq!(responses[0].header.ty, FrameType::Error);
9377 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
9378 }
9379
9380 #[test]
9381 fn hello_requires_exact_protocol_version() {
9382 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
9383 let registry = Arc::new(Registry::default());
9384 let handler = ControlHandler::new(Arc::clone(®istry));
9385 let responses = handler
9386 .handle_control(
9387 ConnectionId::new(connection),
9388 hello_frame("aft", offered, 9),
9389 )
9390 .unwrap();
9391
9392 assert_eq!(responses.len(), 1);
9393 assert_eq!(responses[0].header.ty, FrameType::Error);
9394 let error = parse_error(&responses[0]);
9395 assert_eq!(error["code"], "version_unsupported");
9396 assert!(registry.get_module("aft").unwrap().is_none());
9397 assert_eq!(registry.active_registration_count().unwrap(), 0);
9398 }
9399 }
9400
9401 #[test]
9402 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
9403 let registry = Arc::new(Registry::default());
9404 let forwarding = Arc::new(ForwardingTable::default());
9405 let handler =
9406 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9407 let module_connection = ConnectionId::new(301);
9408 let registration = registry
9409 .register_with_control_ops(
9410 manifest("aft-push", PROTOCOL_VERSION),
9411 PROTOCOL_VERSION,
9412 module_connection,
9413 module_baseline_control_ops(),
9414 )
9415 .unwrap();
9416 let (module_tx, _module_rx) = mpsc::channel(8);
9417 let endpoint = forwarding
9418 .register_module_connection(
9419 module_connection,
9420 "aft-push".to_string(),
9421 PROTOCOL_VERSION,
9422 manifest_concurrency(®istration.manifest),
9423 FrameSink::new(module_tx),
9424 )
9425 .unwrap();
9426
9427 let unknown = Frame::build(
9429 FrameType::Push,
9430 control_flags(),
9431 0,
9432 0,
9433 5,
9434 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
9435 )
9436 .unwrap();
9437 let out = handler.handle_status_update(endpoint, unknown).unwrap();
9438 assert!(
9439 out.is_empty(),
9440 "unknown push op must be ignored, got {out:?}"
9441 );
9442
9443 let malformed = Frame::build(
9445 FrameType::Push,
9446 control_flags(),
9447 0,
9448 0,
9449 6,
9450 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
9451 )
9452 .unwrap();
9453 let out = handler.handle_status_update(endpoint, malformed).unwrap();
9454 assert_eq!(out.len(), 1);
9455 assert_eq!(out[0].header.ty, FrameType::Error);
9456 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
9457 }
9458
9459 #[test]
9460 fn hello_rejected_when_connection_already_owns_client_routes() {
9461 let registry = Arc::new(Registry::default());
9462 let forwarding = Arc::new(ForwardingTable::default());
9463 let handler =
9464 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9465 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
9467 let client_connection = ConnectionId::new(202);
9468
9469 let responses = handler
9472 .handle_control(
9473 client_connection,
9474 hello_frame("aft-second", PROTOCOL_VERSION, 9),
9475 )
9476 .unwrap();
9477 assert_eq!(responses[0].header.ty, FrameType::Error);
9478 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
9479 assert!(registry.get_module("aft-second").unwrap().is_none());
9480 }
9481
9482 #[test]
9483 fn reserved_module_hello_requires_matching_launch_nonce() {
9484 let registry = Arc::new(Registry::default());
9485 let supervisor = SupervisorHandle::new();
9486 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
9489 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9490
9491 let no_nonce = handler
9493 .handle_control(
9494 ConnectionId::new(1),
9495 hello_frame("vault", PROTOCOL_VERSION, 1),
9496 )
9497 .unwrap();
9498 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
9499 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
9500 assert!(registry.get_module("vault").unwrap().is_none());
9501
9502 let wrong = handler
9504 .handle_control(
9505 ConnectionId::new(2),
9506 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
9507 )
9508 .unwrap();
9509 assert_eq!(wrong[0].header.ty, FrameType::Error);
9510 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
9511 assert!(registry.get_module("vault").unwrap().is_none());
9512
9513 let ok = handler
9515 .handle_control(
9516 ConnectionId::new(3),
9517 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
9518 )
9519 .unwrap();
9520 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
9521 assert!(registry.get_module("vault").unwrap().is_some());
9522 }
9523
9524 #[test]
9525 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
9526 let registry = Arc::new(Registry::default());
9527 let supervisor = SupervisorHandle::new();
9528 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
9529 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
9530 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9531
9532 let squat = handler
9533 .handle_control(
9534 ConnectionId::new(1),
9535 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
9536 )
9537 .unwrap();
9538 assert_eq!(squat[0].header.ty, FrameType::Error);
9539 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
9540 assert!(parse_error(&squat[0])["message"]
9541 .as_str()
9542 .unwrap()
9543 .contains("fed:"));
9544
9545 let accepted_peer = handler
9546 .handle_control(
9547 ConnectionId::new(2),
9548 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
9549 )
9550 .unwrap();
9551 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
9552
9553 let accepted_short = handler
9554 .handle_control(
9555 ConnectionId::new(3),
9556 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
9557 )
9558 .unwrap();
9559 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
9560
9561 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
9562 let response = handler
9563 .handle_control(
9564 ConnectionId::new(conn),
9565 hello_frame(module_id, PROTOCOL_VERSION, conn),
9566 )
9567 .unwrap();
9568 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
9569 }
9570 }
9571
9572 #[test]
9573 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
9574 let registry = Arc::new(Registry::default());
9575 let supervisor = SupervisorHandle::new();
9576 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
9577 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
9578 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
9579 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9580
9581 let owner_nonce = handler
9582 .handle_control(
9583 ConnectionId::new(1),
9584 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
9585 )
9586 .unwrap();
9587 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
9588 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
9589 assert!(registry.get_module("fed:special").unwrap().is_none());
9590
9591 let exact_nonce = handler
9592 .handle_control(
9593 ConnectionId::new(2),
9594 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
9595 )
9596 .unwrap();
9597 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
9598 assert!(registry.get_module("fed:special").unwrap().is_some());
9599 }
9600
9601 #[test]
9602 fn non_reserved_module_ignores_launch_nonce() {
9603 let registry = Arc::new(Registry::default());
9604 let handler = ControlHandler::new(Arc::clone(®istry));
9607 let no_nonce = handler
9608 .handle_control(
9609 ConnectionId::new(1),
9610 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
9611 )
9612 .unwrap();
9613 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
9614 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
9615
9616 let echoed_nonce = handler
9617 .handle_control(
9618 ConnectionId::new(2),
9619 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
9620 )
9621 .unwrap();
9622 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
9623 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
9624 }
9625
9626 #[test]
9627 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
9628 let handler = ControlHandler::default();
9629 let conn = ConnectionId::new(1);
9630 let malformed = Frame::build(
9631 FrameType::Hello,
9632 control_flags(),
9633 0,
9634 0,
9635 3,
9636 b"{not json".to_vec(),
9637 )
9638 .unwrap();
9639
9640 let error = handler.handle_control(conn, malformed).unwrap();
9641 assert_eq!(error[0].header.ty, FrameType::Error);
9642 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
9643
9644 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
9645 let pong = handler.handle_control(conn, ping).unwrap();
9646 assert_eq!(pong[0].header.ty, FrameType::Pong);
9647 assert_eq!(pong[0].header.corr, 4);
9648 }
9649
9650 #[test]
9651 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
9652 let registry = Arc::new(Registry::default());
9653 let handler = ControlHandler::new(Arc::clone(®istry));
9654
9655 handler
9656 .handle_control(
9657 ConnectionId::new(1),
9658 hello_frame("aft", PROTOCOL_VERSION, 1),
9659 )
9660 .unwrap();
9661 let duplicate = handler
9662 .handle_control(
9663 ConnectionId::new(2),
9664 hello_frame("aft", PROTOCOL_VERSION, 2),
9665 )
9666 .unwrap();
9667
9668 assert_eq!(duplicate[0].header.ty, FrameType::Error);
9669 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
9670 let registration = registry.get_module("aft").unwrap().unwrap();
9671 assert_eq!(registration.connection_id, ConnectionId::new(1));
9672 }
9673
9674 #[test]
9675 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
9676 let registry = Arc::new(Registry::default());
9677 let forwarding = Arc::new(ForwardingTable::default());
9678 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
9679 let handler =
9680 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9681 .with_process_liveness(process_liveness);
9682 let (ctx, route_channel, route_epoch) =
9683 bind_liveness_route(®istry, &forwarding, "aft-dead");
9684 let responses = handler
9685 .handle_route_poll(
9686 &ctx,
9687 route_poll_frame(41, PollKind::Liveness, route_channel),
9688 route_channel,
9689 route_epoch,
9690 PollKind::Liveness,
9691 )
9692 .unwrap();
9693
9694 assert_eq!(responses.len(), 1);
9695 assert_eq!(responses[0].header.ty, FrameType::Response);
9696 assert_route_poll_liveness(&responses[0], false);
9697 }
9698
9699 #[test]
9700 fn liveness_poll_without_process_source_uses_bound_route() {
9701 let registry = Arc::new(Registry::default());
9702 let forwarding = Arc::new(ForwardingTable::default());
9703 let handler =
9704 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9705 let (ctx, route_channel, route_epoch) =
9706 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
9707 let responses = handler
9708 .handle_route_poll(
9709 &ctx,
9710 route_poll_frame(42, PollKind::Liveness, route_channel),
9711 route_channel,
9712 route_epoch,
9713 PollKind::Liveness,
9714 )
9715 .unwrap();
9716
9717 assert_route_poll_liveness(&responses[0], true);
9718 }
9719
9720 #[test]
9721 fn liveness_poll_untracked_process_source_uses_bound_route() {
9722 let registry = Arc::new(Registry::default());
9723 let forwarding = Arc::new(ForwardingTable::default());
9724 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
9725 let handler =
9726 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9727 .with_process_liveness(process_liveness);
9728 let (ctx, route_channel, route_epoch) =
9729 bind_liveness_route(®istry, &forwarding, "aft-untracked");
9730 let responses = handler
9731 .handle_route_poll(
9732 &ctx,
9733 route_poll_frame(43, PollKind::Liveness, route_channel),
9734 route_channel,
9735 route_epoch,
9736 PollKind::Liveness,
9737 )
9738 .unwrap();
9739
9740 assert_route_poll_liveness(&responses[0], true);
9741 }
9742
9743 #[tokio::test]
9744 async fn unknown_op_returns_unknown_control_op() {
9745 let handler = ControlHandler::default();
9746 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
9747 let request = Frame::build(
9748 FrameType::Request,
9749 control_flags(),
9750 0,
9751 0,
9752 55,
9753 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
9754 )
9755 .unwrap();
9756
9757 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
9758
9759 assert_eq!(response.len(), 1);
9760 assert_eq!(response[0].header.ty, FrameType::Error);
9761 assert_eq!(response[0].header.corr, 55);
9762 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
9763 }
9764
9765 #[tokio::test]
9766 async fn supervisor_provenance_rejects_unknown_exact_module() {
9767 let handler = ControlHandler::default();
9768 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
9769 let request = Frame::build(
9770 FrameType::Request,
9771 control_flags(),
9772 0,
9773 0,
9774 57,
9775 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
9776 )
9777 .unwrap();
9778
9779 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
9780
9781 assert_eq!(response.len(), 1);
9782 assert_eq!(response[0].header.ty, FrameType::Error);
9783 assert_eq!(response[0].header.corr, 57);
9784 let error = parse_error(&response[0]);
9785 assert_eq!(error["code"], "unknown_module");
9786 assert_eq!(error["message"], "module_id 'missing' is not supervised");
9787 }
9788
9789 #[test]
9790 fn provenance_probe_override_keeps_handler_tests_deterministic() {
9791 let expected = subc_control::RunningImageAgreement::Unavailable {
9792 reason: subc_control::RunningImageUnavailableReason::HashFailed,
9793 };
9794 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
9795 assert_eq!(handler.provenance_probe_override, Some(expected));
9796 }
9797
9798 #[test]
9799 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
9800 let verdict = reload_verdict(
9801 std::path::Path::new("/bin/new"),
9802 Some(std::path::Path::new("/bin/old")),
9803 subc_control::RunningImageAgreement::Unavailable {
9804 reason: subc_control::RunningImageUnavailableReason::HashFailed,
9805 },
9806 );
9807 assert!(matches!(
9808 verdict.path,
9809 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
9810 if configured == std::path::Path::new("/bin/new")
9811 && spawned_from == std::path::Path::new("/bin/old")
9812 ));
9813 }
9814
9815 #[test]
9816 fn reload_verdict_detects_replaced_image_at_same_path() {
9817 let image = subc_control::RunningImageAgreement::Mismatch {
9818 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
9819 digest: "old".into(),
9820 },
9821 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
9822 digest: "new".into(),
9823 },
9824 };
9825 let verdict = reload_verdict(
9826 std::path::Path::new("/bin/same"),
9827 Some(std::path::Path::new("/bin/same")),
9828 image.clone(),
9829 );
9830 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
9831 assert_eq!(verdict.image, image);
9832 }
9833
9834 #[test]
9835 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
9836 let image = subc_control::RunningImageAgreement::Unavailable {
9837 reason: subc_control::RunningImageUnavailableReason::NotRunning,
9838 };
9839 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
9840 assert_eq!(
9841 verdict.path,
9842 subc_control::ReloadPathAgreement::Unavailable {
9843 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
9844 }
9845 );
9846 assert_eq!(verdict.image, image);
9847
9848 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
9849 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
9850 };
9851 let verdict = reload_verdict(
9852 std::path::Path::new("/bin/same"),
9853 Some(std::path::Path::new("/bin/same")),
9854 unconfirmed.clone(),
9855 );
9856 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
9857 assert_eq!(verdict.image, unconfirmed);
9858 }
9859
9860 #[test]
9861 fn reload_verdict_preserves_each_image_unavailability_reason() {
9862 use subc_control::RunningImageUnavailableReason as Reason;
9863
9864 for reason in [
9865 Reason::NotRunning,
9866 Reason::UnsupportedPlatform,
9867 Reason::RunningExecutableUnreadable,
9868 Reason::SpawnedPathUnreadable,
9869 Reason::HashFailed,
9870 Reason::ProcessIdentityUnconfirmed,
9871 Reason::Unknown("future_probe_reason".to_string()),
9872 ] {
9873 let image = subc_control::RunningImageAgreement::Unavailable {
9874 reason: reason.clone(),
9875 };
9876 let verdict = reload_verdict(
9877 std::path::Path::new("/bin/same"),
9878 Some(std::path::Path::new("/bin/same")),
9879 image.clone(),
9880 );
9881 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
9882 assert_eq!(verdict.image, image, "{reason:?}");
9883 }
9884 }
9885
9886 #[tokio::test]
9887 async fn malformed_control_bodies_return_invalid_control_body() {
9888 let handler = ControlHandler::default();
9889 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
9890
9891 for (corr, body) in [
9892 (56, br#"{"route_channel":1}"#.as_slice()),
9893 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
9894 (
9895 58,
9896 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
9897 ),
9898 ] {
9899 let request = Frame::build(
9900 FrameType::Request,
9901 control_flags(),
9902 0,
9903 0,
9904 corr,
9905 body.to_vec(),
9906 )
9907 .unwrap();
9908 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
9909
9910 assert_eq!(response.len(), 1);
9911 assert_eq!(response[0].header.ty, FrameType::Error);
9912 assert_eq!(response[0].header.corr, corr);
9913 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
9914 }
9915 }
9916
9917 #[tokio::test]
9918 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
9919 let registry = Arc::new(Registry::default());
9920 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
9921 let router = Router::with_control_handler(Arc::clone(&control));
9922 let connection = router.begin_connection();
9923 let (ctx, mut rx) = route_ctx(connection.id());
9924
9925 router
9926 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
9927 .await
9928 .unwrap();
9929 let response = rx.recv().await.unwrap();
9930 let ack = parse_ack(&response);
9931 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9932 let channel = 1;
9933
9934 let goodbye =
9935 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
9936 router.route_for_connection(&ctx, goodbye).await.unwrap();
9937 assert!(rx.try_recv().is_err());
9938 assert!(registry.get_module("aft").unwrap().is_none());
9939
9940 router
9941 .route_for_connection(&ctx, channel_request(channel, 13))
9942 .await
9943 .unwrap();
9944 let error_frame = rx.recv().await.unwrap();
9945 assert_eq!(error_frame.header.ty, FrameType::Error);
9946 assert_eq!(error_frame.header.channel, channel);
9947 }
9948
9949 #[tokio::test]
9950 async fn dropping_router_connection_releases_registration() {
9951 let registry = Arc::new(Registry::default());
9952 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
9953 let router = Router::with_control_handler(control);
9954 let connection = router.begin_connection();
9955 let (ctx, mut rx) = route_ctx(connection.id());
9956
9957 router
9958 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
9959 .await
9960 .unwrap();
9961 let response = rx.recv().await.unwrap();
9962 let ack = parse_ack(&response);
9963 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9964 assert!(registry.get_module("aft").unwrap().is_some());
9965
9966 drop(connection);
9967
9968 assert!(registry.get_module("aft").unwrap().is_none());
9969 assert_eq!(registry.active_registration_count().unwrap(), 0);
9970 }
9971
9972 fn capability_manifest(
9973 module_id: &str,
9974 provides: &[&str],
9975 must_never_reach: &[&str],
9976 ) -> ModuleManifest {
9977 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
9978 manifest.capabilities = Some(CapabilityDeclarations {
9979 provides: provides
9980 .iter()
9981 .map(|capability| (*capability).to_string())
9982 .collect(),
9983 requires: Vec::new(),
9984 must_never_reach: must_never_reach
9985 .iter()
9986 .map(|capability| (*capability).to_string())
9987 .collect(),
9988 });
9989 manifest
9990 }
9991
9992 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
9993 Frame::build(
9994 FrameType::Hello,
9995 control_flags(),
9996 0,
9997 0,
9998 corr,
9999 serde_json::to_vec(&ModuleHelloBody {
10000 protocol_ver: manifest.protocol_ver,
10001 manifest,
10002 control_ops: None,
10003 launch_nonce: None,
10004 })
10005 .expect("capability test HELLO serializes"),
10006 )
10007 .expect("capability test HELLO frame builds")
10008 }
10009
10010 fn catalog_update_with_capabilities_frame(
10011 corr: u64,
10012 capabilities: CapabilityDeclarations,
10013 ) -> Frame {
10014 Frame::build(
10015 FrameType::Request,
10016 control_flags(),
10017 0,
10018 0,
10019 corr,
10020 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
10021 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
10022 capabilities: Some(capabilities),
10023 ready: None,
10024 })
10025 .expect("capability catalog.update serializes"),
10026 )
10027 .expect("capability catalog.update frame builds")
10028 }
10029
10030 async fn register_capability_manifest(
10031 handler: &ControlHandler,
10032 ctx: &RouteCtx,
10033 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
10034 manifest: ModuleManifest,
10035 corr: u64,
10036 ) {
10037 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
10038 }
10039
10040 async fn open_route_for_capability_test(
10041 handler: &ControlHandler,
10042 target_ctx: &RouteCtx,
10043 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
10044 client_connection_id: u64,
10045 corr: u64,
10046 target_module_id: &str,
10047 consumer_identity: Option<ConsumerIdentity>,
10048 ) -> (
10049 mpsc::Receiver<crate::router::OutboundFrame>,
10050 ModuleControlRequest,
10051 ) {
10052 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
10053 let route_handler = handler.clone();
10054 let target_module_id = target_module_id.to_string();
10055 let route_task = tokio::spawn(async move {
10056 route_handler
10057 .handle_control_frame(
10058 &client_ctx,
10059 route_open_frame_with_admission_facts(
10060 corr,
10061 &target_module_id,
10062 unique_project_root("admission-facts"),
10063 consumer_identity,
10064 None,
10065 ),
10066 )
10067 .await
10068 .expect("capability test route.open succeeds")
10069 });
10070 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
10071 .await
10072 .expect("capability test route.open must reach route.bind")
10073 .expect("target control receiver stays open");
10074 let bind_request: ModuleControlRequest =
10075 serde_json::from_slice(&bind.body).expect("route.bind decodes");
10076 handler
10077 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
10078 .await
10079 .expect("capability test route.bind ACK succeeds");
10080 assert!(route_task.await.expect("route.open task joins").is_empty());
10081 let opened = client_rx
10082 .recv()
10083 .await
10084 .expect("successful route.open publishes a response");
10085 assert!(matches!(
10086 serde_json::from_slice::<ClientControlResponse>(&opened.body),
10087 Ok(ClientControlResponse::RouteOpen { .. })
10088 ));
10089 (client_rx, bind_request)
10090 }
10091
10092 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
10093 assert_eq!(frame.header.ty, FrameType::Push);
10094 assert_eq!(frame.header.channel, 0);
10095 assert_eq!(
10096 serde_json::from_slice::<ClientControlPush>(&frame.body)
10097 .expect("route.closed control push decodes"),
10098 ClientControlPush::RouteClosed {
10099 module_id: target_module_id.to_string(),
10100 reason: RouteCloseReason::CapabilityDenied,
10101 drained: false,
10102 abandoned: 0,
10103 excluded_subscriptions: 0,
10104 terminal: Some(false),
10105 }
10106 );
10107 }
10108
10109 #[tokio::test]
10110 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
10111 let registry = Arc::new(Registry::default());
10112 let forwarding = Arc::new(ForwardingTable::default());
10113 let supervisor = SupervisorHandle::new();
10114 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
10115 let handler =
10116 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10117 .with_supervisor(supervisor);
10118 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
10119 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
10120 register_capability_manifest(
10121 &handler,
10122 &target_ctx,
10123 &mut target_rx,
10124 capability_manifest("target", &["credentials-provider/v1"], &[]),
10125 1,
10126 )
10127 .await;
10128 register_capability_manifest(
10129 &handler,
10130 &opener_ctx,
10131 &mut opener_rx,
10132 capability_manifest("opener", &[], &["credentials-provider/v1"]),
10133 2,
10134 )
10135 .await;
10136
10137 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
10138 let replies = handler
10139 .handle_control_frame(
10140 &client_ctx,
10141 route_open_frame_with_admission_facts(
10142 3,
10143 "target",
10144 unique_project_root("admission-facts"),
10145 Some(ConsumerIdentity {
10146 module_id: "opener".to_string(),
10147 launch_nonce: "opener-nonce".to_string(),
10148 }),
10149 None,
10150 ),
10151 )
10152 .await
10153 .expect("denied route.open returns a typed frame");
10154 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
10155 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
10156 assert!(
10157 target_rx.try_recv().is_err(),
10158 "forbidden route.open must not relay route.bind"
10159 );
10160 }
10161
10162 #[tokio::test]
10163 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
10164 let registry = Arc::new(Registry::default());
10165 let forwarding = Arc::new(ForwardingTable::default());
10166 let supervisor = SupervisorHandle::new();
10167 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
10168 let handler =
10169 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10170 .with_supervisor(supervisor);
10171 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
10172 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
10173 register_capability_manifest(
10174 &handler,
10175 &target_ctx,
10176 &mut target_rx,
10177 capability_manifest("target", &["credentials-provider/v1"], &[]),
10178 1,
10179 )
10180 .await;
10181 register_capability_manifest(
10182 &handler,
10183 &old_opener_ctx,
10184 &mut old_opener_rx,
10185 capability_manifest("opener", &[], &[]),
10186 2,
10187 )
10188 .await;
10189 let (mut client_rx, _) = open_route_for_capability_test(
10190 &handler,
10191 &target_ctx,
10192 &mut target_rx,
10193 712,
10194 3,
10195 "target",
10196 Some(ConsumerIdentity {
10197 module_id: "opener".to_string(),
10198 launch_nonce: "opener-nonce".to_string(),
10199 }),
10200 )
10201 .await;
10202 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
10203
10204 handler
10205 .cleanup_connection(old_opener_ctx.connection_id)
10206 .expect("old opener registration cleans up");
10207 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
10208 register_capability_manifest(
10209 &handler,
10210 &new_opener_ctx,
10211 &mut new_opener_rx,
10212 capability_manifest("opener", &[], &["credentials-provider/v1"]),
10213 4,
10214 )
10215 .await;
10216
10217 assert_capability_denied_push(
10218 client_rx
10219 .try_recv()
10220 .expect("HELLO deny addition must emit route.closed")
10221 .frame,
10222 "target",
10223 );
10224 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
10225 assert!(matches!(
10226 target_rx.try_recv(),
10227 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
10228 ));
10229 }
10230
10231 #[tokio::test]
10232 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
10233 let registry = Arc::new(Registry::default());
10234 let forwarding = Arc::new(ForwardingTable::default());
10235 let supervisor = SupervisorHandle::new();
10236 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
10237 let handler =
10238 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10239 .with_supervisor(supervisor);
10240 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
10241 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
10242 register_capability_manifest(
10243 &handler,
10244 &target_ctx,
10245 &mut target_rx,
10246 capability_manifest("target", &[], &[]),
10247 1,
10248 )
10249 .await;
10250 register_capability_manifest(
10251 &handler,
10252 &opener_ctx,
10253 &mut opener_rx,
10254 capability_manifest("opener", &[], &["credentials-provider/v1"]),
10255 2,
10256 )
10257 .await;
10258 let (mut client_rx, _) = open_route_for_capability_test(
10259 &handler,
10260 &target_ctx,
10261 &mut target_rx,
10262 722,
10263 3,
10264 "target",
10265 Some(ConsumerIdentity {
10266 module_id: "opener".to_string(),
10267 launch_nonce: "opener-nonce".to_string(),
10268 }),
10269 )
10270 .await;
10271 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
10272
10273 let replies = handler
10274 .handle_control_frame(
10275 &target_ctx,
10276 catalog_update_with_capabilities_frame(
10277 4,
10278 CapabilityDeclarations {
10279 provides: vec!["credentials-provider/v1".to_string()],
10280 requires: Vec::new(),
10281 must_never_reach: Vec::new(),
10282 },
10283 ),
10284 )
10285 .await
10286 .expect("claim catalog.update succeeds");
10287 assert!(matches!(
10288 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
10289 Ok(ModuleControlResponseToModule::CatalogUpdate {})
10290 ));
10291 assert_capability_denied_push(
10292 client_rx
10293 .try_recv()
10294 .expect("claim addition must emit route.closed")
10295 .frame,
10296 "target",
10297 );
10298 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
10299 assert!(matches!(
10300 target_rx.try_recv(),
10301 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
10302 ));
10303 }
10304
10305 #[tokio::test]
10306 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
10307 let registry = Arc::new(Registry::default());
10308 let forwarding = Arc::new(ForwardingTable::default());
10309 let supervisor = SupervisorHandle::new();
10310 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
10311 let handler =
10312 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10313 .with_supervisor(supervisor);
10314 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
10315 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
10316 register_capability_manifest(
10317 &handler,
10318 &target_ctx,
10319 &mut target_rx,
10320 capability_manifest("target", &["credentials-provider/v1"], &[]),
10321 1,
10322 )
10323 .await;
10324 register_capability_manifest(
10325 &handler,
10326 &opener_ctx,
10327 &mut opener_rx,
10328 capability_manifest("opener", &[], &[]),
10329 2,
10330 )
10331 .await;
10332 let (mut client_rx, _) = open_route_for_capability_test(
10333 &handler,
10334 &target_ctx,
10335 &mut target_rx,
10336 732,
10337 3,
10338 "target",
10339 Some(ConsumerIdentity {
10340 module_id: "opener".to_string(),
10341 launch_nonce: "opener-nonce".to_string(),
10342 }),
10343 )
10344 .await;
10345 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
10346
10347 handler
10348 .handle_control_frame(
10349 &target_ctx,
10350 catalog_update_with_capabilities_frame(
10351 4,
10352 CapabilityDeclarations {
10353 provides: Vec::new(),
10354 requires: Vec::new(),
10355 must_never_reach: Vec::new(),
10356 },
10357 ),
10358 )
10359 .await
10360 .expect("claim removal catalog.update succeeds");
10361 assert_eq!(
10362 forwarding.active_binding_count().unwrap(),
10363 1,
10364 "removing an attested target claim must leave the route census unchanged"
10365 );
10366 assert!(
10367 client_rx.try_recv().is_err(),
10368 "claim removal must not emit route.closed capability_denied"
10369 );
10370 assert!(
10371 target_rx.try_recv().is_err(),
10372 "claim removal must not send the target a route GOODBYE"
10373 );
10374 }
10375
10376 #[tokio::test]
10379 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
10380 let registry = Arc::new(Registry::default());
10381 let forwarding = Arc::new(ForwardingTable::default());
10382 let supervisor = SupervisorHandle::new();
10383 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
10384 let handler =
10385 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10386 .with_supervisor(supervisor);
10387 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
10388 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
10389 register_capability_manifest(
10390 &handler,
10391 &target_ctx,
10392 &mut target_rx,
10393 capability_manifest("target", &["credentials-provider/v1"], &[]),
10394 1,
10395 )
10396 .await;
10397 register_capability_manifest(
10398 &handler,
10399 &opener_ctx,
10400 &mut opener_rx,
10401 capability_manifest("opener", &[], &["credentials-provider/v1"]),
10402 2,
10403 )
10404 .await;
10405
10406 let (_client_rx, bind) = open_route_for_capability_test(
10407 &handler,
10408 &target_ctx,
10409 &mut target_rx,
10410 742,
10411 3,
10412 "target",
10413 None,
10414 )
10415 .await;
10416 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
10417 panic!("direct scope-honesty route must bind");
10418 };
10419 assert_eq!(principal, Some(Principal::Direct));
10420 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
10421 }
10422
10423 #[tokio::test]
10426 async fn must_never_reach_self_route_is_capability_forbidden() {
10427 let registry = Arc::new(Registry::default());
10428 let forwarding = Arc::new(ForwardingTable::default());
10429 let supervisor = SupervisorHandle::new();
10430 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
10431 let handler =
10432 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10433 .with_supervisor(supervisor);
10434 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
10435 register_capability_manifest(
10436 &handler,
10437 &self_ctx,
10438 &mut self_rx,
10439 capability_manifest(
10440 "self-provider",
10441 &["credentials-provider/v1"],
10442 &["credentials-provider/v1"],
10443 ),
10444 1,
10445 )
10446 .await;
10447
10448 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
10449 let replies = handler
10450 .handle_control_frame(
10451 &client_ctx,
10452 route_open_frame_with_admission_facts(
10453 2,
10454 "self-provider",
10455 unique_project_root("admission-facts"),
10456 Some(ConsumerIdentity {
10457 module_id: "self-provider".to_string(),
10458 launch_nonce: "self-nonce".to_string(),
10459 }),
10460 None,
10461 ),
10462 )
10463 .await
10464 .expect("self-route refusal returns a typed frame");
10465 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
10466 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
10467 assert!(
10468 self_rx.try_recv().is_err(),
10469 "self denial must not relay route.bind"
10470 );
10471 }
10472
10473 #[test]
10474 fn unsupported_channel_zero_frame_returns_error() {
10475 let handler = ControlHandler::default();
10476 let request = Frame::build(
10477 FrameType::Request,
10478 control_flags(),
10479 0,
10480 0,
10481 21,
10482 b"opaque".to_vec(),
10483 )
10484 .unwrap();
10485
10486 let response = handler
10487 .handle_control(ConnectionId::new(1), request)
10488 .unwrap();
10489
10490 assert_eq!(response[0].header.ty, FrameType::Error);
10491 assert_eq!(
10492 parse_error(&response[0])["code"],
10493 "unsupported_control_frame"
10494 );
10495 }
10496
10497 mod swap {
10502 use super::*;
10503
10504 const INCUMBENT: ConnectionId = ConnectionId::new(30);
10505 const CANDIDATE: ConnectionId = ConnectionId::new(40);
10506
10507 struct Swap {
10508 registry: Arc<Registry>,
10509 forwarding: Arc<ForwardingTable>,
10510 handler: ControlHandler,
10511 incumbent_ctx: RouteCtx,
10512 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
10513 candidate_ctx: RouteCtx,
10514 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
10515 }
10516
10517 async fn swap_with_incumbent() -> Swap {
10518 let registry = Arc::new(Registry::default());
10519 let forwarding = Arc::new(ForwardingTable::default());
10520 let handler =
10521 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10522 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
10523 hello_via_sink(
10524 &handler,
10525 &incumbent_ctx,
10526 &mut incumbent_rx,
10527 hello_frame("aft", PROTOCOL_VERSION, 7),
10528 )
10529 .await;
10530 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
10531 Swap {
10532 registry,
10533 forwarding,
10534 handler,
10535 incumbent_ctx,
10536 incumbent_rx,
10537 candidate_ctx,
10538 candidate_rx,
10539 }
10540 }
10541
10542 fn register_candidate(swap: &Swap, ready: Option<bool>) {
10543 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
10544 candidate_manifest.ready = ready;
10545 let registration = swap
10546 .registry
10547 .register_candidate_with_control_ops(
10548 candidate_manifest,
10549 PROTOCOL_VERSION,
10550 CANDIDATE,
10551 module_baseline_control_ops(),
10552 )
10553 .unwrap();
10554 swap.forwarding
10555 .register_candidate_module_connection(
10556 CANDIDATE,
10557 "aft".to_string(),
10558 PROTOCOL_VERSION,
10559 manifest_concurrency(®istration.manifest),
10560 swap.candidate_ctx.egress.clone(),
10561 )
10562 .unwrap();
10563 }
10564
10565 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
10566 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
10567 swap.registry.promote_candidate("aft").unwrap().unwrap();
10568 cutover.incumbent.unwrap()
10569 }
10570
10571 fn keyed_total(counters: &Value, key: &str) -> u64 {
10572 counters[key]
10573 .as_object()
10574 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
10575 .unwrap_or(0)
10576 }
10577
10578 #[tokio::test]
10584 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
10585 let mut swap = swap_with_incumbent().await;
10586 let handler = swap.handler.clone();
10587
10588 let cotenant = ConnectionId::new(31);
10590 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
10591 let (cotenant_task, cotenant_bind) = relay_route_open(
10592 &handler,
10593 cotenant,
10594 &cotenant_ctx.egress,
10595 &mut swap.incumbent_rx,
10596 100,
10597 "aft",
10598 "swap-cotenant",
10599 )
10600 .await;
10601 handler
10602 .handle_control_frame(
10603 &swap.incumbent_ctx,
10604 route_bind_ack(cotenant_bind.header.corr),
10605 )
10606 .await
10607 .unwrap();
10608 assert!(cotenant_task.await.unwrap().is_empty());
10609 let (cotenant_channel, cotenant_epoch) =
10610 published_route(&cotenant_rx.recv().await.unwrap());
10611
10612 let caller = ConnectionId::new(32);
10614 let (caller_ctx, mut caller_rx) = route_ctx(caller);
10615 let (caller_task, caller_bind) = relay_route_open(
10616 &handler,
10617 caller,
10618 &caller_ctx.egress,
10619 &mut swap.incumbent_rx,
10620 101,
10621 "aft",
10622 "swap-caller",
10623 )
10624 .await;
10625 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
10626
10627 register_candidate(&swap, None);
10628 cutover(&swap);
10629
10630 let ack = handler
10632 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
10633 .await;
10634 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
10635 if module_loop_error.is_some() {
10636 handler.cleanup_connection(INCUMBENT).unwrap();
10639 }
10640
10641 assert!(
10643 cotenant_rx.try_recv().is_err(),
10644 "the co-tenant route on the incumbent was torn down by one late ack: \
10645 {module_loop_error:?}"
10646 );
10647 assert!(matches!(
10648 swap.forwarding
10649 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
10650 .unwrap(),
10651 DataRoute::Client(DataRouteState::Bound(_))
10652 ));
10653 assert_eq!(module_loop_error, None);
10654 assert!(swap
10655 .registry
10656 .get_module_by_connection(INCUMBENT)
10657 .unwrap()
10658 .is_some());
10659
10660 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
10662 .await
10663 .expect("the incumbent is told to drop the abandoned binding")
10664 .unwrap()
10665 .frame;
10666 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10667 assert_eq!(goodbye.header.channel, abandoned_channel);
10668 assert_eq!(goodbye.header.epoch, abandoned_epoch);
10669 assert!(swap.incumbent_rx.try_recv().is_err());
10670
10671 let response = caller_task.await.unwrap();
10673 assert_eq!(response.len(), 1);
10674 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
10675 assert!(caller_rx.try_recv().is_err());
10676
10677 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
10681 let counters = handler.counters().snapshot();
10682 assert_eq!(
10683 keyed_total(&counters, "route_open_accepted_by_principal"),
10684 1
10685 );
10686 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
10687 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
10688 }
10689
10690 #[tokio::test]
10694 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
10695 let mut swap = swap_with_incumbent().await;
10696 register_candidate(&swap, None);
10697 let incumbent = cutover(&swap);
10698 swap.forwarding
10699 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
10700 .unwrap()
10701 .expect("the incumbent is still registered");
10702
10703 let client = ConnectionId::new(33);
10704 let (client_ctx, mut client_rx) = route_ctx(client);
10705 let route_handler = swap.handler.clone();
10706 let open_ctx = RouteCtx {
10707 connection_id: client,
10708 egress: client_ctx.egress.clone(),
10709 };
10710 let mut route_task = tokio::spawn(async move {
10711 route_handler
10712 .handle_control_frame(
10713 &open_ctx,
10714 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
10715 )
10716 .await
10717 .unwrap()
10718 });
10719 let bind = tokio::select! {
10720 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
10721 response = &mut route_task => {
10722 let response = response.unwrap();
10723 panic!(
10724 "post-cutover route.open was refused instead of relayed to the candidate: {}",
10725 parse_error(&response[0])["code"]
10726 );
10727 }
10728 };
10729 swap.handler
10730 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
10731 .await
10732 .unwrap();
10733 assert!(route_task.await.unwrap().is_empty());
10734 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
10735 match swap
10736 .forwarding
10737 .lookup_data_route(client, channel, epoch)
10738 .unwrap()
10739 {
10740 DataRoute::Client(DataRouteState::Bound(route)) => {
10741 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
10742 }
10743 other => panic!("expected a bound route on the candidate, got {other:?}"),
10744 }
10745 assert!(swap.incumbent_rx.try_recv().is_err());
10746 }
10747
10748 #[tokio::test]
10753 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
10754 let swap = swap_with_incumbent().await;
10755 register_candidate(&swap, Some(false));
10756 let update = Frame::build(
10757 FrameType::Request,
10758 control_flags(),
10759 0,
10760 0,
10761 55,
10762 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
10763 provides: manifest("aft", PROTOCOL_VERSION).provides,
10764 capabilities: None,
10765 ready: Some(true),
10766 })
10767 .unwrap(),
10768 )
10769 .unwrap();
10770
10771 let replies = swap
10772 .handler
10773 .handle_control_frame(&swap.candidate_ctx, update)
10774 .await
10775 .unwrap();
10776
10777 assert_eq!(replies.len(), 1);
10778 assert_eq!(
10779 replies[0].header.ty,
10780 FrameType::Response,
10781 "candidate catalog.update was refused: {:?}",
10782 serde_json::from_slice::<Value>(&replies[0].body).ok()
10783 );
10784 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
10785 assert_eq!(
10786 swap.registry
10787 .get_module("aft")
10788 .unwrap()
10789 .unwrap()
10790 .connection_id,
10791 INCUMBENT
10792 );
10793 }
10794 }
10795
10796 mod swap_admission {
10800 use super::*;
10801
10802 const INCUMBENT_NONCE: &str = "incumbent-nonce";
10803 const CANDIDATE_NONCE: &str = "candidate-nonce";
10804
10805 fn handler_with_incumbent(
10806 module_id: &str,
10807 reserved: bool,
10808 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
10809 let registry = Arc::new(Registry::default());
10810 let supervisor = SupervisorHandle::new();
10811 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
10812 if reserved {
10813 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
10814 }
10815 let handler =
10816 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
10817 let incumbent = handler
10818 .handle_control(
10819 ConnectionId::new(1),
10820 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
10821 )
10822 .unwrap();
10823 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
10824 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
10825 (registry, supervisor, handler)
10826 }
10827
10828 #[test]
10834 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
10835 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
10836
10837 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
10838 let replies = handler
10839 .handle_control(
10840 ConnectionId::new(connection),
10841 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
10842 )
10843 .unwrap();
10844 assert_eq!(replies[0].header.ty, FrameType::Error);
10845 assert_eq!(
10846 parse_error(&replies[0])["code"],
10847 "swap_token_invalid",
10848 "nonce {nonce:?}"
10849 );
10850 }
10851 assert!(registry.get_candidate("aft").unwrap().is_none());
10852 assert_eq!(
10853 registry.get_module("aft").unwrap().unwrap().connection_id,
10854 ConnectionId::new(1)
10855 );
10856
10857 let admitted = handler
10860 .handle_control(
10861 ConnectionId::new(4),
10862 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
10863 )
10864 .unwrap();
10865 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
10866 assert_eq!(
10867 registry
10868 .get_candidate("aft")
10869 .unwrap()
10870 .unwrap()
10871 .connection_id,
10872 ConnectionId::new(4)
10873 );
10874 assert_eq!(
10875 registry.get_module("aft").unwrap().unwrap().connection_id,
10876 ConnectionId::new(1),
10877 "the candidate must not take the active slot"
10878 );
10879 let replayed = handler
10880 .handle_control(
10881 ConnectionId::new(5),
10882 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
10883 )
10884 .unwrap();
10885 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
10886
10887 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
10891 let squatter = handler
10892 .handle_control(
10893 ConnectionId::new(6),
10894 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
10895 )
10896 .unwrap();
10897 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
10898 assert!(
10899 registry.get_module("aft").unwrap().is_none(),
10900 "a squatter took the active slot of an id being swapped"
10901 );
10902 }
10903
10904 #[test]
10909 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
10910 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
10911
10912 let replies = handler
10913 .handle_control(
10914 ConnectionId::new(2),
10915 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
10916 )
10917 .unwrap();
10918
10919 assert_eq!(
10920 replies[0].header.ty,
10921 FrameType::HelloAck,
10922 "reserved candidate refused: {:?}",
10923 serde_json::from_slice::<Value>(&replies[0].body).ok()
10924 );
10925 assert_eq!(
10926 registry
10927 .get_candidate("vault")
10928 .unwrap()
10929 .unwrap()
10930 .connection_id,
10931 ConnectionId::new(2)
10932 );
10933 }
10934
10935 #[test]
10938 fn without_an_open_swap_the_ordinary_gates_decide() {
10939 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
10940 supervisor.close_swap("vault");
10941
10942 let candidate = handler
10943 .handle_control(
10944 ConnectionId::new(2),
10945 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
10946 )
10947 .unwrap();
10948 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
10949 let duplicate = handler
10950 .handle_control(
10951 ConnectionId::new(3),
10952 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
10953 )
10954 .unwrap();
10955 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
10956 assert!(registry.get_candidate("vault").unwrap().is_none());
10957 }
10958 }
10959}
10960
10961#[cfg(test)]
10962mod concurrency_default_exposure_tests {
10963 use super::*;
10964
10965 fn hello_body(role_json: &str) -> Vec<u8> {
10966 format!(
10967 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
10968 )
10969 .into_bytes()
10970 }
10971
10972 fn manifest_from(body: &[u8]) -> ModuleManifest {
10973 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
10974 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
10975 .expect("manifest parses")
10976 }
10977
10978 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
10979
10980 #[test]
10981 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
10982 let body = hello_body(&format!(
10983 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
10984 ));
10985 let manifest = manifest_from(&body);
10986 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
10989 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
10990 }
10991
10992 #[test]
10993 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
10994 let body = hello_body(&format!(
10995 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
10996 ));
10997 let manifest = manifest_from(&body);
10998 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
10999 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
11000 }
11001
11002 #[test]
11003 fn non_management_roles_are_never_reported() {
11004 let body = hello_body(
11005 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
11006 );
11007 let manifest = manifest_from(&body);
11008 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
11009 }
11010}