1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 path::{Path, PathBuf},
5 sync::{Arc, Mutex},
6 time::{Duration, Instant as StdInstant},
7};
8
9use serde::{Deserialize, Serialize};
10use subc_control::{
11 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
12 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
13 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
14 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
15 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
16 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
17 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
18};
19use subc_protocol::{
20 error_codes,
21 manifest::{
22 validate_hello_capability_grammar, validate_hello_self_signal_declarations,
23 CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
24 ProviderRole,
25 },
26 session::{
27 HealthReport, ModuleControlPush, ModuleControlRequest, ModuleControlRequestFromModule,
28 ModuleControlResponse, ModuleControlResponseToModule, MODULE_CONTROL_OP_HEALTH_CHECK,
29 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
30 },
31 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
32 Priority, RouteTarget, PROTOCOL_VERSION,
33};
34use tokio::time::{timeout_at, Instant};
35use tracing::{debug, info, warn};
36
37use crate::{
38 capability_requirements::{
39 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
40 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
41 RuntimeModule,
42 },
43 daemon_config::RestartRequiredSection,
44 forwarding::{
45 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
46 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
47 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
48 },
49 observability::{
50 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
51 },
52 provenance::{
53 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
54 },
55 registry::{ChannelState, ConnectionId, Registry, RegistryError},
56 router::{RouteCtx, RouterError},
57 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
58 stderr_tail::{CaptureState, TailEntry},
59 supervise::{
60 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
61 SupervisorHandle, SwapHelloAdmission,
62 },
63 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
64};
65
66pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
72
73const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
74const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
75const CAP_PING_PONG: &str = "ping_pong_v1";
76const CAP_SESSION_ATTACH: &str = "session_attach_v1";
77const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
78
79const SUBC_CONTROL_OPS: &[&str] = &[
80 ops::SERVER_DESCRIBE,
81 ops::CATALOG_LIST,
82 ops::ROUTE_OPEN,
83 ops::ROUTE_POLL,
84 ops::ROUTE_CLOSING,
85 ops::ROUTE_CLOSED,
86 ops::SUPERVISOR_LIST,
87 ops::SUPERVISOR_RESTART,
88 ops::SUPERVISOR_SWAP,
89 ops::SUPERVISOR_RELOAD,
90 ops::SUPERVISOR_RESCAN,
91 ops::SUPERVISOR_RELEASE_RESERVED,
92 ops::SUPERVISOR_SET_ENABLED,
93 ops::SUPERVISOR_HEALTH_PROBE,
94 ops::SUPERVISOR_HEALTH,
95 ops::SUPERVISOR_STDERR_TAIL,
96 ops::SUPERVISOR_TERMINALS,
97 ops::SUPERVISOR_ROUTES,
98 ops::SUPERVISOR_PROVENANCE,
99 ops::SUPERVISOR_SPAWN_SNAPSHOT,
100 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
101];
102
103const MODULE_TO_SUBC_CONTROL_OPS: &[&str] =
104 &[MODULE_TO_SUBC_OP_CATALOG_UPDATE, "supervisor.live_roots"];
105
106const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
107
108pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
116
117pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
132
133pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
151
152const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
153const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
154
155fn reload_verdict(
156 configured: &Path,
157 spawned_from: Option<&Path>,
158 image: subc_control::RunningImageAgreement,
159) -> PendingReloadVerdict {
160 let path = match spawned_from {
161 Some(spawned_from) if configured == spawned_from => ReloadPathAgreement::Match,
162 Some(spawned_from) => ReloadPathAgreement::Mismatch {
163 configured: configured.to_path_buf(),
164 spawned_from: spawned_from.to_path_buf(),
165 },
166 None => ReloadPathAgreement::Unavailable {
167 reason: if matches!(
168 image,
169 subc_control::RunningImageAgreement::Unavailable {
170 reason: subc_control::RunningImageUnavailableReason::NotRunning
171 }
172 ) {
173 ReloadPathUnavailableReason::NotRunning
174 } else {
175 ReloadPathUnavailableReason::SpawnedPathUnavailable
176 },
177 },
178 };
179 PendingReloadVerdict { path, image }
180}
181
182#[derive(Clone)]
183struct DaemonProvenanceFacts {
184 build: DaemonBuildProvenance,
185 pid: Option<u32>,
186 started_at_ms: Option<u64>,
187 start_clock: Option<crate::clock::StartClock>,
188 executable_path: Option<PathBuf>,
189 executable_identity: Option<SpawnedFileIdentity>,
190 process_start_time: Option<u64>,
191 probe: ExecutableIdentityProbe,
192}
193
194impl Default for DaemonProvenanceFacts {
195 fn default() -> Self {
196 Self {
197 build: DaemonBuildProvenance {
198 build_git_sha: None,
199 build_lock_digest: None,
200 },
201 pid: None,
202 started_at_ms: None,
203 start_clock: None,
204 executable_path: None,
205 executable_identity: None,
206 process_start_time: None,
207 probe: ExecutableIdentityProbe::default(),
208 }
209 }
210}
211
212#[derive(Debug, Clone)]
213struct SupervisorRescanContext {
214 supervisor: Supervisor,
215 config_path: PathBuf,
216 configured_port: Option<u16>,
217 storage_config: Option<crate::daemon_config::StorageConfig>,
218 admission_facts_carrier_module_id: Option<String>,
219 admission_facts_targets: Option<Vec<String>>,
220}
221
222const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
243 "reloading",
244 "supervisor_not_live",
245 "registration_not_active",
246 "no_forwarding_connection",
247 "relay_send_failed",
248];
249
250#[derive(Clone)]
252pub struct ControlHandler {
253 registry: Arc<Registry>,
254 forwarding: Arc<ForwardingTable>,
255 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
256 supervisor: SupervisorHandle,
257 subc_capabilities: Arc<[String]>,
258 route_bind_relay_timeout: Duration,
262 route_bind_relay_timeouts: BTreeMap<String, Duration>,
266 route_bind_breakers: RouteBindBreakers,
269 route_bind_concurrency: RouteBindConcurrency,
272 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
277 route_bind_breaker_threshold: u32,
279 route_bind_breaker_cooldown: Duration,
281 health_probe_timeout: Duration,
282 storage_config: Option<crate::daemon_config::StorageConfig>,
285 machine_id: Option<crate::machine_id::MachineId>,
289 admission_facts_carrier_module_id: Option<String>,
290 admission_facts_targets: Option<Vec<String>>,
291 rescan: Option<SupervisorRescanContext>,
292 connected_clients: ConnectedClients,
293 counters: DaemonCounters,
294 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
295 daemon_provenance: DaemonProvenanceFacts,
296 #[cfg(test)]
297 control_dispatch_delay: Option<Duration>,
298 #[cfg(test)]
299 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
300}
301
302impl fmt::Debug for ControlHandler {
303 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
304 f.debug_struct("ControlHandler")
305 .field("registry", &self.registry)
306 .field("forwarding", &self.forwarding)
307 .field("process_liveness", &self.process_liveness.is_some())
308 .field("supervisor", &self.supervisor)
309 .field("subc_capabilities", &self.subc_capabilities)
310 .finish()
311 }
312}
313
314struct RouteOpenRequest {
315 target: RouteTarget,
316 identity: BindIdentity,
317 consumer_identity: Option<ConsumerIdentity>,
318 consumer_capabilities: Option<Vec<String>>,
319 admission_facts: Option<serde_json::Value>,
320}
321
322struct RouteBindReservationGuard {
323 forwarding: Arc<ForwardingTable>,
324 endpoint: ModuleEndpointId,
325 relay_corr: u64,
326 armed: bool,
327}
328
329struct ModuleControlRpcGuard {
330 forwarding: Arc<ForwardingTable>,
331 endpoint: ModuleEndpointId,
332 corr: u64,
333 armed: bool,
334}
335
336impl ModuleControlRpcGuard {
337 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
338 Self {
339 forwarding,
340 endpoint,
341 corr,
342 armed: true,
343 }
344 }
345
346 fn disarm(&mut self) {
347 self.armed = false;
348 }
349}
350
351impl Drop for ModuleControlRpcGuard {
352 fn drop(&mut self) {
353 if self.armed {
354 let _ = self
355 .forwarding
356 .cancel_module_control_rpc(self.endpoint, self.corr);
357 }
358 }
359}
360
361impl RouteBindReservationGuard {
362 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
363 Self {
364 forwarding,
365 endpoint,
366 relay_corr,
367 armed: true,
368 }
369 }
370
371 fn release_and_disarm(&mut self) {
372 if !self.armed {
373 return;
374 }
375 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
376 self.endpoint,
377 self.relay_corr,
378 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
379 ) {
380 send_goodbye_target_best_effort(
381 &self.forwarding.counters(),
382 &target,
383 "canceled route.bind",
384 );
385 }
386 self.armed = false;
387 }
388
389 fn disarm(&mut self) {
390 self.armed = false;
391 }
392}
393
394impl Drop for RouteBindReservationGuard {
395 fn drop(&mut self) {
396 self.release_and_disarm();
397 }
398}
399
400#[derive(Debug, Clone, Default)]
430pub(crate) struct RouteBindBreakers {
431 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
432}
433
434#[derive(Debug, Clone, Default)]
435pub(crate) struct RouteBindConcurrency {
436 modules: Arc<Mutex<HashMap<String, usize>>>,
437}
438
439struct RouteBindConcurrencyGuard {
440 concurrency: RouteBindConcurrency,
441 module_id: String,
442}
443
444impl RouteBindConcurrency {
445 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
448 let mut modules = self
449 .modules
450 .lock()
451 .expect("route.bind concurrency mutex poisoned");
452 let in_flight = modules.entry(module_id.to_string()).or_default();
453 if *in_flight >= limit {
454 return Err(*in_flight);
455 }
456 *in_flight += 1;
457 Ok(RouteBindConcurrencyGuard {
458 concurrency: self.clone(),
459 module_id: module_id.to_string(),
460 })
461 }
462}
463
464impl Drop for RouteBindConcurrencyGuard {
465 fn drop(&mut self) {
466 let mut modules = self
467 .concurrency
468 .modules
469 .lock()
470 .expect("route.bind concurrency mutex poisoned");
471 let remove = {
472 let in_flight = modules
473 .get_mut(&self.module_id)
474 .expect("admitted route.bind has a concurrency entry");
475 *in_flight -= 1;
476 *in_flight == 0
477 };
478 if remove {
479 modules.remove(&self.module_id);
480 }
481 }
482}
483
484#[derive(Debug, Default)]
485struct ModuleBreakerState {
486 consecutive_timeouts: u32,
488 cooldown_until: Option<Instant>,
491 probe_in_flight: bool,
496}
497
498enum RouteBindAdmission<'a> {
500 Admitted {
501 guard: RouteBindBreakerGuard<'a>,
502 probe: bool,
505 },
506 Refused {
507 consecutive_timeouts: u32,
508 retry_in: Duration,
512 probe_in_flight: bool,
513 },
514}
515
516struct RouteBindBreakerGuard<'a> {
523 breakers: RouteBindBreakers,
524 module_id: &'a str,
525 settled: bool,
526}
527
528impl RouteBindBreakerGuard<'_> {
529 fn record_accepted(&mut self) -> bool {
533 self.settled = true;
534 self.breakers.record_accepted(self.module_id)
535 }
536
537 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
540 self.settled = true;
541 self.breakers
542 .record_timeout(self.module_id, threshold, cooldown)
543 }
544
545 fn record_inconclusive(&mut self) {
554 self.settled = true;
555 self.breakers.record_inconclusive(self.module_id);
556 }
557}
558
559impl Drop for RouteBindBreakerGuard<'_> {
560 fn drop(&mut self) {
561 if !self.settled {
562 self.breakers.record_inconclusive(self.module_id);
563 }
564 }
565}
566
567struct BreakerOpened {
571 consecutive_timeouts: u32,
572 reopened_after_probe: bool,
575}
576
577impl RouteBindBreakers {
578 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
579 self.modules
580 .lock()
581 .expect("route.bind breaker mutex poisoned")
582 }
583
584 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
587 let admitted = |probe| RouteBindAdmission::Admitted {
588 guard: RouteBindBreakerGuard {
589 breakers: self.clone(),
590 module_id,
591 settled: false,
592 },
593 probe,
594 };
595
596 let mut modules = self.lock();
597 let Some(state) = modules.get_mut(module_id) else {
598 return admitted(false);
599 };
600 let Some(cooldown_until) = state.cooldown_until else {
601 return admitted(false);
602 };
603 if state.probe_in_flight {
604 return RouteBindAdmission::Refused {
605 consecutive_timeouts: state.consecutive_timeouts,
606 retry_in: Duration::ZERO,
607 probe_in_flight: true,
608 };
609 }
610 let now = Instant::now();
611 if now < cooldown_until {
612 return RouteBindAdmission::Refused {
613 consecutive_timeouts: state.consecutive_timeouts,
614 retry_in: cooldown_until - now,
615 probe_in_flight: false,
616 };
617 }
618 state.probe_in_flight = true;
619 admitted(true)
620 }
621
622 fn record_accepted(&self, module_id: &str) -> bool {
623 self.lock()
624 .remove(module_id)
625 .is_some_and(|state| state.cooldown_until.is_some())
626 }
627
628 fn record_timeout(
629 &self,
630 module_id: &str,
631 threshold: u32,
632 cooldown: Duration,
633 ) -> Option<BreakerOpened> {
634 let mut modules = self.lock();
635 let state = modules.entry(module_id.to_string()).or_default();
636 let was_open = state.cooldown_until.is_some();
637 let was_probe = state.probe_in_flight;
638 state.probe_in_flight = false;
639 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
640 if state.consecutive_timeouts < threshold {
641 return None;
642 }
643 state.cooldown_until = Some(Instant::now() + cooldown);
644 Some(BreakerOpened {
645 consecutive_timeouts: state.consecutive_timeouts,
646 reopened_after_probe: was_open && was_probe,
647 })
648 }
649
650 fn record_inconclusive(&self, module_id: &str) {
651 if let Some(state) = self.lock().get_mut(module_id) {
652 state.probe_in_flight = false;
653 }
654 }
655
656 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
671 self.lock()
672 .remove(module_id)
673 .map(|state| state.consecutive_timeouts)
674 .filter(|discarded| *discarded > 0)
675 }
676
677 fn open_snapshot(&self) -> Option<serde_json::Value> {
684 let now = Instant::now();
685 let modules = self.lock();
686 let open = modules
687 .iter()
688 .filter_map(|(module_id, state)| {
689 let cooldown_until = state.cooldown_until?;
690 Some((
691 module_id.clone(),
692 serde_json::json!({
693 "consecutive_timeouts": state.consecutive_timeouts,
694 "cooldown_remaining_ms":
695 cooldown_until.saturating_duration_since(now).as_millis() as u64,
696 "probe_in_flight": state.probe_in_flight,
697 }),
698 ))
699 })
700 .collect::<serde_json::Map<String, serde_json::Value>>();
701 (!open.is_empty()).then_some(serde_json::Value::Object(open))
702 }
703}
704
705impl ControlHandler {
706 pub fn new(registry: Arc<Registry>) -> Self {
707 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
708 }
709
710 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
711 let counters = forwarding.counters();
712 let route_bind_breakers = forwarding.route_bind_breakers();
716 let route_bind_concurrency = forwarding.route_bind_concurrency();
717 let route_outages = forwarding.route_outages();
718 Self {
719 registry,
720 forwarding,
721 process_liveness: None,
722 supervisor: SupervisorHandle::new(),
723 subc_capabilities: Arc::from([
724 CAP_MANIFEST_REGISTRATION.to_string(),
725 CAP_CHANNEL_LIFECYCLE.to_string(),
726 CAP_PING_PONG.to_string(),
727 CAP_SESSION_ATTACH.to_string(),
728 CAP_ADMISSION_FACTS_RELAY.to_string(),
729 ]),
730 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
731 route_bind_relay_timeouts: BTreeMap::new(),
732 route_bind_breakers,
733 route_bind_concurrency,
734 route_outages,
735 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
736 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
737 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
738 storage_config: None,
739 machine_id: None,
740 admission_facts_carrier_module_id: None,
741 admission_facts_targets: None,
742 rescan: None,
743 connected_clients: ConnectedClients::new(),
744 counters,
745 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
746 daemon_provenance: DaemonProvenanceFacts::default(),
747 #[cfg(test)]
748 control_dispatch_delay: None,
749 #[cfg(test)]
750 provenance_probe_override: None,
751 }
752 }
753
754 pub fn with_storage_config(
757 mut self,
758 storage_config: Option<crate::daemon_config::StorageConfig>,
759 ) -> Self {
760 self.storage_config = storage_config;
761 self
762 }
763
764 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
767 self.machine_id = machine_id;
768 self
769 }
770
771 pub fn with_admission_facts_config(
775 mut self,
776 carrier_module_id: Option<String>,
777 targets: Option<Vec<String>>,
778 ) -> Self {
779 self.admission_facts_carrier_module_id = carrier_module_id;
780 self.admission_facts_targets = targets;
781 self
782 }
783
784 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
787 self.route_bind_relay_timeout = timeout;
788 self
789 }
790
791 pub fn with_route_bind_relay_timeouts(
797 mut self,
798 timeouts: impl IntoIterator<Item = (String, Duration)>,
799 ) -> Self {
800 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
801 self
802 }
803
804 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
810 self.route_bind_relay_timeouts
811 .get(module_id)
812 .copied()
813 .unwrap_or(self.route_bind_relay_timeout)
814 }
815
816 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
824 self.route_bind_breaker_threshold = threshold.max(1);
825 self.route_bind_breaker_cooldown = cooldown;
826 self
827 }
828
829 #[cfg(test)]
830 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
831 self.health_probe_timeout = timeout;
832 self
833 }
834
835 #[cfg(test)]
836 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
837 self.control_dispatch_delay = Some(delay);
838 self
839 }
840
841 pub fn with_process_liveness(
842 mut self,
843 process_liveness: Arc<dyn ModuleProcessLiveness>,
844 ) -> Self {
845 self.process_liveness = Some(process_liveness);
846 self
847 }
848
849 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
850 self.supervisor = supervisor;
851 self
852 }
853
854 pub fn with_daemon_provenance(
855 mut self,
856 pid: u32,
857 started_at_ms: u64,
858 executable_path: Option<PathBuf>,
859 build_git_sha: Option<String>,
860 build_lock_digest: Option<String>,
861 ) -> Self {
862 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
863 let process_start_time = process_start_time(pid);
864 self.daemon_provenance = DaemonProvenanceFacts {
865 build: DaemonBuildProvenance {
866 build_git_sha,
867 build_lock_digest,
868 },
869 pid: Some(pid),
870 started_at_ms: Some(started_at_ms),
871 start_clock: None,
872 executable_path,
873 executable_identity,
874 process_start_time,
875 probe: ExecutableIdentityProbe::default(),
876 };
877 self
878 }
879
880 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
881 self.daemon_provenance.start_clock = Some(clock);
882 self
883 }
884
885 #[cfg(test)]
886 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
887 self.provenance_probe_override = Some(result);
888 self
889 }
890
891 pub fn with_capability_config(
895 self,
896 modules: impl IntoIterator<Item = (String, bool)>,
897 reserved_capabilities: BTreeMap<String, String>,
898 ) -> Self {
899 self.capability_evaluator
900 .configure(modules, reserved_capabilities);
901 self
902 }
903
904 pub fn with_supervisor_rescan(
905 mut self,
906 supervisor: Supervisor,
907 config_path: impl Into<PathBuf>,
908 configured_port: Option<u16>,
909 ) -> Self {
910 self.rescan = Some(SupervisorRescanContext {
911 supervisor,
912 config_path: config_path.into(),
913 configured_port,
914 storage_config: self.storage_config.clone(),
915 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
916 admission_facts_targets: self.admission_facts_targets.clone(),
917 });
918 self
919 }
920
921 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
922 self.connected_clients = connected_clients;
923 self
924 }
925
926 pub fn forwarding(&self) -> Arc<ForwardingTable> {
927 Arc::clone(&self.forwarding)
928 }
929
930 pub(crate) fn counters(&self) -> DaemonCounters {
931 self.counters.clone()
932 }
933
934 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
937 tokio::spawn(async move {
938 loop {
939 self.capability_evaluator
940 .wait_for_change_or_deadline()
941 .await;
942 self.refresh_capability_requirements();
943 }
944 });
945 }
946
947 fn runtime_capability_snapshot(
948 &self,
949 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
950 let runtime = self
951 .supervisor
952 .list()
953 .into_iter()
954 .map(|module| {
955 let status = module.status().map_err(|err| {
956 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
957 })?;
958 Ok(RuntimeModule {
959 module_id: status.module_id,
960 state: status.state,
961 enabled: status.enabled,
962 })
963 })
964 .collect::<Result<Vec<_>, RouterError>>()?;
965 let (_, registrations) = self.registry.list_modules().map_err(|err| {
966 RouterError::backend(
967 0,
968 0,
969 format!("failed to list capability registrations: {err}"),
970 )
971 })?;
972 let registrations = registrations
973 .into_iter()
974 .map(|registration| RegisteredModule {
975 module_id: registration.manifest.module_id,
976 module_version: registration.manifest.module_version,
977 capabilities: registration.manifest.capabilities,
978 })
979 .collect();
980 Ok((runtime, registrations))
981 }
982
983 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
990 let cached_registration = RegisteredModule {
991 module_id: registration.manifest.module_id.clone(),
992 module_version: registration.manifest.module_version.clone(),
993 capabilities: registration.manifest.capabilities.clone(),
994 };
995 if self.capability_evaluator.record_hello(&cached_registration) {
996 warn!(
997 module_id = %cached_registration.module_id,
998 "capability claims drifted from the cached manifest"
999 );
1000 }
1001 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1002 self.enforce_capability_denies();
1003 }
1004 self.refresh_capability_requirements();
1005 }
1006
1007 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1011 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1012 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1013 self.supervisor.set_swap_promotion_observer(observer);
1014 }
1015
1016 pub fn refresh_capability_requirements(&self) {
1017 match self.runtime_capability_snapshot() {
1018 Ok((runtime, registrations)) => {
1019 log_requirement_events(
1020 self.capability_evaluator
1021 .evaluate_now(&runtime, ®istrations),
1022 );
1023 }
1024 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1025 }
1026 }
1027
1028 fn enforce_capability_denies(&self) {
1032 let (_, registrations) = match self.registry.list_modules() {
1033 Ok(snapshot) => snapshot,
1034 Err(err) => {
1035 warn!(error = %err, "failed to read registrations for capability deny census");
1036 return;
1037 }
1038 };
1039 let manifests = registrations
1040 .into_iter()
1041 .map(|registration| {
1042 (
1043 registration.manifest.module_id.clone(),
1044 registration.manifest,
1045 )
1046 })
1047 .collect::<BTreeMap<_, _>>();
1048 let census = match self.forwarding.route_census(None) {
1049 Ok(census) => census,
1050 Err(err) => {
1051 warn!(error = %err, "failed to read route census for capability deny enforcement");
1052 return;
1053 }
1054 };
1055
1056 for (target_module_id, routes) in census {
1057 let Some(target_manifest) = manifests.get(&target_module_id) else {
1058 continue;
1059 };
1060 let mut closed_routes = Vec::new();
1061 let mut module_goodbyes = Vec::new();
1062 for route in routes {
1063 let Principal::Reserved {
1064 module_id: opening_module_id,
1065 } = &route.principal
1066 else {
1067 continue;
1068 };
1069 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1070 continue;
1071 };
1072 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1073 continue;
1074 };
1075
1076 match self.forwarding.release_client_route(
1077 route.goodbye_target.connection_id,
1078 route.goodbye_target.channel,
1079 route.goodbye_target.epoch,
1080 ) {
1081 Ok(RouteRelease::Removed(module_goodbye)) => {
1082 warn!(
1083 opening_module_id,
1084 target_module_id,
1085 capability,
1086 "force-closing route because an attested capability deny edge now matches"
1087 );
1088 closed_routes.push(route);
1089 module_goodbyes.push(module_goodbye);
1090 }
1091 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1092 Err(err) => warn!(
1093 opening_module_id,
1094 target_module_id,
1095 capability,
1096 error = %err,
1097 "failed to force-close capability-denied route"
1098 ),
1099 }
1100 }
1101
1102 if closed_routes.is_empty() {
1103 continue;
1104 }
1105 send_route_control_pushes(
1106 &self.forwarding,
1107 closed_routes,
1108 ClientControlPush::RouteClosed {
1109 module_id: target_module_id,
1110 reason: RouteCloseReason::CapabilityDenied,
1111 drained: false,
1112 abandoned: 0,
1113 excluded_subscriptions: 0,
1114 terminal: Some(false),
1115 },
1116 );
1117 self.emit_route_goodbyes(module_goodbyes);
1118 }
1119 }
1120
1121 fn not_ready_reason(
1127 &self,
1128 registration: &crate::registry::ModuleRegistration,
1129 ) -> Option<NotReadyReason> {
1130 if !registration.ready {
1131 return Some(NotReadyReason {
1132 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1133 capability: None,
1134 });
1135 }
1136 self.first_unprovided_required_capability(registration)
1137 .map(|capability| NotReadyReason {
1138 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1139 capability: Some(capability),
1140 })
1141 }
1142
1143 fn first_unprovided_required_capability(
1161 &self,
1162 registration: &crate::registry::ModuleRegistration,
1163 ) -> Option<String> {
1164 let required = registration
1165 .manifest
1166 .capabilities
1167 .iter()
1168 .flat_map(|declarations| declarations.requires.iter())
1169 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1170 .map(|requirement| requirement.capability.as_str())
1171 .collect::<BTreeSet<_>>();
1172 if required.is_empty() {
1173 return None;
1174 }
1175 let module_id = registration.manifest.module_id.as_str();
1176 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1177 if required
1178 .iter()
1179 .any(|capability| verdict(capability).is_none())
1180 {
1181 self.refresh_capability_requirements();
1182 }
1183 required
1184 .into_iter()
1185 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1186 .map(str::to_string)
1187 }
1188
1189 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1190 self.capability_evaluator
1191 .statuses()
1192 .into_iter()
1193 .map(capability_requirement_status)
1194 .collect()
1195 }
1196
1197 fn deregister_connection(
1204 &self,
1205 connection_id: ConnectionId,
1206 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1207 self.registry.deregister_connection(connection_id)
1208 }
1209
1210 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1211 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1212 return None;
1213 }
1214 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1215 parse_client_control_request(&frame.body)
1216 else {
1217 return None;
1218 };
1219 Some(target_module_id(&target).to_string())
1220 }
1221
1222 pub(crate) fn route_open_capacity_refusal(
1223 &self,
1224 ctx: &RouteCtx,
1225 frame: &Frame,
1226 target_module_id: &str,
1227 in_flight: usize,
1228 limit: usize,
1229 ) -> Result<Frame, RouterError> {
1230 self.route_open_admission_refusal_frame(
1231 ctx,
1232 frame,
1233 target_module_id,
1234 "open_admission_full",
1235 (in_flight, limit),
1236 format!(
1237 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1238 ),
1239 )
1240 }
1241
1242 fn route_open_target_capacity_refusal(
1243 &self,
1244 ctx: &RouteCtx,
1245 frame: &Frame,
1246 target_module_id: &str,
1247 in_flight: usize,
1248 ) -> Result<Frame, RouterError> {
1249 self.route_open_admission_refusal_frame(
1250 ctx,
1251 frame,
1252 target_module_id,
1253 "target_binds_full",
1254 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1255 format!(
1256 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1257 ),
1258 )
1259 }
1260
1261 fn route_open_admission_refusal_frame(
1269 &self,
1270 ctx: &RouteCtx,
1271 frame: &Frame,
1272 target_module_id: &str,
1273 reason: &'static str,
1274 (in_flight, limit): (usize, usize),
1275 message: impl Into<String>,
1276 ) -> Result<Frame, RouterError> {
1277 let code = error_codes::TARGET_UNAVAILABLE;
1278 self.counters.increment_route_open_refused(code);
1279 info!(
1280 target: "control",
1281 code,
1282 reason,
1283 module_id = ?target_module_id,
1284 connection_id = ctx.connection_id.get(),
1285 in_flight,
1286 limit,
1287 "route.open refused"
1288 );
1289 control_error_frame(frame, code, message.into())
1290 }
1291
1292 #[cfg(test)]
1298 pub fn handle_control(
1299 &self,
1300 connection_id: ConnectionId,
1301 frame: Frame,
1302 ) -> Result<Vec<Frame>, RouterError> {
1303 match frame.header.ty {
1304 FrameType::Ping => Ok(vec![pong(&frame)?]),
1305 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1306 FrameType::Goodbye => self.handle_goodbye(connection_id),
1307 ty => Ok(vec![control_error_frame(
1308 &frame,
1309 "unsupported_control_frame",
1310 format!("unsupported channel-0 frame {ty:?}"),
1311 )?]),
1312 }
1313 }
1314
1315 pub async fn handle_control_frame(
1316 &self,
1317 ctx: &RouteCtx,
1318 frame: Frame,
1319 ) -> Result<Vec<Frame>, RouterError> {
1320 self.handle_control_frame_timed(ctx, frame, None).await
1321 }
1322
1323 pub(crate) async fn handle_control_frame_timed(
1324 &self,
1325 ctx: &RouteCtx,
1326 frame: Frame,
1327 dispatch_started_at: Option<StdInstant>,
1328 ) -> Result<Vec<Frame>, RouterError> {
1329 match frame.header.ty {
1330 FrameType::Ping => Ok(vec![pong(&frame)?]),
1331 FrameType::Hello => {
1332 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1333 }
1334 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1335 FrameType::Cancel => {
1336 if self
1337 .supervisor
1338 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1339 {
1340 Ok(Vec::new())
1341 } else {
1342 Ok(vec![control_error_frame(
1343 &frame,
1344 "unknown_subscription",
1345 "no supervisor spawn subscription has this correlation id",
1346 )?])
1347 }
1348 }
1349 FrameType::Request => {
1350 if self
1351 .forwarding
1352 .module_endpoint_for_connection(ctx.connection_id)
1353 .map_err(RouterError::Forwarding)?
1354 .is_some()
1355 {
1356 if !is_known_module_request_op(&frame.body) {
1357 return Ok(vec![control_error_frame(
1358 &frame,
1359 "unsupported_control_frame",
1360 "module-originated channel-0 REQUEST is not supported",
1361 )?]);
1362 }
1363 let request = match parse_module_control_request_from_module(&frame.body) {
1364 Ok(request) => request,
1365 Err((err, ControlRequestBodyError::UnknownOp)) => {
1366 return Ok(vec![control_error_frame(
1367 &frame,
1368 "unsupported_control_frame",
1369 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1370 )?])
1371 }
1372 Err((err, ControlRequestBodyError::InvalidBody)) => {
1373 return Ok(vec![control_error_frame(
1374 &frame,
1375 "invalid_control_body",
1376 format!("malformed module control body: {err}"),
1377 )?])
1378 }
1379 };
1380 let op = module_control_request_op(&request);
1381 let corr = frame.header.corr;
1382 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1383 let result =
1384 self.handle_module_control_request(ctx.connection_id, frame, request);
1385 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1386 return result;
1387 }
1388
1389 if is_known_module_request_op(&frame.body) {
1390 return Ok(vec![control_error_frame(
1391 &frame,
1392 "not_registered",
1393 "catalog.update requires an active module registration owned by this connection",
1394 )?]);
1395 }
1396
1397 let request = match parse_client_control_request(&frame.body) {
1398 Ok(request) => request,
1399 Err((err, ControlRequestBodyError::UnknownOp)) => {
1400 return Ok(vec![control_error_frame(
1401 &frame,
1402 "unknown_control_op",
1403 format!("unknown client control op: {err}"),
1404 )?])
1405 }
1406 Err((err, ControlRequestBodyError::InvalidBody)) => {
1407 return Ok(vec![control_error_frame(
1408 &frame,
1409 "invalid_control_body",
1410 format!("malformed client control body: {err}"),
1411 )?])
1412 }
1413 };
1414 let op = client_control_request_op(&request);
1415 let corr = frame.header.corr;
1416 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1417 #[cfg(test)]
1418 if let Some(delay) = self.control_dispatch_delay {
1419 tokio::time::sleep(delay).await;
1420 }
1421 let result = self
1422 .handle_client_control_request(ctx, frame, request)
1423 .await;
1424 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1425 result
1426 }
1427 FrameType::Push => {
1428 let Some(endpoint) = self
1429 .forwarding
1430 .module_endpoint_for_connection(ctx.connection_id)
1431 .map_err(RouterError::Forwarding)?
1432 else {
1433 return Ok(vec![control_error_frame(
1434 &frame,
1435 "unsupported_control_frame",
1436 "client-originated channel-0 PUSH is not supported",
1437 )?]);
1438 };
1439 self.handle_status_update(endpoint, frame)
1440 }
1441 FrameType::Response | FrameType::Error
1442 if self
1443 .forwarding
1444 .module_endpoint_for_connection(ctx.connection_id)
1445 .map_err(RouterError::Forwarding)?
1446 .is_some() =>
1447 {
1448 self.handle_module_relay_response(ctx.connection_id, frame)
1449 }
1450 ty => Ok(vec![control_error_frame(
1451 &frame,
1452 "unsupported_control_frame",
1453 format!("unsupported channel-0 frame {ty:?}"),
1454 )?]),
1455 }
1456 }
1457
1458 pub fn cleanup_connection(
1459 &self,
1460 connection_id: ConnectionId,
1461 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1462 let crash_closed = self
1463 .registry
1464 .get_module_by_connection(connection_id)?
1465 .and_then(|registration| {
1466 self.forwarding
1467 .module_endpoint_for_connection(connection_id)
1468 .ok()
1469 .flatten()
1470 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1471 .map(|routes| (registration.manifest.module_id, routes))
1472 });
1473 let crash_closed = crash_closed.map(|(module_id, routes)| {
1474 let terminal = match self.supervisor.get(&module_id) {
1475 None => false,
1476 Some(module) => match module.will_recover_after_connection_loss() {
1477 Ok(will_recover) => !will_recover,
1478 Err(err) => {
1479 warn!(
1480 %module_id,
1481 error = %err,
1482 "failed to read crash recovery verdict; reporting non-terminal conservatively"
1483 );
1484 false
1485 }
1486 },
1487 };
1488 let reason = match self.forwarding.is_daemon_draining() {
1492 Ok(true) => RouteCloseReason::Restart,
1493 Ok(false) => RouteCloseReason::Crash,
1494 Err(err) => {
1495 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
1496 RouteCloseReason::Crash
1497 }
1498 };
1499 (module_id, routes, reason, terminal)
1500 });
1501 let registrations = self.deregister_connection(connection_id);
1502 let cleanup = self.forwarding.cleanup_connection_counted(connection_id);
1503 if let Some((module_id, routes, reason, terminal)) = crash_closed {
1508 let abandoned = cleanup
1509 .as_ref()
1510 .map_or(0, |cleanup| cleanup.abandoned_relays);
1511 send_route_control_pushes(
1512 &self.forwarding,
1513 routes,
1514 ClientControlPush::RouteClosed {
1515 module_id,
1516 reason,
1517 drained: false,
1518 abandoned,
1519 excluded_subscriptions: 0,
1520 terminal: Some(terminal),
1521 },
1522 );
1523 }
1524 if let Ok(cleanup) = cleanup {
1525 self.emit_route_goodbyes(cleanup.released);
1526 }
1527 if matches!(®istrations, Ok(r) if !r.is_empty()) {
1531 crate::supervise::notify_registration_release();
1532 self.capability_evaluator.wake_deadline_loop();
1533 self.refresh_capability_requirements();
1534 }
1535 self.supervisor.remove_spawn_subscribers(connection_id);
1536 registrations
1537 }
1538
1539 pub(crate) fn handle_route_goodbye(
1540 &self,
1541 connection_id: ConnectionId,
1542 route_channel: u16,
1543 route_epoch: u32,
1544 ) -> Result<bool, RouterError> {
1545 debug!(
1546 connection_id = connection_id.get(),
1547 route_channel, route_epoch, "handling route GOODBYE"
1548 );
1549 let RouteRelease::Removed(released_route) = self
1550 .forwarding
1551 .release_client_route(connection_id, route_channel, route_epoch)
1552 .map_err(RouterError::Forwarding)?
1553 else {
1554 return Ok(false);
1555 };
1556 self.emit_route_goodbyes(vec![released_route]);
1557 Ok(true)
1558 }
1559
1560 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
1561 for released in released_routes {
1562 let frame = match Frame::build_with_version(
1563 released.negotiated_ver,
1564 FrameType::Goodbye,
1565 control_flags(),
1566 released.channel,
1567 released.epoch,
1568 0,
1569 Vec::new(),
1570 ) {
1571 Ok(frame) => frame,
1572 Err(err) => {
1573 warn!(
1574 route_channel = released.channel,
1575 error = %err,
1576 "failed to build route GOODBYE frame"
1577 );
1578 continue;
1579 }
1580 };
1581 if !released.close_on_delivery_failure() {
1582 crate::forwarding::send_module_route_goodbye(
1583 &self.counters,
1584 &released.sink,
1585 frame,
1586 released.module_id.as_deref(),
1587 "client route released",
1588 );
1589 continue;
1590 }
1591 if let Err(err) = released.sink.try_send(frame) {
1592 warn!(
1593 target_connection_id = released.connection_id.get(),
1594 route_channel = released.channel,
1595 error = %err,
1596 "route GOODBYE was not delivered to client; closing target connection"
1597 );
1598 if self
1599 .forwarding
1600 .escalate_client_delivery_failure(
1601 released.connection_id,
1602 released.channel,
1603 released.epoch,
1604 CloseReason::new(
1605 "route_goodbye_delivery_failed",
1606 format!(
1607 "failed to enqueue route GOODBYE for channel {}: {err}",
1608 released.channel
1609 ),
1610 ),
1611 crate::forwarding::UndeliveredFrame {
1612 module_id: released.module_id.as_deref(),
1613 sink: &released.sink,
1614 },
1615 )
1616 .unwrap_or(false)
1617 {
1618 self.counters.increment_goodbye_relay_client_failed();
1619 }
1620 }
1621 }
1622 }
1623
1624 fn send_abandoned_route_bind_goodbye(
1636 &self,
1637 module_sink: &crate::FrameSink,
1638 negotiated_ver: u8,
1639 module_channel: u16,
1640 module_epoch: u32,
1641 ) {
1642 let frame = match Frame::build_with_version(
1643 negotiated_ver,
1644 FrameType::Goodbye,
1645 control_flags(),
1646 module_channel,
1647 module_epoch,
1648 0,
1649 Vec::new(),
1650 ) {
1651 Ok(frame) => frame,
1652 Err(err) => {
1653 warn!(
1654 route_channel = module_channel,
1655 error = %err,
1656 "failed to build GOODBYE for abandoned route.bind"
1657 );
1658 return;
1659 }
1660 };
1661 crate::forwarding::send_module_route_goodbye(
1662 &self.counters,
1663 module_sink,
1664 frame,
1665 None,
1666 "abandoned route.bind",
1667 );
1668 }
1669
1670 fn handle_hello(
1671 &self,
1672 connection_id: ConnectionId,
1673 sink: Option<crate::FrameSink>,
1674 frame: Frame,
1675 ) -> Result<Vec<Frame>, RouterError> {
1676 debug!(
1677 connection_id = connection_id.get(),
1678 corr = frame.header.corr,
1679 "handling HELLO"
1680 );
1681 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
1682 Ok(value) => value,
1683 Err(err) => {
1684 return Ok(vec![control_error_frame(
1685 &frame,
1686 "invalid_hello",
1687 format!("malformed HELLO body: {err}"),
1688 )?])
1689 }
1690 };
1691 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
1692 return Ok(vec![control_error_frame(
1693 &frame,
1694 "invalid_capability_grammar",
1695 err.to_string(),
1696 )?]);
1697 }
1698 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
1699 return Ok(vec![control_error_frame(
1700 &frame,
1701 "invalid_manifest",
1702 err.to_string(),
1703 )?]);
1704 }
1705 if let Some(provenance) = hello_value
1706 .get("manifest")
1707 .and_then(|manifest| manifest.get("provenance"))
1708 {
1709 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
1710 return Ok(vec![control_error_frame(
1711 &frame,
1712 "invalid_manifest",
1713 format!("malformed manifest provenance: {err}"),
1714 )?]);
1715 }
1716 }
1717 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
1718 Ok(hello) => hello,
1719 Err(err) => {
1720 return Ok(vec![control_error_frame(
1721 &frame,
1722 "invalid_hello",
1723 format!("malformed HELLO body: {err}"),
1724 )?])
1725 }
1726 };
1727
1728 if hello.protocol_ver != hello.manifest.protocol_ver {
1729 return Ok(vec![control_error_frame(
1730 &frame,
1731 "invalid_manifest",
1732 format!(
1733 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
1734 hello.protocol_ver, hello.manifest.protocol_ver
1735 ),
1736 )?]);
1737 }
1738
1739 if hello.manifest.module_id.trim().is_empty() {
1740 return Ok(vec![control_error_frame(
1741 &frame,
1742 "invalid_manifest",
1743 "manifest module_id must not be empty",
1744 )?]);
1745 }
1746
1747 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
1748 Ok(negotiated_ver) => negotiated_ver,
1749 Err(message) => {
1750 return Ok(vec![control_error_frame(
1751 &frame,
1752 "version_unsupported",
1753 message,
1754 )?])
1755 }
1756 };
1757
1758 let swap_admission = self
1766 .supervisor
1767 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
1768 if swap_admission == SwapHelloAdmission::Refused {
1769 warn!(
1770 module_id = %hello.manifest.module_id,
1771 connection_id = connection_id.get(),
1772 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
1773 );
1774 return Ok(vec![control_error_frame(
1775 &frame,
1776 "swap_token_invalid",
1777 format!(
1778 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
1779 hello.manifest.module_id
1780 ),
1781 )?]);
1782 }
1783 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
1784
1785 if let Some(rejection) = (!swap_candidate)
1793 .then(|| {
1794 self.supervisor.reserved_hello_rejection(
1795 &hello.manifest.module_id,
1796 hello.launch_nonce.as_deref(),
1797 )
1798 })
1799 .flatten()
1800 {
1801 let message = match rejection {
1802 ReservedHelloRejection::Exact { module_id } => format!(
1803 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
1804 ),
1805 ReservedHelloRejection::Prefix {
1806 prefix,
1807 owner_module_id,
1808 } => format!(
1809 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
1810 hello.manifest.module_id
1811 ),
1812 };
1813 return Ok(vec![control_error_frame(
1814 &frame,
1815 "reserved_module",
1816 message,
1817 )?]);
1818 }
1819
1820 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
1821 &hello.manifest.module_id,
1822 hello.manifest.capabilities.as_ref(),
1823 );
1824 if let Some(refusal) = reserved_capability_refusals.first() {
1825 let capability = refusal.capability.clone();
1826 let bound_module = refusal.claimants[0].clone();
1827 log_duplicate_claim_events(reserved_capability_refusals);
1828 return Ok(vec![control_error_frame(
1829 &frame,
1830 "reserved_capability",
1831 format!(
1832 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
1833 capability, bound_module, hello.manifest.module_id
1834 ),
1835 )?]);
1836 }
1837
1838 if self
1841 .forwarding
1842 .connection_has_client_routes(connection_id)
1843 .map_err(RouterError::Forwarding)?
1844 {
1845 return Ok(vec![control_error_frame(
1846 &frame,
1847 "invalid_hello",
1848 "connection has open client routes and cannot also register as a module",
1849 )?]);
1850 }
1851
1852 let control_ops = effective_module_control_ops(hello.control_ops);
1853 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
1856 if swap_candidate {
1857 return self.register_swap_candidate(
1858 connection_id,
1859 sink,
1860 &frame,
1861 hello.manifest,
1862 negotiated_ver,
1863 control_ops,
1864 hello_ack,
1865 );
1866 }
1867 let registration = match self.registry.register_with_control_ops(
1868 hello.manifest,
1869 negotiated_ver,
1870 connection_id,
1871 control_ops,
1872 ) {
1873 Ok(registration) => registration,
1874 Err(RegistryError::DuplicateModuleId { module_id }) => {
1875 return Ok(vec![control_error_frame(
1876 &frame,
1877 "duplicate_module_id",
1878 format!(
1879 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
1880 ),
1881 )?])
1882 }
1883 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
1884 return Ok(vec![control_error_frame(
1885 &frame,
1886 "invalid_module_id",
1887 err.to_string(),
1888 )?])
1889 }
1890 Err(err) => {
1891 return Ok(vec![control_error_frame(
1892 &frame,
1893 "registry_error",
1894 err.to_string(),
1895 )?])
1896 }
1897 };
1898
1899 let reply = if let Some(sink) = sink {
1900 let concurrency = manifest_concurrency(®istration.manifest);
1916 if let Err(err) = self.forwarding.register_module_connection_acked(
1917 connection_id,
1918 registration.manifest.module_id.clone(),
1919 negotiated_ver,
1920 concurrency,
1921 sink,
1922 hello_ack,
1923 ) {
1924 if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
1928 crate::supervise::notify_registration_release();
1929 }
1930 return Ok(vec![control_error_frame(
1931 &frame,
1932 forwarding_error_code(&err),
1933 err.to_string(),
1934 )?]);
1935 }
1936 Vec::new()
1937 } else {
1938 vec![hello_ack]
1941 };
1942
1943 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
1952 info!(
1953 module_id = %registration.manifest.module_id,
1954 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
1955 );
1956 }
1957
1958 self.apply_registration_capabilities(®istration);
1959
1960 info!(
1961 module_id = %registration.manifest.module_id,
1962 module_version = %registration.manifest.module_version,
1963 negotiated_ver,
1964 routable_provider = manifest_provides_routable_role(®istration.manifest),
1965 connection_id = connection_id.get(),
1966 "module registered"
1967 );
1968
1969 Ok(reply)
1970 }
1971
1972 #[allow(clippy::too_many_arguments)]
1982 fn register_swap_candidate(
1983 &self,
1984 connection_id: ConnectionId,
1985 sink: Option<crate::FrameSink>,
1986 frame: &Frame,
1987 manifest: ModuleManifest,
1988 negotiated_ver: u8,
1989 control_ops: Vec<String>,
1990 hello_ack: Frame,
1991 ) -> Result<Vec<Frame>, RouterError> {
1992 let module_id = manifest.module_id.clone();
1993 let registration = match self.registry.register_candidate_with_control_ops(
1994 manifest,
1995 negotiated_ver,
1996 connection_id,
1997 control_ops,
1998 ) {
1999 Ok(registration) => registration,
2000 Err(RegistryError::DuplicateModuleId { module_id }) => {
2001 return Ok(vec![control_error_frame(
2002 frame,
2003 "duplicate_module_id",
2004 format!(
2005 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2006 ),
2007 )?])
2008 }
2009 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2010 return Ok(vec![control_error_frame(
2011 frame,
2012 "invalid_module_id",
2013 err.to_string(),
2014 )?])
2015 }
2016 Err(err) => {
2017 return Ok(vec![control_error_frame(
2018 frame,
2019 "registry_error",
2020 err.to_string(),
2021 )?])
2022 }
2023 };
2024 let reply = if let Some(sink) = sink {
2025 let concurrency = manifest_concurrency(®istration.manifest);
2029 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2030 connection_id,
2031 module_id.clone(),
2032 negotiated_ver,
2033 concurrency,
2034 sink,
2035 hello_ack,
2036 ) {
2037 if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
2038 crate::supervise::notify_registration_release();
2039 }
2040 return Ok(vec![control_error_frame(
2041 frame,
2042 forwarding_error_code(&err),
2043 err.to_string(),
2044 )?]);
2045 }
2046 Vec::new()
2047 } else {
2048 vec![hello_ack]
2049 };
2050 self.supervisor.mark_swap_candidate_admitted(&module_id);
2051 info!(
2052 module_id = %module_id,
2053 module_version = %registration.manifest.module_version,
2054 negotiated_ver,
2055 ready = registration.ready,
2056 connection_id = connection_id.get(),
2057 "swap candidate registered; not routable until cutover"
2058 );
2059 Ok(reply)
2060 }
2061
2062 fn build_hello_ack(
2063 &self,
2064 frame: &Frame,
2065 negotiated_ver: u8,
2066 module_id: &str,
2067 ) -> Result<Frame, RouterError> {
2068 let ack = ModuleHelloAckBody {
2069 negotiated_ver,
2070 subc_ops: module_subc_ops(),
2071 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2072 storage: self
2073 .storage_config
2074 .as_ref()
2075 .map(|cfg| cfg.descriptor_for(module_id)),
2076 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2077 };
2078 let body = serde_json::to_vec(&ack).map_err(|err| {
2079 RouterError::backend(
2080 0,
2081 frame.header.corr,
2082 format!("failed to encode HELLO_ACK: {err}"),
2083 )
2084 })?;
2085
2086 Frame::build_with_version(
2087 negotiated_ver,
2088 FrameType::HelloAck,
2089 control_flags(),
2090 0,
2091 0,
2092 frame.header.corr,
2093 body,
2094 )
2095 .map_err(RouterError::FrameBuild)
2096 }
2097
2098 async fn handle_client_control_request(
2099 &self,
2100 ctx: &RouteCtx,
2101 frame: Frame,
2102 request: ClientControlRequest,
2103 ) -> Result<Vec<Frame>, RouterError> {
2104 match request {
2105 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2106 ClientControlRequest::CatalogList { module_id } => {
2107 self.handle_catalog_list(frame, module_id)
2108 }
2109 ClientControlRequest::RouteOpen {
2110 target,
2111 identity,
2112 consumer_identity,
2113 consumer_capabilities,
2114 admission_facts,
2115 } => {
2116 self.handle_route_open(
2117 ctx,
2118 frame,
2119 RouteOpenRequest {
2120 target,
2121 identity,
2122 consumer_identity,
2123 consumer_capabilities,
2124 admission_facts,
2125 },
2126 )
2127 .await
2128 }
2129 ClientControlRequest::RoutePoll {
2130 route_channel,
2131 route_epoch,
2132 kind,
2133 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2134 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2135 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2136 self.handle_supervisor_spawn_snapshot(frame)
2137 }
2138 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2139 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2140 }
2141 ClientControlRequest::SupervisorRestart {
2142 module_id,
2143 drain_timeout_ms,
2144 } => {
2145 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms)
2146 .await
2147 }
2148 ClientControlRequest::SupervisorSwap {
2149 module_id,
2150 ready_timeout_ms,
2151 } => {
2152 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms)
2153 .await
2154 }
2155 ClientControlRequest::SupervisorReload { module_id } => {
2156 self.handle_supervisor_reload(frame, module_id).await
2157 }
2158 ClientControlRequest::SupervisorRescan { preview } => {
2159 self.handle_supervisor_rescan(frame, preview).await
2160 }
2161 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2162 self.handle_supervisor_release_reserved(frame, module_id)
2163 .await
2164 }
2165 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2166 self.handle_supervisor_set_enabled(frame, module_id, enabled)
2167 .await
2168 }
2169 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2170 self.handle_supervisor_health_probe(frame, module_id).await
2171 }
2172 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2173 ClientControlRequest::SupervisorRoutes { module_id } => {
2174 self.handle_supervisor_routes(frame, module_id)
2175 }
2176 ClientControlRequest::SupervisorProvenance { module_id } => {
2177 self.handle_supervisor_provenance(frame, module_id).await
2178 }
2179 ClientControlRequest::SupervisorStderrTail {
2180 module_id,
2181 max_lines,
2182 max_bytes,
2183 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2184 ClientControlRequest::SupervisorTerminals { module_id } => {
2185 self.handle_supervisor_terminals(frame, module_id).await
2186 }
2187 }
2188 }
2189
2190 fn handle_module_control_request(
2191 &self,
2192 connection_id: ConnectionId,
2193 frame: Frame,
2194 request: ModuleControlRequestFromModule,
2195 ) -> Result<Vec<Frame>, RouterError> {
2196 match request {
2197 ModuleControlRequestFromModule::CatalogUpdate {
2198 provides,
2199 capabilities,
2200 ready,
2201 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2202 ModuleControlRequestFromModule::LiveRoots {} => {
2203 let registered = self
2204 .registry
2205 .get_module_by_connection(connection_id)
2206 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2207 let Some(registration) = registered else {
2208 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2209 };
2210 let response = self
2211 .forwarding
2212 .live_roots(®istration.manifest.module_id)
2213 .map_err(RouterError::Forwarding)?;
2214 Ok(vec![control_response_body_frame(
2215 &frame,
2216 &response,
2217 "ModuleControlResponseToModule::LiveRoots",
2218 )?])
2219 }
2220 }
2221 }
2222
2223 fn handle_catalog_update(
2224 &self,
2225 connection_id: ConnectionId,
2226 frame: Frame,
2227 provides: Vec<ProviderRole>,
2228 capabilities: Option<CapabilityDeclarations>,
2229 ready: Option<bool>,
2230 ) -> Result<Vec<Frame>, RouterError> {
2231 self.refresh_capability_requirements();
2232 let Some(registration) = self
2233 .registry
2234 .get_module_by_connection(connection_id)
2235 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2236 else {
2237 return Ok(vec![control_error_frame(
2238 &frame,
2239 "not_registered",
2240 "catalog.update requires an active module registration owned by this connection",
2241 )?]);
2242 };
2243
2244 if let Some(message) =
2245 catalog_update_frozen_field_message(®istration.manifest, &provides)
2246 {
2247 return Ok(vec![control_error_frame(
2248 &frame,
2249 "catalog_update_frozen_field",
2250 message,
2251 )?]);
2252 }
2253
2254 let mut candidate = registration.manifest.clone();
2255 candidate.provides = provides.clone();
2256 candidate.capabilities = capabilities
2257 .clone()
2258 .or_else(|| registration.manifest.capabilities.clone());
2259 if let Err(err) = candidate.validate_capability_grammar() {
2260 return Ok(vec![control_error_frame(
2261 &frame,
2262 "invalid_capability_grammar",
2263 err.to_string(),
2264 )?]);
2265 }
2266
2267 let updated = self
2268 .registry
2269 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
2270 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2271 if updated.is_none() {
2272 return Ok(vec![control_error_frame(
2273 &frame,
2274 "not_registered",
2275 "catalog.update requires an active module registration owned by this connection",
2276 )?]);
2277 }
2278 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
2279 log_duplicate_claim_events(
2280 self.capability_evaluator
2281 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
2282 );
2283 }
2284 if capability_census_trigger(
2285 registration.manifest.capabilities.as_ref(),
2286 updated
2287 .as_ref()
2288 .and_then(|entry| entry.manifest.capabilities.as_ref()),
2289 ) {
2290 self.enforce_capability_denies();
2291 }
2292 self.refresh_capability_requirements();
2293
2294 let response = ModuleControlResponseToModule::CatalogUpdate {};
2295 control_response_body_frame(
2296 &frame,
2297 &response,
2298 "ModuleControlResponseToModule::CatalogUpdate",
2299 )
2300 .map(|frame| vec![frame])
2301 }
2302
2303 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
2304 self.refresh_capability_requirements();
2305 let mut counters = self.counters.snapshot();
2311 if let (Ok((connections_with_routes, max)), Some(obj)) = (
2312 self.forwarding.client_route_concentration(),
2313 counters.as_object_mut(),
2314 ) {
2315 obj.insert(
2316 "client_connections_with_routes".into(),
2317 connections_with_routes.into(),
2318 );
2319 obj.insert("max_routes_on_one_connection".into(), max.into());
2320 }
2321 if let (Some(open_breakers), Some(obj)) = (
2327 self.route_bind_breakers.open_snapshot(),
2328 counters.as_object_mut(),
2329 ) {
2330 obj.insert("route_bind_breakers_open".into(), open_breakers);
2331 }
2332 let response = ClientControlResponse::ServerDescribe {
2333 protocol_ver: PROTOCOL_VERSION,
2334 subc_ops: subc_ops(),
2335 capabilities: self.subc_capabilities.as_ref().to_vec(),
2336 connected_clients: self.connected_clients.count(),
2337 counters: Some(counters),
2338 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
2339 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
2340 capability_requirements: self.capability_requirement_statuses(),
2341 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2342 };
2343 Ok(vec![control_response_body_frame(
2344 &frame,
2345 &response,
2346 "ClientControlResponse::ServerDescribe",
2347 )?])
2348 }
2349
2350 fn handle_catalog_list(
2351 &self,
2352 frame: Frame,
2353 module_id: Option<String>,
2354 ) -> Result<Vec<Frame>, RouterError> {
2355 let (generation, modules) = self.registry.list_modules().map_err(|err| {
2356 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
2357 })?;
2358 let entries = modules
2359 .into_iter()
2360 .filter(|registration| {
2361 module_id
2362 .as_deref()
2363 .map(|wanted| registration.manifest.module_id == wanted)
2364 .unwrap_or(true)
2365 })
2366 .map(|registration| {
2367 let not_ready = self.not_ready_reason(®istration);
2368 let roles = registration.manifest.provides;
2369 CatalogEntry {
2370 module_id: registration.manifest.module_id,
2371 ready: not_ready.is_none(),
2372 not_ready,
2373 module_version: Some(registration.manifest.module_version),
2374 roles,
2375 control_ops: registration.control_ops,
2376 capabilities: registration.manifest.capabilities,
2377 self_signals: registration.manifest.self_signals,
2378 }
2379 })
2380 .collect();
2381 let response = ClientControlResponse::CatalogList {
2382 generation,
2383 modules: entries,
2384 subc_ops: subc_ops(),
2385 };
2386 Ok(vec![control_response_body_frame(
2387 &frame,
2388 &response,
2389 "ClientControlResponse::CatalogList",
2390 )?])
2391 }
2392
2393 fn route_open_principal(
2394 &self,
2395 frame: &Frame,
2396 consumer_identity: Option<ConsumerIdentity>,
2397 ) -> Result<Result<Principal, Frame>, RouterError> {
2398 let Some(consumer_identity) = consumer_identity else {
2399 return Ok(Ok(Principal::Direct));
2400 };
2401
2402 if self.supervisor.spawned_consumer_authorized(
2403 &consumer_identity.module_id,
2404 &consumer_identity.launch_nonce,
2405 ) {
2406 return Ok(Ok(Principal::Reserved {
2407 module_id: consumer_identity.module_id,
2408 }));
2409 }
2410
2411 Ok(Err(control_error_frame(
2412 frame,
2413 "bad_consumer_identity",
2414 format!(
2415 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
2416 consumer_identity.module_id
2417 ),
2418 )?))
2419 }
2420
2421 fn route_open_refusal_frame(
2428 &self,
2429 ctx: &RouteCtx,
2430 frame: &Frame,
2431 module_id: &str,
2432 reason: &'static str,
2433 code: &'static str,
2434 message: impl Into<String>,
2435 ) -> Result<Frame, RouterError> {
2436 self.observe_route_open_refusal(ctx, module_id, reason, code);
2437 control_error_frame(frame, code, message.into())
2438 }
2439
2440 fn route_open_breaker_refusal_frame(
2458 &self,
2459 ctx: &RouteCtx,
2460 frame: &Frame,
2461 module_id: &str,
2462 consecutive_timeouts: u32,
2463 retry_in: Duration,
2464 probe_in_flight: bool,
2465 ) -> Result<Frame, RouterError> {
2466 self.counters
2467 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
2468 debug!(
2469 target: "control",
2470 code = "module_timeout",
2471 module_id = ?module_id,
2472 connection_id = ctx.connection_id.get(),
2473 consecutive_timeouts,
2474 retry_in_ms = retry_in.as_millis() as u64,
2475 probe_in_flight,
2476 "route.open refused by open bind-relay breaker"
2477 );
2478 let detail = if probe_in_flight {
2479 "one probe bind is already in flight; retry once it settles".to_string()
2480 } else {
2481 format!("not relaying for another {retry_in:?}")
2482 };
2483 control_error_frame(
2484 frame,
2485 "module_timeout",
2486 format!(
2487 "module_id '{module_id}' failed {consecutive_timeouts} consecutive route.bind \
2488 relays; {detail}"
2489 ),
2490 )
2491 }
2492
2493 fn observe_route_open_refusal(
2506 &self,
2507 ctx: &RouteCtx,
2508 module_id: &str,
2509 reason: &'static str,
2510 code: &'static str,
2511 ) {
2512 self.counters.increment_route_open_refused(code);
2513 info!(
2514 target: "control",
2515 code,
2516 reason,
2517 module_id = ?module_id,
2518 connection_id = ctx.connection_id.get(),
2519 "route.open refused"
2520 );
2521 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
2522 self.route_outages.record_not_serving(module_id, reason);
2523 }
2524 }
2525
2526 fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
2571 self.route_outages.record_accepted(module_id);
2572 self.counters.increment_route_open_accepted(principal);
2573 info!(
2574 target: "control",
2575 principal,
2576 module_id,
2577 connection_id = ctx.connection_id.get(),
2578 "route.open accepted"
2579 );
2580 }
2581
2582 fn supervised_absent_route_open_refusal_frame(
2583 &self,
2584 ctx: &RouteCtx,
2585 frame: &Frame,
2586 module_id: &str,
2587 code: &'static str,
2588 status: &crate::supervise::ModuleStatus,
2589 ) -> Result<Frame, RouterError> {
2590 self.counters.increment_route_open_refused(code);
2591 info!(
2592 target: "control",
2593 code,
2594 reason = "supervised_not_registered",
2595 module_id = ?module_id,
2596 connection_id = ctx.connection_id.get(),
2597 state = %status.state,
2598 enabled = status.enabled,
2599 live = status.live,
2600 "route.open refused"
2601 );
2602 self.route_outages
2606 .record_not_serving(module_id, "supervised_not_registered");
2607 control_error_frame(
2608 frame,
2609 code,
2610 format!(
2611 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
2612 status.state, status.enabled, status.live
2613 ),
2614 )
2615 }
2616
2617 async fn handle_route_open(
2618 &self,
2619 ctx: &RouteCtx,
2620 frame: Frame,
2621 request: RouteOpenRequest,
2622 ) -> Result<Vec<Frame>, RouterError> {
2623 let RouteOpenRequest {
2624 target,
2625 mut identity,
2626 consumer_identity,
2627 consumer_capabilities,
2628 admission_facts,
2629 } = request;
2630 let target_module_id = target_module_id(&target).to_string();
2631 debug!(
2632 connection_id = ctx.connection_id.get(),
2633 corr = frame.header.corr,
2634 module_id = %target_module_id,
2635 "handling route.open"
2636 );
2637
2638 let Some(registration) = self
2657 .registry
2658 .get_module(&target_module_id)
2659 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2660 else {
2661 if let Some((status, warming)) =
2662 self.supervisor_status(&target_module_id, frame.header.corr)?
2663 {
2664 if status.protocol == ModuleProtocol::None {
2671 return Ok(vec![self.route_open_refusal_frame(
2672 ctx,
2673 &frame,
2674 &target_module_id,
2675 "protocol_none",
2676 error_codes::MODULE_NO_PROTOCOL,
2677 format!(
2678 "module_id '{target_module_id}' is declared protocol: none; \
2679 it speaks no subc wire and serves no routes"
2680 ),
2681 )?]);
2682 }
2683 let code = if warming {
2684 "module_warming"
2685 } else {
2686 "target_unavailable"
2687 };
2688 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
2689 ctx,
2690 &frame,
2691 &target_module_id,
2692 code,
2693 &status,
2694 )?]);
2695 }
2696 if let Some(removed_ago_ms) =
2697 self.supervisor.removal_tombstone_age_ms(&target_module_id)
2698 {
2699 return Ok(vec![self.route_open_refusal_frame(
2700 ctx,
2701 &frame,
2702 &target_module_id,
2703 "removed",
2704 error_codes::MODULE_REMOVED,
2705 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
2706 )?]);
2707 }
2708 return Ok(vec![self.route_open_refusal_frame(
2709 ctx,
2710 &frame,
2711 &target_module_id,
2712 "not_registered",
2713 error_codes::UNKNOWN_MODULE,
2714 format!("module_id '{target_module_id}' is not registered"),
2715 )?]);
2716 };
2717
2718 if !registration.ready {
2722 self.counters
2723 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
2724 info!(
2725 target: "control",
2726 code = error_codes::MODULE_WARMING,
2727 module_id = ?target_module_id,
2728 connection_id = ctx.connection_id.get(),
2729 reason = "declared_not_ready",
2730 "route.open refused"
2731 );
2732 self.route_outages
2735 .record_not_serving(&target_module_id, "declared_not_ready");
2736 return Ok(vec![control_error_body_frame(
2737 &frame,
2738 ErrorBody {
2739 code: error_codes::MODULE_WARMING.to_string(),
2740 message: format!(
2741 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
2742 ),
2743 detail: Some(serde_json::json!({
2744 "reason": "declared_not_ready"
2745 })),
2746 },
2747 )?]);
2748 }
2749
2750 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
2769 self.counters
2770 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
2771 info!(
2772 target: "control",
2773 code = error_codes::MODULE_WARMING,
2774 module_id = ?target_module_id,
2775 connection_id = ctx.connection_id.get(),
2776 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
2777 capability = %capability,
2778 "route.open refused"
2779 );
2780 return Ok(vec![control_error_body_frame(
2781 &frame,
2782 ErrorBody {
2783 code: error_codes::MODULE_WARMING.to_string(),
2784 message: format!(
2785 "module_id '{target_module_id}' requires capability '{capability}', \
2786 which no registered module provides; retry"
2787 ),
2788 detail: Some(serde_json::json!({
2789 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
2790 "capability": capability,
2791 })),
2792 },
2793 )?]);
2794 }
2795
2796 if !target_has_required_role(&target, ®istration.manifest.provides) {
2797 return Ok(vec![self.route_open_refusal_frame(
2798 ctx,
2799 &frame,
2800 &target_module_id,
2801 "role_not_provided",
2802 "target_unavailable",
2803 format!("module_id '{target_module_id}' does not provide the requested target"),
2804 )?]);
2805 }
2806
2807 if registration.state != ChannelState::Active {
2808 return Ok(vec![self.route_open_refusal_frame(
2809 ctx,
2810 &frame,
2811 &target_module_id,
2812 "registration_not_active",
2813 "target_unavailable",
2814 format!("module_id '{target_module_id}' is not active"),
2815 )?]);
2816 }
2817
2818 if self
2819 .forwarding
2820 .module_is_draining(&target_module_id)
2821 .map_err(RouterError::Forwarding)?
2822 {
2823 return Ok(vec![self.route_open_refusal_frame(
2824 ctx,
2825 &frame,
2826 &target_module_id,
2827 "reloading",
2828 "module_reloading",
2829 format!("module_id '{target_module_id}' is reloading"),
2830 )?]);
2831 }
2832
2833 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
2834 process_liveness.process_live(&target_module_id) == Some(false)
2835 }) {
2836 if process_liveness.process_replacing(&target_module_id) {
2843 return Ok(vec![self.route_open_refusal_frame(
2844 ctx,
2845 &frame,
2846 &target_module_id,
2847 "reloading",
2848 "module_reloading",
2849 format!("module_id '{target_module_id}' is reloading"),
2850 )?]);
2851 }
2852 return Ok(vec![self.route_open_refusal_frame(
2853 ctx,
2854 &frame,
2855 &target_module_id,
2856 "supervisor_not_live",
2857 "target_unavailable",
2858 format!("module_id '{target_module_id}' is not live"),
2859 )?]);
2860 }
2861
2862 if !self
2863 .forwarding
2864 .has_live_module_connection(&target_module_id)
2865 .map_err(RouterError::Forwarding)?
2866 {
2867 return Ok(vec![self.route_open_refusal_frame(
2868 ctx,
2869 &frame,
2870 &target_module_id,
2871 "no_forwarding_connection",
2872 "target_unavailable",
2873 format!("module_id '{target_module_id}' has no live forwarding connection"),
2874 )?]);
2875 }
2876
2877 if let Some(error) =
2878 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
2879 {
2880 self.observe_route_open_refusal(
2881 ctx,
2882 &target_module_id,
2883 "op_not_allowed",
2884 "op_not_allowed",
2885 );
2886 return Ok(vec![error]);
2887 }
2888
2889 let principal = match self.route_open_principal(&frame, consumer_identity)? {
2890 Ok(principal) => principal,
2891 Err(error) => {
2892 self.observe_route_open_refusal(
2893 ctx,
2894 &target_module_id,
2895 "bad_consumer_identity",
2896 "bad_consumer_identity",
2897 );
2898 return Ok(vec![error]);
2899 }
2900 };
2901
2902 if let Principal::Reserved {
2906 module_id: opening_module_id,
2907 } = &principal
2908 {
2909 if let Some(opening_registration) = self
2910 .registry
2911 .get_module(opening_module_id)
2912 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2913 {
2914 if let Some(capability) =
2915 denied_capability(&opening_registration.manifest, ®istration.manifest)
2916 {
2917 warn!(
2918 opening_module_id,
2919 target_module_id,
2920 capability,
2921 "refusing route.open because an attested capability deny edge matches"
2922 );
2923 return Ok(vec![self.route_open_refusal_frame(
2924 ctx,
2925 &frame,
2926 &target_module_id,
2927 "capability_deny_edge",
2928 "capability_forbidden",
2929 format!(
2930 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
2931 ),
2932 )?]);
2933 }
2934 }
2935 }
2936
2937 if admission_facts.is_some() {
2938 let carrier_matches = matches!(
2939 &principal,
2940 Principal::Reserved { module_id }
2941 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
2942 );
2943 if !carrier_matches {
2944 return Ok(vec![self.route_open_refusal_frame(
2945 ctx,
2946 &frame,
2947 &target_module_id,
2948 "admission_facts_carrier_not_permitted",
2949 "admission_facts_not_permitted",
2950 "admission facts may only be carried by the configured reserved module",
2951 )?]);
2952 }
2953
2954 let target_allowed = self
2955 .admission_facts_targets
2956 .as_ref()
2957 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
2958 if !target_allowed {
2959 return Ok(vec![self.route_open_refusal_frame(
2960 ctx,
2961 &frame,
2962 &target_module_id,
2963 "admission_facts_target_not_listed",
2964 "admission_facts_target_not_allowed",
2965 format!(
2966 "admission facts are not permitted for target module_id '{target_module_id}'"
2967 ),
2968 )?]);
2969 }
2970
2971 }
2975
2976 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
2998 Ok(project_root) => project_root,
2999 Err(err) => {
3000 return Ok(vec![control_error_frame(
3001 &frame,
3002 "invalid_project_root",
3003 err.to_string(),
3004 )?])
3005 }
3006 };
3007 identity.project_root = project_root.as_path().to_path_buf();
3008
3009 let _concurrency_guard = match self
3020 .route_bind_concurrency
3021 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
3022 {
3023 Ok(guard) => guard,
3024 Err(in_flight) => {
3025 return Ok(vec![self.route_open_target_capacity_refusal(
3026 ctx,
3027 &frame,
3028 &target_module_id,
3029 in_flight,
3030 )?]);
3031 }
3032 };
3033
3034 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
3037 RouteBindAdmission::Admitted { guard, probe } => {
3038 if probe {
3039 info!(
3040 module_id = %target_module_id,
3041 connection_id = ctx.connection_id.get(),
3042 "route.bind breaker half-open: admitting one probe"
3043 );
3044 }
3045 guard
3046 }
3047 RouteBindAdmission::Refused {
3048 consecutive_timeouts,
3049 retry_in,
3050 probe_in_flight,
3051 } => {
3052 return Ok(vec![self.route_open_breaker_refusal_frame(
3053 ctx,
3054 &frame,
3055 &target_module_id,
3056 consecutive_timeouts,
3057 retry_in,
3058 probe_in_flight,
3059 )?]);
3060 }
3061 };
3062
3063 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
3068 let relay_deadline = Instant::now() + route_bind_relay_timeout;
3069 let pending = match self
3070 .forwarding
3071 .begin_route_bind_relay_for(
3072 ctx.connection_id,
3073 ctx.egress.clone(),
3074 response_version(&frame),
3075 frame.header.corr,
3076 &target_module_id,
3077 principal.clone(),
3078 Some(project_root),
3079 relay_deadline,
3080 )
3081 .await
3082 {
3083 Ok(pending) => pending,
3084 Err(err) => {
3085 return Ok(vec![self.route_open_refusal_frame(
3086 ctx,
3087 &frame,
3088 &target_module_id,
3089 "relay_reservation_failed",
3090 forwarding_error_code(&err),
3091 err.to_string(),
3092 )?])
3093 }
3094 };
3095 let crate::forwarding::PendingRouteBindRelay {
3096 endpoint,
3097 module_sink,
3098 negotiated_ver,
3099 client_channel,
3100 client_epoch,
3101 module_channel,
3102 module_epoch,
3103 corr: relay_corr,
3104 receiver,
3105 } = pending;
3106 let mut reservation =
3107 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
3108
3109 debug!(
3110 connection_id = ctx.connection_id.get(),
3111 client_channel,
3112 client_epoch,
3113 module_channel,
3114 module_epoch,
3115 "reserved route handle pair"
3116 );
3117 let principal_label = match &principal {
3120 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
3121 Principal::Direct => "direct".to_string(),
3122 other => format!("{other:?}"),
3123 };
3124 let relay = ModuleControlRequest::RouteBind {
3125 route_channel: module_channel,
3126 epoch: module_epoch,
3127 target,
3128 identity,
3129 principal: Some(principal),
3130 consumer_capabilities,
3131 admission_facts,
3132 };
3133 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
3134 RouterError::backend(
3135 0,
3136 frame.header.corr,
3137 format!("failed to encode route.bind request: {err}"),
3138 )
3139 })?;
3140 let relay_frame = Frame::build_with_version(
3141 negotiated_ver,
3142 FrameType::Request,
3143 control_flags(),
3144 0,
3145 0,
3146 relay_corr,
3147 relay_body,
3148 )
3149 .map_err(RouterError::FrameBuild)?;
3150
3151 if let Err(err) = module_sink.send(relay_frame).await {
3152 reservation.release_and_disarm();
3153 return Ok(vec![self.route_open_refusal_frame(
3154 ctx,
3155 &frame,
3156 &target_module_id,
3157 "relay_send_failed",
3158 "target_unavailable",
3159 err.to_string(),
3160 )?]);
3161 }
3162
3163 if !self
3164 .forwarding
3165 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
3166 .map_err(RouterError::Forwarding)?
3167 {
3168 self.send_abandoned_route_bind_goodbye(
3169 &module_sink,
3170 negotiated_ver,
3171 module_channel,
3172 module_epoch,
3173 );
3174 }
3175
3176 match timeout_at(relay_deadline, receiver).await {
3177 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
3178 reservation.disarm();
3179 if breaker.record_accepted() {
3180 info!(
3181 module_id = %target_module_id,
3182 "route.bind breaker closed: the probe was accepted"
3183 );
3184 }
3185 self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
3186 Ok(Vec::new())
3187 }
3188 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
3189 reservation.release_and_disarm();
3190 breaker.record_inconclusive();
3194 self.counters
3195 .increment_route_open_refused("module_rejected");
3196 info!(
3197 target: "control",
3198 code = "module_rejected",
3199 module_code = ?body.code,
3200 module_id = ?target_module_id,
3201 connection_id = ctx.connection_id.get(),
3202 "route.open refused"
3203 );
3204 Ok(vec![control_error_body_frame(&frame, body)?])
3205 }
3206 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
3207 reservation.release_and_disarm();
3208 breaker.record_inconclusive();
3209 tracing::warn!(
3216 module_id = %target_module_id,
3217 "route.bind relay abandoned: {message}"
3218 );
3219 Ok(vec![self.route_open_refusal_frame(
3220 ctx,
3221 &frame,
3222 &target_module_id,
3223 "relay_abandoned",
3224 "target_unavailable",
3225 message,
3226 )?])
3227 }
3228 Ok(Err(_)) => {
3229 reservation.release_and_disarm();
3230 breaker.record_inconclusive();
3231 Ok(vec![self.route_open_refusal_frame(
3232 ctx,
3233 &frame,
3234 &target_module_id,
3235 "relay_waiter_canceled",
3236 "target_unavailable",
3237 "route.bind relay waiter was canceled before the module responded",
3238 )?])
3239 }
3240 Err(_) => {
3241 reservation.release_and_disarm();
3242 if let Some(opened) = breaker.record_timeout(
3246 self.route_bind_breaker_threshold,
3247 self.route_bind_breaker_cooldown,
3248 ) {
3249 warn!(
3250 module_id = %target_module_id,
3251 consecutive_timeouts = opened.consecutive_timeouts,
3252 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
3253 reopened_after_probe = opened.reopened_after_probe,
3254 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
3255 );
3256 }
3257 tracing::warn!(
3265 module_id = %target_module_id,
3266 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
3267 "route.bind relay timed out: module did not ack within budget"
3268 );
3269 Ok(vec![self.route_open_refusal_frame(
3270 ctx,
3271 &frame,
3272 &target_module_id,
3273 "relay_timed_out",
3274 "module_timeout",
3275 format!(
3276 "module_id '{target_module_id}' did not answer route.bind within {:?}",
3277 route_bind_relay_timeout
3278 ),
3279 )?])
3280 }
3281 }
3282 }
3283
3284 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3285 let response = ClientControlResponse::SupervisorSpawnSnapshot {
3286 snapshot: self.supervisor.spawn_snapshot(),
3287 };
3288 Ok(vec![control_response_body_frame(
3289 &frame,
3290 &response,
3291 "ClientControlResponse::SupervisorSpawnSnapshot",
3292 )?])
3293 }
3294
3295 fn handle_supervisor_spawn_subscribe(
3296 &self,
3297 ctx: &RouteCtx,
3298 frame: Frame,
3299 since: Option<SpawnCursor>,
3300 ) -> Result<Vec<Frame>, RouterError> {
3301 match self.supervisor.subscribe_spawns(
3302 ctx.connection_id,
3303 frame.header.corr,
3304 response_version(&frame),
3305 since,
3306 ctx.egress.clone(),
3307 ) {
3308 Ok(()) => Ok(Vec::new()),
3309 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
3310 Ok(vec![control_error_body_frame(
3311 &frame,
3312 ErrorBody {
3313 code: "spawn_cursor_incarnation_mismatch".to_string(),
3314 message: "spawn cursor belongs to a different daemon incarnation"
3315 .to_string(),
3316 detail: Some(serde_json::json!({
3317 "current_daemon_incarnation": current
3318 })),
3319 },
3320 )?])
3321 }
3322 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
3323 &frame,
3324 ErrorBody {
3325 code: "spawn_cursor_too_old".to_string(),
3326 message: "spawn cursor predates the retained event ring".to_string(),
3327 detail: Some(serde_json::json!({
3328 "oldest_retained_cursor": oldest
3329 })),
3330 },
3331 )?]),
3332 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
3333 0,
3334 frame.header.corr,
3335 format!("failed to open supervisor spawn subscription: {error}"),
3336 )),
3337 }
3338 }
3339
3340 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3341 let generation = self
3342 .registry
3343 .generation()
3344 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3345 let mut modules = Vec::new();
3346 for module in self.supervisor.list() {
3347 let status = module.status_for_control("list").map_err(|err| {
3348 RouterError::backend(
3349 0,
3350 frame.header.corr,
3351 format!("failed to read supervisor status: {err}"),
3352 )
3353 })?;
3354 let (configured, _) = module.configuration().map_err(|err| {
3355 RouterError::backend(
3356 0,
3357 frame.header.corr,
3358 format!("failed to read module configuration: {err}"),
3359 )
3360 })?;
3361 let image = module.running_image_agreement().await;
3363 let resources = Some(module.child_resource_usage());
3366 let pending_reload = Some(reload_verdict(
3367 &configured.program,
3368 status.spawned_from.as_deref(),
3369 image,
3370 ));
3371 modules.push(SupervisorEntry {
3372 module_id: status.module_id,
3373 state: status.state.to_string(),
3374 enabled: status.enabled,
3375 live: status.live,
3376 protocol: status.protocol,
3377 health: status.health.status,
3378 pending_reload,
3379 last_probe_ms: status.health.last_probe_ms,
3380 last_exit_code: status.last_exit.as_ref().and_then(|e| e.code),
3381 last_exit_signal: status.last_exit.as_ref().and_then(|e| e.signal),
3382 last_exit_ms: status.last_exit.as_ref().map(|e| e.at_ms),
3383 last_exit_kind: status.last_exit.as_ref().map(|e| e.kind.into()),
3384 restart_count: Some(status.restart_count),
3385 max_restarts: Some(status.max_restarts),
3386 lifetime_restarts: Some(status.lifetime_restarts),
3387 spawn_generation: Some(status.spawn_generation),
3388 restart_window_secs: Some(status.restart_window.as_secs()),
3389 drain_timeout_ms: Some(status.drain_timeout.as_millis() as u64),
3390 restart_backoff_ms: Some(status.restart_backoff.as_millis() as u64),
3391 restart_max_backoff_ms: Some(status.restart_max_backoff.as_millis() as u64),
3392 resources,
3393 });
3394 }
3395 let response = ClientControlResponse::SupervisorList {
3396 generation,
3397 modules,
3398 };
3399 Ok(vec![control_response_body_frame(
3400 &frame,
3401 &response,
3402 "ClientControlResponse::SupervisorList",
3403 )?])
3404 }
3405
3406 fn handle_supervisor_stderr_tail(
3407 &self,
3408 frame: Frame,
3409 module_id: String,
3410 max_lines: Option<u32>,
3411 max_bytes: Option<u32>,
3412 ) -> Result<Vec<Frame>, RouterError> {
3413 let Some(module) = self.supervisor.get(&module_id) else {
3414 return Ok(vec![control_error_frame(
3415 &frame,
3416 "unknown_module",
3417 format!("module_id '{module_id}' is not supervised"),
3418 )?]);
3419 };
3420
3421 let snapshot = module.stderr_tail(
3422 max_lines.map(|value| value as usize),
3423 max_bytes.map(|value| value as usize),
3424 );
3425
3426 let response = ClientControlResponse::SupervisorStderrTail {
3427 module_id,
3428 tail: StderrTail {
3429 capture: match snapshot.capture {
3430 CaptureState::Captured => StderrCaptureState::Captured,
3431 CaptureState::Incomplete { reason } => {
3432 StderrCaptureState::Incomplete { reason }
3433 }
3434 CaptureState::NotCaptured { reason } => {
3435 StderrCaptureState::NotCaptured { reason }
3436 }
3437 },
3438 entries: snapshot
3439 .entries
3440 .into_iter()
3441 .map(|entry| match entry {
3442 TailEntry::Line { text, truncated } => {
3443 StderrTailEntry::Line { text, truncated }
3444 }
3445 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
3446 })
3447 .collect(),
3448 dropped_lines: snapshot.dropped_lines,
3449 },
3450 };
3451 Ok(vec![control_response_body_frame(
3452 &frame,
3453 &response,
3454 "ClientControlResponse::SupervisorStderrTail",
3455 )?])
3456 }
3457
3458 async fn handle_supervisor_terminals(
3459 &self,
3460 frame: Frame,
3461 module_id: String,
3462 ) -> Result<Vec<Frame>, RouterError> {
3463 let Some(module) = self.supervisor.get(&module_id) else {
3464 return Ok(vec![control_error_frame(
3465 &frame,
3466 "unknown_module",
3467 format!("module_id '{module_id}' is not supervised"),
3468 )?]);
3469 };
3470
3471 let terminals = module
3474 .read_durable_terminal_history()
3475 .await
3476 .map_err(|error| {
3477 RouterError::backend(
3478 0,
3479 frame.header.corr,
3480 format!("failed to read terminal history: {error}"),
3481 )
3482 })?;
3483 let response = ClientControlResponse::SupervisorTerminals {
3484 module_id,
3485 terminals,
3486 };
3487 Ok(vec![control_response_body_frame(
3488 &frame,
3489 &response,
3490 "ClientControlResponse::SupervisorTerminals",
3491 )?])
3492 }
3493
3494 fn handle_supervisor_routes(
3495 &self,
3496 frame: Frame,
3497 module_id: Option<String>,
3498 ) -> Result<Vec<Frame>, RouterError> {
3499 let modules = self
3500 .forwarding
3501 .route_census(module_id.as_deref())
3502 .map_err(RouterError::Forwarding)?
3503 .into_iter()
3504 .map(|(module_id, routes)| SupervisorRouteModule {
3505 module_id,
3506 routes: routes
3507 .into_iter()
3508 .map(|route| SupervisorRoute {
3509 consumer: match route.principal {
3510 Principal::Reserved { module_id } => {
3511 SupervisorRouteConsumer::Reserved { module_id }
3512 }
3513 Principal::Direct | Principal::Unverified => {
3514 SupervisorRouteConsumer::Direct {
3515 connection_id: route.goodbye_target.connection_id.get(),
3516 }
3517 }
3518 },
3519 age_ms: Instant::now()
3520 .saturating_duration_since(route.bound_at)
3521 .as_millis()
3522 .try_into()
3523 .unwrap_or(u64::MAX),
3524 draining: route.draining,
3525 drain_reason: route.drain_reason,
3526 })
3527 .collect(),
3528 })
3529 .collect();
3530 let response = ClientControlResponse::SupervisorRoutes { modules };
3531 Ok(vec![control_response_body_frame(
3532 &frame,
3533 &response,
3534 "ClientControlResponse::SupervisorRoutes",
3535 )?])
3536 }
3537
3538 async fn handle_supervisor_provenance(
3539 &self,
3540 frame: Frame,
3541 module_id: Option<String>,
3542 ) -> Result<Vec<Frame>, RouterError> {
3543 let mut selected = if let Some(module_id) = module_id {
3544 let Some(module) = self.supervisor.get(&module_id) else {
3545 return Ok(vec![control_error_frame(
3546 &frame,
3547 "unknown_module",
3548 format!("module_id '{module_id}' is not supervised"),
3549 )?]);
3550 };
3551 vec![module]
3552 } else {
3553 self.supervisor.list()
3554 };
3555
3556 let mut modules = Vec::with_capacity(selected.len());
3557 for module in selected.drain(..) {
3558 let status = module.status().map_err(|err| {
3559 RouterError::backend(
3560 0,
3561 frame.header.corr,
3562 format!("failed to read supervisor status: {err}"),
3563 )
3564 })?;
3565 let module_declared = self
3566 .registry
3567 .get_module(&status.module_id)
3568 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3569 .and_then(|registration| registration.manifest.provenance)
3570 .map(|build| ModuleDeclaredProvenance::Reported { build })
3571 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
3572 #[cfg(test)]
3573 let running_image = match &self.provenance_probe_override {
3574 Some(result) => result.clone(),
3575 None => module.running_image_agreement().await,
3576 };
3577 #[cfg(not(test))]
3578 let running_image = module.running_image_agreement().await;
3579 modules.push(SupervisorModuleProvenance {
3580 module_id: status.module_id,
3581 module_declared,
3582 daemon_observed: SupervisorObservedProcess {
3583 pid: status.pid,
3584 spawned_at_ms: status.spawned_at_ms,
3585 spawned_from: status.spawned_from,
3586 running_image,
3587 },
3588 });
3589 }
3590 let daemon = SupervisorDaemonProvenance {
3591 daemon_build: self.daemon_provenance.build.clone(),
3592 daemon_observed: DaemonObservedProcess {
3593 pid: self.daemon_provenance.pid,
3594 started_at_ms: self
3595 .daemon_provenance
3596 .start_clock
3597 .map(|clock| clock.started_at_ms())
3598 .or(self.daemon_provenance.started_at_ms),
3599 running_image: self
3600 .daemon_provenance
3601 .probe
3602 .observe(
3603 self.daemon_provenance.pid,
3604 self.daemon_provenance.executable_path.as_deref(),
3605 self.daemon_provenance.executable_identity,
3606 self.daemon_provenance.process_start_time,
3607 )
3608 .await,
3609 },
3610 };
3611 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
3612 Ok(vec![control_response_body_frame(
3613 &frame,
3614 &response,
3615 "ClientControlResponse::SupervisorProvenance",
3616 )?])
3617 }
3618
3619 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3620 self.refresh_capability_requirements();
3621 let generation = self
3622 .registry
3623 .generation()
3624 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3625 let modules = self
3626 .supervisor
3627 .list()
3628 .into_iter()
3629 .map(|module| {
3630 let status = module.status_for_control("health").map_err(|err| {
3631 RouterError::backend(
3632 0,
3633 frame.header.corr,
3634 format!("failed to read supervisor health: {err}"),
3635 )
3636 })?;
3637 let module_id = status.module_id;
3638 let capability_detail = self
3639 .capability_evaluator
3640 .required_problem_detail(&module_id);
3641 Ok(SupervisorHealthEntry {
3642 module_id,
3643 status: status.health.status,
3644 detail: append_capability_problem_detail(
3645 status.health.detail,
3646 capability_detail,
3647 ),
3648 metrics: status.health.metrics,
3649 consecutive_failures: status.health.consecutive_failures,
3650 late_answer_count: status.health.late_answer_count,
3651 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
3652 last_action: status.health.last_action,
3653 last_action_ms: status.health.last_action_ms,
3654 last_probe_ms: status.health.last_probe_ms,
3655 })
3656 })
3657 .collect::<Result<Vec<_>, RouterError>>()?;
3658 let response = ClientControlResponse::SupervisorHealth {
3659 generation,
3660 modules,
3661 };
3662 Ok(vec![control_response_body_frame(
3663 &frame,
3664 &response,
3665 "ClientControlResponse::SupervisorHealth",
3666 )?])
3667 }
3668
3669 async fn handle_supervisor_restart(
3670 &self,
3671 frame: Frame,
3672 module_id: String,
3673 drain_timeout_ms: Option<u64>,
3674 ) -> Result<Vec<Frame>, RouterError> {
3675 let operation_lock = self.supervisor.operation_lock();
3676 let _operation_guard = operation_lock.lock().await;
3677 let Some(module) = self.supervisor.get(&module_id) else {
3678 return Ok(vec![control_error_frame(
3679 &frame,
3680 "unknown_module",
3681 format!("module_id '{module_id}' is not supervised"),
3682 )?]);
3683 };
3684
3685 self.route_outages.mark_operator_action(&module_id);
3686 if let Err(err) = module.restart(drain_timeout_ms).await {
3687 self.route_outages
3688 .operator_action_ended_unrefused(&module_id);
3689 let (code, message) = match err {
3690 crate::supervise::SuperviseError::Disabled { .. } => {
3691 ("module_disabled", err.to_string())
3692 }
3693 crate::supervise::SuperviseError::SwapInProgress { .. } => {
3694 ("swap_in_progress", err.to_string())
3695 }
3696 _ => (
3697 "target_unavailable",
3698 format!("failed to restart module_id '{module_id}': {err}"),
3699 ),
3700 };
3701 return Ok(vec![control_error_frame(&frame, code, message)?]);
3702 }
3703
3704 let response = ClientControlResponse::SupervisorAck {
3705 module_id,
3706 applied: true,
3707 };
3708 Ok(vec![control_response_body_frame(
3709 &frame,
3710 &response,
3711 "ClientControlResponse::SupervisorAck",
3712 )?])
3713 }
3714
3715 async fn handle_supervisor_swap(
3719 &self,
3720 frame: Frame,
3721 module_id: String,
3722 ready_timeout_ms: Option<u64>,
3723 ) -> Result<Vec<Frame>, RouterError> {
3724 let module = {
3731 let operation_lock = self.supervisor.operation_lock();
3732 let _operation_guard = operation_lock.lock().await;
3733 self.supervisor.get(&module_id)
3734 };
3735 let Some(module) = module else {
3736 return Ok(vec![control_error_frame(
3737 &frame,
3738 "unknown_module",
3739 format!("module_id '{module_id}' is not supervised"),
3740 )?]);
3741 };
3742
3743 self.route_outages.mark_operator_action(&module_id);
3744 if let Err(err) = module
3745 .swap(ready_timeout_ms.map(Duration::from_millis))
3746 .await
3747 {
3748 self.route_outages
3749 .operator_action_ended_unrefused(&module_id);
3750 use crate::supervise::SuperviseError;
3751 let message = err.to_string();
3752 let error = match err {
3753 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
3754 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
3755 code: "swap_refused".to_string(),
3756 message,
3757 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
3758 },
3759 SuperviseError::SwapFailed {
3760 arm,
3761 candidate_exit,
3762 ..
3763 } => ErrorBody {
3764 code: "swap_failed".to_string(),
3765 message,
3766 detail: Some(serde_json::json!({
3767 "arm": arm.as_str(),
3768 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
3769 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
3770 })),
3771 },
3772 _ => ErrorBody::new(
3773 "target_unavailable",
3774 format!("failed to swap module_id '{module_id}': {message}"),
3775 ),
3776 };
3777 return Ok(vec![control_error_body_frame(&frame, error)?]);
3778 }
3779 self.route_outages
3783 .operator_action_ended_unrefused(&module_id);
3784
3785 let response = ClientControlResponse::SupervisorAck {
3786 module_id,
3787 applied: true,
3788 };
3789 Ok(vec![control_response_body_frame(
3790 &frame,
3791 &response,
3792 "ClientControlResponse::SupervisorAck",
3793 )?])
3794 }
3795
3796 async fn handle_supervisor_reload(
3797 &self,
3798 frame: Frame,
3799 module_id: String,
3800 ) -> Result<Vec<Frame>, RouterError> {
3801 let operation_lock = self.supervisor.operation_lock();
3802 let _operation_guard = operation_lock.lock().await;
3803 let Some(module) = self.supervisor.get(&module_id) else {
3804 return Ok(vec![control_error_frame(
3805 &frame,
3806 "unknown_module",
3807 format!("module_id '{module_id}' is not supervised"),
3808 )?]);
3809 };
3810
3811 self.route_outages.mark_operator_action(&module_id);
3812 if let Err(err) = module.reload().await {
3813 self.route_outages
3814 .operator_action_ended_unrefused(&module_id);
3815 let (code, message) = match err {
3816 crate::supervise::SuperviseError::Disabled { .. } => {
3817 ("module_disabled", err.to_string())
3818 }
3819 crate::supervise::SuperviseError::SwapInProgress { .. } => {
3820 ("swap_in_progress", err.to_string())
3821 }
3822 _ => (
3823 "reload_failed",
3824 format!("failed to reload module_id '{module_id}': {err}"),
3825 ),
3826 };
3827 return Ok(vec![control_error_frame(&frame, code, message)?]);
3828 }
3829
3830 let response = ClientControlResponse::SupervisorAck {
3831 module_id,
3832 applied: true,
3833 };
3834 Ok(vec![control_response_body_frame(
3835 &frame,
3836 &response,
3837 "ClientControlResponse::SupervisorAck",
3838 )?])
3839 }
3840
3841 async fn handle_supervisor_rescan(
3842 &self,
3843 frame: Frame,
3844 preview: bool,
3845 ) -> Result<Vec<Frame>, RouterError> {
3846 let Some(context) = self.rescan.clone() else {
3847 return Ok(vec![control_error_frame(
3848 &frame,
3849 "rescan_unavailable",
3850 "the daemon was not started with a reloadable config path".to_string(),
3851 )?]);
3852 };
3853
3854 let operation_lock = self.supervisor.operation_lock();
3855 let _operation_guard = operation_lock.lock().await;
3856 let loaded = match crate::daemon_config::load(&context.config_path) {
3857 Ok(config) => config,
3858 Err(err) => {
3859 return Ok(vec![control_error_frame(
3860 &frame,
3861 "invalid_daemon_config",
3862 format!("supervisor rescan rejected daemon config: {err}"),
3863 )?])
3864 }
3865 };
3866 let Some(config) = loaded else {
3874 return Ok(vec![control_error_frame(
3875 &frame,
3876 "invalid_daemon_config",
3877 format!(
3878 "daemon config not found at {}; refusing to rescan (an absent config would \
3879 retire every supervised module)",
3880 context.config_path.display()
3881 ),
3882 )?]);
3883 };
3884 let (
3885 configured_port,
3886 storage_config,
3887 admission_facts_carrier_module_id,
3888 admission_facts_targets,
3889 modules,
3890 reserved_capabilities,
3891 ) = (
3892 config.port,
3893 config.storage,
3894 config.admission_facts_carrier_module_id,
3895 config.admission_facts_targets,
3896 config.modules,
3897 config.reserved_capabilities,
3898 );
3899
3900 let mut restart_required = Vec::new();
3909 for section in RestartRequiredSection::ALL {
3910 let changed = match section {
3911 RestartRequiredSection::Port => configured_port != context.configured_port,
3912 RestartRequiredSection::Storage => storage_config != context.storage_config,
3913 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
3914 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
3915 }
3916 RestartRequiredSection::AdmissionFactsTargets => {
3917 admission_facts_targets != context.admission_facts_targets
3918 }
3919 };
3920 if changed {
3921 restart_required.push(section.label().to_string());
3922 }
3923 }
3924 if !restart_required.is_empty() {
3925 warn!(
3926 config_path = %context.config_path.display(),
3927 sections = %restart_required.join(", "),
3928 "daemon config changed outside the modules section; restart the daemon to apply those changes"
3929 );
3930 }
3931
3932 for configured in &modules {
3933 if let Err(err) = validate_spec(&configured.module_spec()) {
3934 return Ok(vec![control_error_frame(
3935 &frame,
3936 "invalid_daemon_config",
3937 format!("supervisor rescan rejected daemon config: {err}"),
3938 )?]);
3939 }
3940 }
3941
3942 let configured_capabilities = modules
3943 .iter()
3944 .map(|module| (module.module_id.clone(), module.enabled))
3945 .collect::<Vec<_>>();
3946 let preview_capability_warnings = if preview {
3947 let (_, registrations) = self.runtime_capability_snapshot()?;
3948 let current_modules = self
3949 .supervisor
3950 .list()
3951 .into_iter()
3952 .map(|module| module.module_id().to_string())
3953 .collect::<BTreeSet<_>>();
3954 let resulting_modules = configured_capabilities.clone();
3955 let removed = current_modules
3956 .into_iter()
3957 .filter(|module_id| {
3958 !resulting_modules
3959 .iter()
3960 .any(|(configured_id, _)| configured_id == module_id)
3961 })
3962 .collect::<Vec<_>>();
3963 self.capability_evaluator.preview_removal_warnings(
3964 resulting_modules,
3965 &removed,
3966 ®istrations,
3967 )
3968 } else {
3969 Vec::new()
3970 };
3971 let result = match self
3972 .reconcile_supervised_modules(&context.supervisor, modules, preview)
3973 .await
3974 {
3975 Ok(result) => result,
3976 Err(message) => {
3977 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
3978 }
3979 };
3980 if !preview {
3981 self.capability_evaluator
3982 .configure(configured_capabilities, reserved_capabilities);
3983 self.capability_evaluator.wake_deadline_loop();
3984 self.refresh_capability_requirements();
3985 }
3986 let mut result = result;
3987 result.restart_required = restart_required;
3988 result.capability_warnings = preview_capability_warnings;
3989 let response = ClientControlResponse::SupervisorRescan { result };
3990 Ok(vec![control_response_body_frame(
3991 &frame,
3992 &response,
3993 "ClientControlResponse::SupervisorRescan",
3994 )?])
3995 }
3996
3997 async fn handle_supervisor_release_reserved(
3998 &self,
3999 frame: Frame,
4000 module_id: String,
4001 ) -> Result<Vec<Frame>, RouterError> {
4002 let Some(context) = self.rescan.clone() else {
4003 return Ok(vec![control_error_frame(
4004 &frame,
4005 "release_unavailable",
4006 "reserved-id release requires a daemon started with a reloadable config path",
4007 )?]);
4008 };
4009 let operation_lock = self.supervisor.operation_lock();
4010 let _operation_guard = operation_lock.lock().await;
4011 let loaded = match crate::daemon_config::load(&context.config_path) {
4012 Ok(Some(config)) => config,
4013 Ok(None) => {
4014 return Ok(vec![control_error_frame(
4015 &frame,
4016 "invalid_daemon_config",
4017 format!(
4018 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
4019 context.config_path.display()
4020 ),
4021 )?])
4022 }
4023 Err(err) => {
4024 return Ok(vec![control_error_frame(
4025 &frame,
4026 "invalid_daemon_config",
4027 format!("unable to verify reserved-id release against daemon config: {err}"),
4028 )?])
4029 }
4030 };
4031 if loaded
4032 .modules
4033 .iter()
4034 .any(|configured| configured.module_id == module_id)
4035 {
4036 return Ok(vec![control_error_frame(
4037 &frame,
4038 "reserved_module_configured",
4039 format!(
4040 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
4041 ),
4042 )?]);
4043 }
4044 if !self.supervisor.release_retained_reserved_gate(&module_id) {
4045 return Ok(vec![control_error_frame(
4046 &frame,
4047 "reserved_gate_not_retained",
4048 format!(
4049 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
4050 ),
4051 )?]);
4052 }
4053
4054 let response = ClientControlResponse::SupervisorAck {
4055 module_id,
4056 applied: true,
4057 };
4058 Ok(vec![control_response_body_frame(
4059 &frame,
4060 &response,
4061 "ClientControlResponse::SupervisorAck",
4062 )?])
4063 }
4064
4065 async fn reconcile_supervised_modules(
4074 &self,
4075 supervisor: &Supervisor,
4076 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
4077 preview: bool,
4078 ) -> Result<SupervisorRescanResult, String> {
4079 let mut current = BTreeMap::new();
4080 for module in self.supervisor.list() {
4081 let (spec, health) = module.configuration().map_err(|err| {
4082 format!(
4083 "failed to read configuration for module_id '{}': {err}",
4084 module.module_id()
4085 )
4086 })?;
4087 let enabled = module
4088 .status()
4089 .map_err(|err| {
4090 format!(
4091 "failed to read status for module_id '{}': {err}",
4092 module.module_id()
4093 )
4094 })?
4095 .enabled;
4096 current.insert(
4097 module.module_id().to_string(),
4098 (module, spec, health, enabled),
4099 );
4100 }
4101 let configured = configured_modules
4102 .into_iter()
4103 .map(|module| (module.module_id.clone(), module))
4104 .collect::<BTreeMap<_, _>>();
4105
4106 let added = configured
4107 .keys()
4108 .filter(|module_id| !current.contains_key(*module_id))
4109 .cloned()
4110 .collect::<Vec<_>>();
4111 let removed = current
4112 .keys()
4113 .filter(|module_id| !configured.contains_key(*module_id))
4114 .cloned()
4115 .collect::<Vec<_>>();
4116 let mut changed_pending_reload = Vec::new();
4117 let mut configuration_changes = BTreeSet::new();
4118 let mut enabled_changes = BTreeSet::new();
4119 let mut unchanged = 0_u32;
4120
4121 for (module_id, configured_module) in &configured {
4122 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
4123 else {
4124 continue;
4125 };
4126 let configuration_changed = *current_spec != configured_module.module_spec()
4127 || *current_health != configured_module.health;
4128 let enabled_changed = *current_enabled != configured_module.enabled;
4129 if configuration_changed {
4130 configuration_changes.insert(module_id.clone());
4131 changed_pending_reload.push(module_id.clone());
4132 }
4133 if enabled_changed {
4134 enabled_changes.insert(module_id.clone());
4135 }
4136 if !configuration_changed && !enabled_changed {
4137 unchanged = unchanged.saturating_add(1);
4138 }
4139 }
4140
4141 if preview {
4147 return Ok(SupervisorRescanResult {
4148 added,
4149 removed,
4150 changed_pending_reload,
4151 enabled_changes: enabled_changes.iter().cloned().collect(),
4152 unchanged,
4153 preview: true,
4154 restart_required: Vec::new(),
4158 capability_warnings: Vec::new(),
4159 });
4160 }
4161
4162 for module_id in &removed {
4163 let module = ¤t
4164 .get(module_id)
4165 .expect("removed module came from current supervisor state")
4166 .0;
4167 module.retire().await.map_err(|err| {
4168 format!("failed to retire module_id '{module_id}' during rescan: {err}")
4169 })?;
4170 self.supervisor.record_rescan_removal(module_id);
4204 self.supervisor.retire(module_id);
4205 self.route_outages.forget(module_id);
4206 }
4207
4208 for module_id in configured.keys() {
4209 let Some((module, _, _, _)) = current.get(module_id) else {
4210 continue;
4211 };
4212 let configured_module = configured
4213 .get(module_id)
4214 .expect("configured module id came from configured map");
4215 if configuration_changes.contains(module_id) {
4216 module
4217 .update_configuration(
4218 configured_module.module_spec(),
4219 configured_module.health,
4220 configured_module.drain_timeout_ms,
4221 )
4222 .await
4223 .map_err(|err| {
4224 format!(
4225 "failed to update module_id '{module_id}' configuration during rescan: {err}"
4226 )
4227 })?;
4228 }
4229 if enabled_changes.contains(module_id) {
4230 self.route_outages.mark_operator_action(module_id);
4233 module
4234 .set_enabled(configured_module.enabled)
4235 .await
4236 .map_err(|err| {
4237 self.route_outages.operator_action_ended_unrefused(module_id);
4238 format!(
4239 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
4240 configured_module.enabled
4241 )
4242 })?;
4243 }
4244 }
4245
4246 for module_id in &added {
4247 let configured_module = configured
4248 .get(module_id)
4249 .expect("added module id came from configured map");
4250 supervisor
4251 .supervise_configured_with_health(
4252 configured_module.module_spec(),
4253 configured_module.enabled,
4254 configured_module.health,
4255 configured_module.drain_timeout_ms,
4256 configured_module.restart,
4257 )
4258 .map_err(|err| {
4259 format!("failed to add module_id '{module_id}' during rescan: {err}")
4260 })?;
4261 }
4262
4263 Ok(SupervisorRescanResult {
4264 added,
4265 removed,
4266 changed_pending_reload,
4267 enabled_changes: enabled_changes.iter().cloned().collect(),
4268 unchanged,
4269 preview: false,
4270 restart_required: Vec::new(),
4273 capability_warnings: Vec::new(),
4274 })
4275 }
4276
4277 async fn handle_supervisor_set_enabled(
4278 &self,
4279 frame: Frame,
4280 module_id: String,
4281 enabled: bool,
4282 ) -> Result<Vec<Frame>, RouterError> {
4283 let operation_lock = self.supervisor.operation_lock();
4284 let _operation_guard = operation_lock.lock().await;
4285 let Some(module) = self.supervisor.get(&module_id) else {
4286 return Ok(vec![control_error_frame(
4287 &frame,
4288 "unknown_module",
4289 format!("module_id '{module_id}' is not supervised"),
4290 )?]);
4291 };
4292
4293 self.route_outages.mark_operator_action(&module_id);
4296 let applied = match module.set_enabled(enabled).await {
4297 Ok(applied) => applied,
4298 Err(err) => {
4299 self.route_outages
4300 .operator_action_ended_unrefused(&module_id);
4301 return Ok(vec![control_error_frame(
4302 &frame,
4303 "target_unavailable",
4304 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
4305 )?]);
4306 }
4307 };
4308 if !applied {
4309 self.route_outages
4312 .operator_action_ended_unrefused(&module_id);
4313 }
4314
4315 self.capability_evaluator.wake_deadline_loop();
4316 self.refresh_capability_requirements();
4317 let response = ClientControlResponse::SupervisorAck { module_id, applied };
4318 Ok(vec![control_response_body_frame(
4319 &frame,
4320 &response,
4321 "ClientControlResponse::SupervisorAck",
4322 )?])
4323 }
4324
4325 async fn handle_supervisor_health_probe(
4326 &self,
4327 frame: Frame,
4328 module_id: String,
4329 ) -> Result<Vec<Frame>, RouterError> {
4330 self.refresh_capability_requirements();
4331 let Some(registration) = self
4332 .registry
4333 .get_module(&module_id)
4334 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4335 else {
4336 return Ok(vec![control_error_frame(
4337 &frame,
4338 "unknown_module",
4339 format!("module_id '{module_id}' is not registered"),
4340 )?]);
4341 };
4342
4343 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
4356 {
4357 return Ok(vec![control_error_frame(
4358 &frame,
4359 "health_not_advertised",
4360 format!("module_id '{module_id}' did not advertise health.check"),
4361 )?]);
4362 }
4363
4364 let deadline = Instant::now() + self.health_probe_timeout;
4365 let pending = match self.forwarding.begin_module_control_rpc_for(
4366 &module_id,
4367 MODULE_CONTROL_OP_HEALTH_CHECK,
4368 deadline,
4369 ) {
4370 Ok(pending) => pending,
4371 Err(err) => {
4372 return Ok(vec![control_error_frame(
4373 &frame,
4374 forwarding_error_code(&err),
4375 err.to_string(),
4376 )?])
4377 }
4378 };
4379
4380 let PendingModuleControlRpc {
4381 endpoint,
4382 module_sink,
4383 negotiated_ver,
4384 corr: probe_corr,
4385 receiver,
4386 } = pending;
4387 let mut guard =
4388 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
4389 let probe_body =
4390 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
4391 RouterError::backend(
4392 0,
4393 frame.header.corr,
4394 format!("failed to encode health.check request: {err}"),
4395 )
4396 })?;
4397 let probe_frame = Frame::build_with_version(
4398 negotiated_ver,
4399 FrameType::Request,
4400 control_flags(),
4401 0,
4402 0,
4403 probe_corr,
4404 probe_body,
4405 )
4406 .map_err(RouterError::FrameBuild)?;
4407
4408 if let Err(err) = module_sink.send(probe_frame).await {
4409 return Ok(vec![control_error_frame(
4410 &frame,
4411 "target_unavailable",
4412 err.to_string(),
4413 )?]);
4414 }
4415
4416 match timeout_at(deadline, receiver).await {
4417 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
4418 guard.disarm();
4419 let Some(report) = response.health_report() else {
4420 return Ok(vec![control_error_frame(
4421 &frame,
4422 "invalid_control_body",
4423 "health.check RPC returned a non-health response",
4424 )?]);
4425 };
4426 let HealthReport {
4431 status,
4432 detail,
4433 metrics,
4434 } = report;
4435 let capability_detail = self
4436 .capability_evaluator
4437 .required_problem_detail(&module_id);
4438 let response = ClientControlResponse::SupervisorHealthProbe {
4439 module_id,
4440 status,
4441 detail: append_capability_problem_detail(detail, capability_detail),
4442 metrics,
4443 };
4444 Ok(vec![control_response_body_frame(
4445 &frame,
4446 &response,
4447 "ClientControlResponse::SupervisorHealthProbe",
4448 )?])
4449 }
4450 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
4451 guard.disarm();
4452 Ok(vec![control_error_body_frame(&frame, body)?])
4453 }
4454 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
4455 guard.disarm();
4456 Ok(vec![control_error_frame(
4457 &frame,
4458 "target_unavailable",
4459 message,
4460 )?])
4461 }
4462 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
4463 guard.disarm();
4464 Ok(vec![control_error_frame(
4465 &frame,
4466 "invalid_control_body",
4467 message,
4468 )?])
4469 }
4470 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
4471 guard.disarm();
4472 Ok(vec![control_error_frame(
4473 &frame,
4474 "invalid_control_body",
4475 format!("expected module-control op '{expected}', got '{actual}'"),
4476 )?])
4477 }
4478 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
4479 guard.disarm();
4480 Ok(vec![control_error_frame(
4481 &frame,
4482 "module_timeout",
4483 format!(
4484 "module_id '{module_id}' answered health.check after {:?}",
4485 self.health_probe_timeout
4486 ),
4487 )?])
4488 }
4489 Ok(Err(_)) => Ok(vec![control_error_frame(
4490 &frame,
4491 "target_unavailable",
4492 "health.check waiter was canceled before the module responded",
4493 )?]),
4494 Err(_) => Ok(vec![control_error_frame(
4495 &frame,
4496 "module_timeout",
4497 format!(
4498 "module_id '{module_id}' did not answer health.check within {:?}",
4499 self.health_probe_timeout
4500 ),
4501 )?]),
4502 }
4503 }
4504
4505 fn supervisor_status(
4506 &self,
4507 module_id: &str,
4508 corr: u64,
4509 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
4510 self.supervisor
4511 .get(module_id)
4512 .map(|module| {
4513 let warming = module.is_warming_for_control("status").map_err(|err| {
4514 RouterError::backend(
4515 0,
4516 corr,
4517 format!(
4518 "failed to read supervisor warming state for module_id '{module_id}': {err}"
4519 ),
4520 )
4521 })?;
4522 module.status_for_control("status").map_err(|err| {
4523 RouterError::backend(
4524 0,
4525 corr,
4526 format!(
4527 "failed to read supervisor status for module_id '{module_id}': {err}"
4528 ),
4529 )
4530 }).map(|status| (status, warming))
4531 })
4532 .transpose()
4533 }
4534
4535 fn guard_module_control_op(
4536 &self,
4537 frame: &Frame,
4538 module_id: &str,
4539 op: &str,
4540 ) -> Result<Option<Frame>, RouterError> {
4541 if self.module_grants_op(module_id, op, frame.header.corr)? {
4542 return Ok(None);
4543 }
4544
4545 Ok(Some(control_error_frame(
4546 frame,
4547 "op_not_allowed",
4548 format!("module_id '{module_id}' did not grant control op '{op}'"),
4549 )?))
4550 }
4551
4552 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
4553 let Some(registration) = self
4554 .registry
4555 .get_module(module_id)
4556 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
4557 else {
4558 return Ok(false);
4559 };
4560 Ok(module_registration_grants_op(®istration.control_ops, op))
4561 }
4562
4563 fn handle_status_update(
4564 &self,
4565 endpoint: ModuleEndpointId,
4566 frame: Frame,
4567 ) -> Result<Vec<Frame>, RouterError> {
4568 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
4569 Ok(update) => update,
4570 Err(err) => {
4571 if is_known_module_push_op(&frame.body) {
4576 return Ok(vec![control_error_frame(
4577 &frame,
4578 "invalid_control_body",
4579 format!("malformed module control push body: {err}"),
4580 )?]);
4581 }
4582 return Ok(Vec::new());
4583 }
4584 };
4585
4586 match update {
4587 ModuleControlPush::RouteStatus {
4588 route_channel,
4589 route_epoch,
4590 status,
4591 } => {
4592 self.forwarding
4593 .cache_status(endpoint, route_channel, route_epoch, status)
4594 .map_err(RouterError::Forwarding)?;
4595 }
4596 }
4597 Ok(Vec::new())
4598 }
4599
4600 fn handle_route_poll(
4601 &self,
4602 ctx: &RouteCtx,
4603 frame: Frame,
4604 route_channel: u16,
4605 route_epoch: u32,
4606 kind: PollKind,
4607 ) -> Result<Vec<Frame>, RouterError> {
4608 let snapshot = self
4609 .forwarding
4610 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
4611 .map_err(RouterError::Forwarding)?;
4612 let response = match (kind, snapshot) {
4613 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
4614 ClientControlResponse::RoutePoll {
4615 route_channel,
4616 route_epoch,
4617 status,
4618 live: None,
4619 }
4620 }
4621 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
4622 route_channel,
4623 route_epoch,
4624 status: None,
4625 live: None,
4626 },
4627 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
4628 let live = self
4654 .process_liveness
4655 .as_ref()
4656 .and_then(|source| source.process_live(&module_id))
4657 .unwrap_or(true);
4658 ClientControlResponse::RoutePoll {
4659 route_channel,
4660 route_epoch,
4661 status: None,
4662 live: Some(live),
4663 }
4664 }
4665 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
4666 route_channel,
4667 route_epoch,
4668 status: None,
4669 live: Some(false),
4670 },
4671 };
4672
4673 Ok(vec![control_response_body_frame(
4674 &frame,
4675 &response,
4676 "ClientControlResponse::RoutePoll",
4677 )?])
4678 }
4679
4680 pub(crate) fn observe_module_control_completion(
4681 &self,
4682 completion: ModuleControlRpcCompletion,
4683 ) -> bool {
4684 match completion {
4685 ModuleControlRpcCompletion::Unknown => false,
4686 ModuleControlRpcCompletion::Settled => true,
4687 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
4688 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
4689 info!(
4690 module_id = %module_id,
4691 latency_ms,
4692 "late health.check answer proves the module is alive"
4693 );
4694 match self
4695 .supervisor
4696 .record_late_health_answer(&module_id, latency_ms)
4697 {
4698 Ok(true) => {}
4699 Ok(false) => debug!(
4700 module_id = %module_id,
4701 latency_ms,
4702 "late health.check answer has no active supervisor snapshot"
4703 ),
4704 Err(err) => warn!(
4705 module_id = %module_id,
4706 latency_ms,
4707 error = %err,
4708 "failed to record late health.check answer"
4709 ),
4710 }
4711 true
4712 }
4713 }
4714 }
4715
4716 fn refuse_to_end_module_connection_for_a_client(
4737 &self,
4738 module_connection_id: ConnectionId,
4739 corr: u64,
4740 err: ForwardingError,
4741 ) -> Result<(), RouterError> {
4742 if let ForwardingError::ConnectionClosing { connection_id } = err {
4743 if connection_id != module_connection_id {
4744 warn!(
4745 module_connection_id = module_connection_id.get(),
4746 client_connection_id = connection_id.get(),
4747 corr,
4748 "dropping a route.bind response for a closing client; the module connection keeps serving"
4749 );
4750 return Ok(());
4751 }
4752 }
4753 Err(RouterError::Forwarding(err))
4754 }
4755
4756 fn handle_module_relay_response(
4757 &self,
4758 connection_id: ConnectionId,
4759 frame: Frame,
4760 ) -> Result<Vec<Frame>, RouterError> {
4761 let mut secondary_error = None;
4762 let outcome = match frame.header.ty {
4763 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
4764 Ok(probe) if probe.op == "route.bind" => {
4765 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
4766 Ok(ModuleControlResponse::RouteBindAck {}) => {
4767 RouteBindRelayOutcome::Accepted
4768 }
4769 Ok(other) => {
4770 let message =
4771 format!("route.bind response carried unexpected body: {other:?}");
4772 secondary_error = Some(control_error_frame(
4773 &frame,
4774 "invalid_control_body",
4775 message.clone(),
4776 )?);
4777 RouteBindRelayOutcome::ModuleGone(message)
4778 }
4779 Err(err) => {
4780 let message = format!("malformed route.bind response body: {err}");
4781 secondary_error = Some(control_error_frame(
4782 &frame,
4783 "invalid_control_body",
4784 message.clone(),
4785 )?);
4786 RouteBindRelayOutcome::ModuleGone(message)
4787 }
4788 }
4789 }
4790 Ok(probe) => {
4791 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
4792 {
4793 Ok(response) => ModuleControlRpcOutcome::Response(response),
4794 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
4795 "malformed {} response body: {err}",
4796 probe.op
4797 )),
4798 };
4799 let completion = self
4800 .forwarding
4801 .complete_module_control_rpc(
4802 connection_id,
4803 frame.header.corr,
4804 Some(&probe.op),
4805 outcome,
4806 )
4807 .map_err(RouterError::Forwarding)?;
4808 if !self.observe_module_control_completion(completion) {
4809 debug!(
4810 connection_id = connection_id.get(),
4811 corr = frame.header.corr,
4812 op = %probe.op,
4813 "dropping late or unknown module-control RPC response"
4814 );
4815 }
4816 return Ok(Vec::new());
4817 }
4818 Err(err) => {
4819 if let Some(expected_op) = self
4820 .forwarding
4821 .pending_module_control_op(connection_id, frame.header.corr)
4822 .map_err(RouterError::Forwarding)?
4823 {
4824 let completion = self
4825 .forwarding
4826 .complete_module_control_rpc(
4827 connection_id,
4828 frame.header.corr,
4829 None,
4830 ModuleControlRpcOutcome::MalformedResponse(format!(
4831 "malformed {expected_op} response body: {err}"
4832 )),
4833 )
4834 .map_err(RouterError::Forwarding)?;
4835 if !self.observe_module_control_completion(completion) {
4836 debug!(
4837 connection_id = connection_id.get(),
4838 corr = frame.header.corr,
4839 "dropping late malformed module-control RPC response"
4840 );
4841 }
4842 return Ok(Vec::new());
4843 }
4844 let message = format!("malformed route.bind response body: {err}");
4845 secondary_error = Some(control_error_frame(
4846 &frame,
4847 "invalid_control_body",
4848 message.clone(),
4849 )?);
4850 RouteBindRelayOutcome::ModuleGone(message)
4851 }
4852 },
4853 FrameType::Error => {
4854 if self
4855 .forwarding
4856 .pending_module_control_op(connection_id, frame.header.corr)
4857 .map_err(RouterError::Forwarding)?
4858 .is_some()
4859 {
4860 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
4861 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
4862 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
4863 "malformed module-control ERROR body: {err}"
4864 )),
4865 };
4866 let completion = self
4867 .forwarding
4868 .complete_module_control_rpc(
4869 connection_id,
4870 frame.header.corr,
4871 None,
4872 outcome,
4873 )
4874 .map_err(RouterError::Forwarding)?;
4875 if !self.observe_module_control_completion(completion) {
4876 debug!(
4877 connection_id = connection_id.get(),
4878 corr = frame.header.corr,
4879 "dropping late or unknown module-control RPC error"
4880 );
4881 }
4882 return Ok(Vec::new());
4883 }
4884 match serde_json::from_slice::<ErrorBody>(&frame.body) {
4885 Ok(body) => RouteBindRelayOutcome::Rejected(body),
4886 Err(err) => {
4887 let message = format!("malformed route.bind ERROR body: {err}");
4888 secondary_error = Some(control_error_frame(
4889 &frame,
4890 "invalid_control_body",
4891 message.clone(),
4892 )?);
4893 RouteBindRelayOutcome::ModuleGone(message)
4894 }
4895 }
4896 }
4897 ty => {
4898 return Ok(vec![control_error_frame(
4899 &frame,
4900 "unsupported_control_frame",
4901 format!("unsupported module channel-0 frame {ty:?}"),
4902 )?])
4903 }
4904 };
4905
4906 let settled =
4907 self.forwarding
4908 .complete_pending_relay(connection_id, frame.header.corr, outcome);
4909 let completion = match settled {
4910 Ok(completion) => completion,
4911 Err(err) => {
4912 self.refuse_to_end_module_connection_for_a_client(
4913 connection_id,
4914 frame.header.corr,
4915 err,
4916 )?;
4917 return Ok(secondary_error.into_iter().collect());
4918 }
4919 };
4920 if let Some(target) = completion.abandoned.as_ref() {
4921 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
4922 }
4923 if !completion.settled {
4924 debug!(
4925 connection_id = connection_id.get(),
4926 corr = frame.header.corr,
4927 frame_type = ?frame.header.ty,
4928 "dropping late or unknown route.bind relay response"
4929 );
4930 }
4931 Ok(secondary_error.into_iter().collect())
4932 }
4933
4934 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
4935 debug!(connection_id = connection_id.get(), "handling GOODBYE");
4936 let registrations = self
4937 .deregister_connection(connection_id)
4938 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
4939 let released_routes = self
4940 .forwarding
4941 .cleanup_connection(connection_id)
4942 .map_err(RouterError::Forwarding)?;
4943 self.emit_route_goodbyes(released_routes);
4944 if !registrations.is_empty() {
4946 crate::supervise::notify_registration_release();
4947 }
4948 Ok(Vec::new())
4949 }
4950}
4951
4952impl Default for ControlHandler {
4953 fn default() -> Self {
4954 Self::new(Arc::new(Registry::default()))
4955 }
4956}
4957
4958impl crate::supervise::SwapPromotionObserver for ControlHandler {
4959 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
4960 self.apply_registration_capabilities(registration);
4961 }
4962}
4963
4964fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
4965 CapabilityRequirementStatus {
4966 consumer: status.consumer,
4967 capability: status.capability,
4968 need: match status.need {
4969 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
4970 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
4971 },
4972 verdict: status.verdict.as_str().to_string(),
4973 episode_seq: status.episode_seq,
4974 config_satisfiable: status.config_satisfiable,
4975 runtime_available: status.runtime_available,
4976 detail: status.detail,
4977 }
4978}
4979
4980fn append_capability_problem_detail(
4981 detail: Option<String>,
4982 capability_detail: Option<String>,
4983) -> Option<String> {
4984 match (detail, capability_detail) {
4985 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
4986 (Some(detail), None) => Some(detail),
4987 (None, Some(capability_detail)) => Some(capability_detail),
4988 (None, None) => None,
4989 }
4990}
4991
4992fn subc_ops() -> Vec<String> {
4993 SUBC_CONTROL_OPS
4994 .iter()
4995 .map(|op| (*op).to_string())
4996 .collect()
4997}
4998
4999fn module_subc_ops() -> Vec<String> {
5000 SUBC_CONTROL_OPS
5001 .iter()
5002 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
5003 .map(|op| (*op).to_string())
5004 .collect()
5005}
5006
5007#[cfg(test)]
5008fn module_baseline_control_ops() -> Vec<String> {
5009 MODULE_BASELINE_CONTROL_OPS
5010 .iter()
5011 .map(|op| (*op).to_string())
5012 .collect()
5013}
5014
5015fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
5016 let mut seen = HashSet::new();
5017 let mut effective = Vec::new();
5018 for op in MODULE_BASELINE_CONTROL_OPS {
5019 if seen.insert((*op).to_string()) {
5020 effective.push((*op).to_string());
5021 }
5022 }
5023 for op in declared.unwrap_or_default() {
5024 if seen.insert(op.clone()) {
5025 effective.push(op);
5026 }
5027 }
5028 effective
5029}
5030
5031fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
5032 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
5033}
5034
5035fn target_module_id(target: &RouteTarget) -> &str {
5036 match target {
5037 RouteTarget::ToolProvider { module_id }
5038 | RouteTarget::ManagementSurface { module_id }
5039 | RouteTarget::InternalService { module_id, .. } => module_id,
5040 }
5041}
5042
5043fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
5044 roles.iter().any(|role| match (target, role) {
5045 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
5046 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
5047 (
5048 RouteTarget::InternalService { service_id, .. },
5049 ProviderRole::InternalService {
5050 service_id: provided,
5051 ..
5052 },
5053 ) => service_id == provided,
5054 _ => false,
5055 })
5056}
5057
5058fn is_routable_role(role: &ProviderRole) -> bool {
5059 matches!(
5060 role,
5061 ProviderRole::ToolProvider { .. }
5062 | ProviderRole::ManagementSurface { .. }
5063 | ProviderRole::InternalService { .. }
5064 )
5065}
5066
5067#[derive(Debug, Clone, Copy, PartialEq, Eq)]
5068enum ControlRequestBodyError {
5069 UnknownOp,
5070 InvalidBody,
5071}
5072
5073#[derive(Debug, Deserialize)]
5074struct ControlOpProbe {
5075 op: String,
5076}
5077
5078const MODULE_PUSH_OPS: &[&str] = &["route.status"];
5081
5082fn is_known_module_push_op(body: &[u8]) -> bool {
5083 serde_json::from_slice::<ControlOpProbe>(body)
5084 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
5085 .unwrap_or(false)
5086}
5087
5088fn is_known_module_request_op(body: &[u8]) -> bool {
5089 serde_json::from_slice::<ControlOpProbe>(body)
5090 .map(|probe| MODULE_TO_SUBC_CONTROL_OPS.contains(&probe.op.as_str()))
5091 .unwrap_or(false)
5092}
5093
5094fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
5095 debug!(
5096 op = %op,
5097 connection_id = connection_id.get(),
5098 corr,
5099 "control dispatch"
5100 );
5101}
5102
5103fn log_slow_control_dispatch(
5104 dispatch_started_at: Option<StdInstant>,
5105 op: &'static str,
5106 connection_id: ConnectionId,
5107 corr: u64,
5108) {
5109 let Some(dispatch_started_at) = dispatch_started_at else {
5110 return;
5111 };
5112 let elapsed = dispatch_started_at.elapsed();
5113 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
5114 warn!(
5115 op = %op,
5116 connection_id = connection_id.get(),
5117 corr,
5118 elapsed_ms = elapsed.as_millis() as u64,
5119 "slow control dispatch"
5120 );
5121 }
5122}
5123
5124fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
5125 match request {
5126 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
5127 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
5128 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
5129 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
5130 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
5131 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
5132 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
5133 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
5134 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
5135 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
5136 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
5137 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
5138 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
5139 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
5140 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
5141 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
5142 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
5143 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
5144 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
5145 }
5146}
5147
5148fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
5149 match request {
5150 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
5151 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
5152 }
5153}
5154
5155fn parse_client_control_request(
5156 body: &[u8],
5157) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
5158 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
5159 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5160 Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
5161 ControlRequestBodyError::InvalidBody
5162 }
5163 Ok(_) => ControlRequestBodyError::UnknownOp,
5164 Err(_) => ControlRequestBodyError::InvalidBody,
5165 };
5166 (err, classification)
5167 })
5168}
5169
5170fn parse_module_control_request_from_module(
5171 body: &[u8],
5172) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
5173 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
5174 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5175 Ok(probe) if MODULE_TO_SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
5176 ControlRequestBodyError::InvalidBody
5177 }
5178 Ok(_) => ControlRequestBodyError::UnknownOp,
5179 Err(_) => ControlRequestBodyError::InvalidBody,
5180 };
5181 (err, classification)
5182 })
5183}
5184
5185#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
5186enum ProviderRoleKind {
5187 ToolProvider,
5188 PipelineStage,
5189 ManagementSurface,
5190 InternalService,
5191}
5192
5193fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
5194 match role {
5195 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
5196 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
5197 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
5198 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
5199 }
5200}
5201
5202fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
5203 roles.iter().map(provider_role_kind).collect()
5204}
5205
5206fn capability_census_trigger(
5210 old: Option<&CapabilityDeclarations>,
5211 new: Option<&CapabilityDeclarations>,
5212) -> bool {
5213 let old_provides = old
5214 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
5215 .unwrap_or_default();
5216 let old_denies = old
5217 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
5218 .unwrap_or_default();
5219 let new = new.cloned().unwrap_or(CapabilityDeclarations {
5220 provides: Vec::new(),
5221 requires: Vec::new(),
5222 must_never_reach: Vec::new(),
5223 });
5224
5225 new.provides
5226 .iter()
5227 .any(|capability| !old_provides.contains(capability))
5228 || new
5229 .must_never_reach
5230 .iter()
5231 .any(|capability| !old_denies.contains(capability))
5232}
5233
5234fn denied_capability<'a>(
5237 opening_manifest: &'a ModuleManifest,
5238 target_manifest: &ModuleManifest,
5239) -> Option<&'a str> {
5240 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
5241 let target_capabilities = target_manifest.capabilities.as_ref()?;
5242 opening_capabilities
5243 .must_never_reach
5244 .iter()
5245 .find(|denied| {
5246 target_capabilities
5247 .provides
5248 .iter()
5249 .any(|provided| provided == *denied)
5250 })
5251 .map(String::as_str)
5252}
5253
5254fn catalog_update_frozen_field_message(
5255 registered: &ModuleManifest,
5256 provides: &[ProviderRole],
5257) -> Option<String> {
5258 let old_has_provides = !registered.provides.is_empty();
5259 let new_has_provides = !provides.is_empty();
5260 if old_has_provides != new_has_provides {
5261 return Some(format!(
5262 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
5263 registered.module_id
5264 ));
5265 }
5266
5267 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
5268 return Some(format!(
5269 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
5270 registered.module_id
5271 ));
5272 }
5273
5274 let registered_concurrency = manifest_concurrency(registered);
5275 let mut candidate = registered.clone();
5276 candidate.provides = provides.to_vec();
5277 let candidate_concurrency = manifest_concurrency(&candidate);
5278 if candidate_concurrency != registered_concurrency {
5279 return Some(format!(
5280 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
5281 registered.module_id, registered_concurrency, candidate_concurrency
5282 ));
5283 }
5284
5285 None
5288}
5289
5290fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
5291 manifest.provides.iter().any(is_routable_role)
5292}
5293
5294fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
5300 manifest
5301 .provides
5302 .iter()
5303 .find_map(|provider| match provider {
5304 ProviderRole::ToolProvider { concurrency, .. }
5305 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
5306 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
5307 })
5308 .unwrap_or(Concurrency::ModuleManaged)
5309}
5310
5311fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
5318 let has_management_surface = manifest
5319 .provides
5320 .iter()
5321 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
5322 if !has_management_surface {
5323 return false;
5324 }
5325 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
5326 return false;
5327 };
5328 let Some(provides) = raw
5329 .get("manifest")
5330 .and_then(|manifest| manifest.get("provides"))
5331 .and_then(serde_json::Value::as_array)
5332 else {
5333 return false;
5334 };
5335 provides.iter().any(|role| {
5340 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
5341 && role.get("concurrency").is_none()
5342 })
5343}
5344
5345fn negotiate_version(peer_version: u8) -> Result<u8, String> {
5346 if peer_version != PROTOCOL_VERSION {
5347 return Err(format!(
5348 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
5349 ));
5350 }
5351 Ok(PROTOCOL_VERSION)
5352}
5353
5354fn pong(frame: &Frame) -> Result<Frame, RouterError> {
5355 Frame::build_with_version(
5356 response_version(frame),
5357 FrameType::Pong,
5358 frame.header.flags,
5359 0,
5360 0,
5361 frame.header.corr,
5362 Vec::new(),
5363 )
5364 .map_err(RouterError::FrameBuild)
5365}
5366
5367fn control_error_frame(
5368 frame: &Frame,
5369 code: &'static str,
5370 message: impl Into<String>,
5371) -> Result<Frame, RouterError> {
5372 control_error_body_frame(
5373 frame,
5374 ErrorBody {
5375 code: code.to_string(),
5376 message: message.into(),
5377 detail: None,
5378 },
5379 )
5380}
5381
5382fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
5383 let body = serde_json::to_vec(&error).map_err(|err| {
5384 RouterError::backend(
5385 0,
5386 frame.header.corr,
5387 format!("failed to encode control ERROR: {err}"),
5388 )
5389 })?;
5390
5391 Frame::build_with_version(
5392 response_version(frame),
5393 FrameType::Error,
5394 control_flags(),
5395 0,
5396 0,
5397 frame.header.corr,
5398 body,
5399 )
5400 .map_err(RouterError::FrameBuild)
5401}
5402
5403fn control_response_body_frame<T: Serialize>(
5404 frame: &Frame,
5405 reply: &T,
5406 label: &'static str,
5407) -> Result<Frame, RouterError> {
5408 let body = serde_json::to_vec(reply).map_err(|err| {
5409 RouterError::backend(
5410 0,
5411 frame.header.corr,
5412 format!("failed to encode {label}: {err}"),
5413 )
5414 })?;
5415
5416 Frame::build_with_version(
5417 response_version(frame),
5418 FrameType::Response,
5419 control_flags(),
5420 0,
5421 0,
5422 frame.header.corr,
5423 body,
5424 )
5425 .map_err(RouterError::FrameBuild)
5426}
5427
5428fn forwarding_error_code(err: &ForwardingError) -> &'static str {
5458 match err {
5459 ForwardingError::NoModuleConnection => "target_unavailable",
5460 ForwardingError::ModuleReloading { .. } => "module_reloading",
5461 ForwardingError::ClientRouteChannelExhausted { .. }
5462 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
5463 ForwardingError::StaleModuleEndpoint
5464 | ForwardingError::UnknownReservation { .. }
5465 | ForwardingError::ConnectionClosing { .. }
5466 | ForwardingError::ClientEgressClosed { .. }
5467 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
5468 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
5471 ForwardingError::RelayCorrelationExhausted
5472 | ForwardingError::RouteOpenBuild(_)
5473 | ForwardingError::Poisoned => "forwarding_error",
5474 }
5475}
5476
5477fn response_version(frame: &Frame) -> u8 {
5478 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
5479 frame.header.ver
5480 } else {
5481 PROTOCOL_VERSION
5482 }
5483}
5484
5485fn control_flags() -> Flags {
5486 Flags::new(false, Priority::Passive, false)
5487}
5488
5489fn send_goodbye_target_best_effort(
5494 counters: &DaemonCounters,
5495 target: &GoodbyeTarget,
5496 context: &'static str,
5497) {
5498 let Ok(frame) = Frame::build_with_version(
5499 target.negotiated_ver,
5500 FrameType::Goodbye,
5501 control_flags(),
5502 target.channel,
5503 target.epoch,
5504 0,
5505 Vec::new(),
5506 ) else {
5507 return;
5508 };
5509 crate::forwarding::send_module_route_goodbye(
5510 counters,
5511 &target.sink,
5512 frame,
5513 target.module_id.as_deref(),
5514 context,
5515 );
5516}
5517
5518pub(crate) fn send_route_control_pushes(
5519 forwarding: &ForwardingTable,
5520 routes: Vec<EndpointRoute>,
5521 push: ClientControlPush,
5522) {
5523 let body = match serde_json::to_vec(&push) {
5524 Ok(body) => body,
5525 Err(err) => {
5526 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
5527 return;
5528 }
5529 };
5530 let mut targets = Vec::new();
5531 for route in routes {
5532 let target = route.goodbye_target;
5533 if let Some(existing) = targets
5534 .iter()
5535 .find(|existing: &&GoodbyeTarget| existing.connection_id == target.connection_id)
5536 {
5537 debug_assert_eq!(
5538 existing.negotiated_ver, target.negotiated_ver,
5539 "one connection cannot negotiate multiple frame versions"
5540 );
5541 continue;
5542 }
5543 targets.push(target);
5544 }
5545 for target in targets {
5546 let frame = match Frame::build_with_version(
5547 target.negotiated_ver,
5548 FrameType::Push,
5549 control_flags(),
5550 0,
5551 0,
5552 0,
5553 body.clone(),
5554 ) {
5555 Ok(frame) => frame,
5556 Err(err) => {
5557 warn!(
5558 route_channel = target.channel,
5559 error = %err,
5560 "failed to build route lifecycle control PUSH frame"
5561 );
5562 continue;
5563 }
5564 };
5565 if let Err(err) = target.sink.try_send(frame) {
5566 if target.close_on_delivery_failure() {
5567 warn!(
5568 target_connection_id = target.connection_id.get(),
5569 route_channel = target.channel,
5570 error = %err,
5571 "route lifecycle control PUSH was not delivered to client; closing target connection"
5572 );
5573 let _ = forwarding.escalate_client_delivery_failure(
5574 target.connection_id,
5575 target.channel,
5576 target.epoch,
5577 CloseReason::new(
5578 "route_lifecycle_push_delivery_failed",
5579 format!(
5580 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
5581 target.channel
5582 ),
5583 ),
5584 crate::forwarding::UndeliveredFrame {
5585 module_id: target.module_id.as_deref(),
5586 sink: &target.sink,
5587 },
5588 );
5589 }
5590 }
5591 }
5592}
5593
5594#[cfg(test)]
5595mod tests {
5596 use std::{
5597 collections::BTreeMap,
5598 fmt,
5599 path::PathBuf,
5600 sync::{Arc, Mutex},
5601 time::Duration,
5602 };
5603 use subc_test_support::TestTempDir;
5604
5605 use serde_json::{json, Value};
5606 use subc_protocol::{
5607 manifest::{
5608 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
5609 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
5610 },
5611 session::HealthStatus,
5612 FrameType,
5613 };
5614
5615 use super::*;
5616 use crate::{
5617 forwarding::{DataRoute, DataRouteState},
5618 registry::ChannelState,
5619 router::FrameSink,
5620 stderr_tail::DEFAULT_MAX_LINE_BYTES,
5621 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
5622 RouteCtx, Router,
5623 };
5624 use tokio::{
5625 sync::mpsc,
5626 time::{sleep, Instant},
5627 };
5628 use tracing::{
5629 field::{Field, Visit},
5630 Event, Subscriber,
5631 };
5632 use tracing_subscriber::{layer::Context, prelude::*, Layer};
5633
5634 fn fake_aft_stub_path() -> PathBuf {
5653 let mut path = std::env::current_exe().expect("current_exe available in tests");
5654 path.pop(); path.pop(); path.push(if cfg!(windows) {
5657 "fake-aft-stub.exe"
5658 } else {
5659 "fake-aft-stub"
5660 });
5661 assert!(
5662 path.exists(),
5663 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
5664 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
5665 path.display()
5666 );
5667 path
5668 }
5669
5670 fn client_retries(code: &str) -> bool {
5675 subc_protocol::error_codes::is_retryable_route_open(code)
5676 }
5677
5678 #[test]
5689 fn retryability_of_forwarding_codes_matches_the_failure() {
5690 let transient = [
5693 ForwardingError::NoModuleConnection,
5694 ForwardingError::ModuleReloading {
5695 module_id: "m".into(),
5696 },
5697 ForwardingError::StaleModuleEndpoint,
5698 ForwardingError::UnknownReservation {
5699 client_channel: 1,
5700 module_channel: 1,
5701 },
5702 ForwardingError::ConnectionClosing {
5703 connection_id: ConnectionId::new(1),
5704 },
5705 ForwardingError::ClientEgressClosed {
5706 connection_id: ConnectionId::new(1),
5707 },
5708 ForwardingError::ModuleEgressUnavailable {
5709 connection_id: ConnectionId::new(1),
5710 },
5711 ];
5712 for err in transient {
5713 let code = forwarding_error_code(&err);
5714 assert!(
5715 client_retries(code),
5716 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
5717 );
5718 }
5719
5720 let permanent = [
5725 ForwardingError::ClientRouteChannelExhausted {
5726 connection_id: ConnectionId::new(1),
5727 },
5728 ForwardingError::ModuleRouteChannelExhausted {
5729 endpoint: ModuleEndpointId {
5730 connection_id: ConnectionId::new(1),
5731 generation: 1,
5732 },
5733 },
5734 ForwardingError::RelayCorrelationExhausted,
5735 ForwardingError::RouteOpenBuild("x".into()),
5736 ForwardingError::Poisoned,
5737 ];
5738 for err in permanent {
5739 let code = forwarding_error_code(&err);
5740 assert!(
5741 !client_retries(code),
5742 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
5743 );
5744 }
5745 }
5746
5747 #[tokio::test]
5775 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
5776 let handler = ControlHandler::default();
5777 let frame =
5778 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
5779
5780 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
5784 assert_eq!(
5785 stamped,
5786 Principal::Direct,
5787 "a caller that proved nothing must not be stamped as a supervised module"
5788 );
5789
5790 let forged = handler
5795 .route_open_principal(
5796 &frame,
5797 Some(ConsumerIdentity {
5798 module_id: "aft".to_string(),
5799 launch_nonce: "not-a-real-nonce".to_string(),
5800 }),
5801 )
5802 .unwrap();
5803 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
5804 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
5805 }
5806
5807 #[test]
5824 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
5825 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
5826 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
5827 let ClientControlRequest::RouteOpen {
5828 consumer_identity, ..
5829 } = parsed
5830 else {
5831 panic!("route.open body must parse as RouteOpen");
5832 };
5833 assert_eq!(
5834 consumer_identity,
5835 Some(ConsumerIdentity {
5836 module_id: "aft".to_string(),
5837 launch_nonce: "n".to_string(),
5838 }),
5839 "the wire field name must reach the value route_open_principal reads"
5840 );
5841 }
5842
5843 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
5844 ModuleManifest::builder(module_id, "0.1.0")
5845 .protocol_ver(protocol_ver)
5846 .provides(vec![ProviderRole::ToolProvider {
5847 tools: vec![Tool {
5848 name: "read".to_string(),
5849 description: None,
5850 execution_mode: ExecutionMode::Pure,
5851 schema: json!({"type": "object"}),
5852 }],
5853 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
5854 concurrency: Concurrency::ModuleManaged,
5855 emits_push: true,
5856 sub_supervises: true,
5857 }])
5858 .build()
5859 }
5860
5861 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
5862 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
5863 }
5864
5865 fn hello_frame_with_control_ops(
5866 module_id: &str,
5867 protocol_ver: u8,
5868 corr: u64,
5869 control_ops: Option<Vec<String>>,
5870 ) -> Frame {
5871 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
5872 }
5873
5874 fn hello_frame_with_nonce(
5875 module_id: &str,
5876 protocol_ver: u8,
5877 corr: u64,
5878 launch_nonce: Option<&str>,
5879 ) -> Frame {
5880 hello_frame_full(
5881 module_id,
5882 protocol_ver,
5883 corr,
5884 None,
5885 launch_nonce.map(ToOwned::to_owned),
5886 )
5887 }
5888
5889 fn hello_frame_full(
5890 module_id: &str,
5891 protocol_ver: u8,
5892 corr: u64,
5893 control_ops: Option<Vec<String>>,
5894 launch_nonce: Option<String>,
5895 ) -> Frame {
5896 let body = serde_json::to_vec(&ModuleHelloBody {
5897 manifest: manifest(module_id, protocol_ver),
5898 protocol_ver,
5899 control_ops,
5900 launch_nonce,
5901 })
5902 .unwrap();
5903 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
5904 }
5905
5906 fn non_routable_hello_frame_with_control_ops(
5907 module_id: &str,
5908 corr: u64,
5909 control_ops: Option<Vec<String>>,
5910 ) -> Frame {
5911 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
5912 manifest.provides.clear();
5913 let body = serde_json::to_vec(&ModuleHelloBody {
5914 manifest,
5915 protocol_ver: PROTOCOL_VERSION,
5916 control_ops,
5917 launch_nonce: None,
5918 })
5919 .unwrap();
5920 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
5921 }
5922
5923 fn capability_grammar_hello_frame(
5924 capabilities: Value,
5925 runtime_computed: Option<Value>,
5926 corr: u64,
5927 ) -> Frame {
5928 let mut body = serde_json::to_value(ModuleHelloBody {
5929 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
5930 protocol_ver: PROTOCOL_VERSION,
5931 control_ops: None,
5932 launch_nonce: None,
5933 })
5934 .expect("HELLO body serializes");
5935 body["manifest"]["capabilities"] = capabilities;
5936 if let Some(runtime_computed) = runtime_computed {
5937 body["runtime_computed"] = runtime_computed;
5938 }
5939 Frame::build(
5940 FrameType::Hello,
5941 control_flags(),
5942 0,
5943 0,
5944 corr,
5945 serde_json::to_vec(&body).expect("HELLO body reserializes"),
5946 )
5947 .expect("HELLO frame builds")
5948 }
5949
5950 fn channel_request(channel: u16, corr: u64) -> Frame {
5951 Frame::build(
5952 FrameType::Request,
5953 Flags::new(true, Priority::Interactive, false),
5954 channel,
5955 0,
5956 corr,
5957 b"opaque".to_vec(),
5958 )
5959 .unwrap()
5960 }
5961
5962 fn route_ctx(
5963 connection_id: ConnectionId,
5964 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
5965 let (tx, rx) = mpsc::channel(8);
5966 (
5967 RouteCtx {
5968 connection_id,
5969 egress: FrameSink::new(tx),
5970 },
5971 rx,
5972 )
5973 }
5974
5975 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
5976 serde_json::from_slice(&frame.body).unwrap()
5977 }
5978
5979 async fn hello_via_sink(
5984 handler: &ControlHandler,
5985 ctx: &RouteCtx,
5986 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
5987 hello: Frame,
5988 ) -> Frame {
5989 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
5990 assert!(
5991 replies.is_empty(),
5992 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
5993 );
5994 let ack = rx
5995 .try_recv()
5996 .expect("HELLO_ACK is queued on the module sink")
5997 .frame;
5998 assert_eq!(ack.header.ty, FrameType::HelloAck);
5999 ack
6000 }
6001
6002 fn parse_error(frame: &Frame) -> Value {
6003 serde_json::from_slice(&frame.body).unwrap()
6004 }
6005
6006 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
6007 serde_json::from_slice(&frame.body).unwrap()
6008 }
6009
6010 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
6011 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
6012 route_channel,
6013 route_epoch: 0,
6014 kind,
6015 })
6016 .unwrap();
6017 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6018 }
6019
6020 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
6021 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
6022 module_id: module_id.to_string(),
6023 })
6024 .unwrap();
6025 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6026 }
6027
6028 fn route_open_frame(corr: u64, module_id: &str, project_root: TestTempDir) -> Frame {
6029 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
6030 }
6031
6032 fn route_open_frame_with_consumer_capabilities(
6033 corr: u64,
6034 module_id: &str,
6035 project_root: TestTempDir,
6036 consumer_capabilities: Option<Vec<String>>,
6037 ) -> Frame {
6038 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6039 target: RouteTarget::ToolProvider {
6040 module_id: module_id.to_string(),
6041 },
6042 identity: BindIdentity::new(
6043 project_root.path().to_path_buf(),
6044 "unit".to_string(),
6045 "session".to_string(),
6046 ),
6047 consumer_identity: None,
6048 consumer_capabilities,
6049 admission_facts: None,
6050 })
6051 .unwrap();
6052 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6053 }
6054
6055 fn route_open_frame_with_admission_facts(
6056 corr: u64,
6057 module_id: &str,
6058 project_root: TestTempDir,
6059 consumer_identity: Option<subc_control::ConsumerIdentity>,
6060 facts: Option<Value>,
6061 ) -> Frame {
6062 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6063 target: RouteTarget::ToolProvider {
6064 module_id: module_id.to_string(),
6065 },
6066 identity: BindIdentity::new(
6067 project_root.path().to_path_buf(),
6068 "unit".to_string(),
6069 format!("session-{corr}"),
6070 ),
6071 consumer_identity,
6072 consumer_capabilities: None,
6073 admission_facts: facts,
6074 })
6075 .unwrap();
6076 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6077 }
6078
6079 #[derive(Clone, Default)]
6080 struct EventCapture {
6081 events: Arc<Mutex<Vec<CapturedEvent>>>,
6082 }
6083
6084 #[derive(Clone, Debug)]
6085 struct CapturedEvent {
6086 target: String,
6087 level: tracing::Level,
6088 fields: BTreeMap<String, String>,
6089 }
6090
6091 impl EventCapture {
6092 fn events(&self) -> Vec<CapturedEvent> {
6093 self.events.lock().unwrap().clone()
6094 }
6095 }
6096
6097 impl<S> Layer<S> for EventCapture
6098 where
6099 S: Subscriber,
6100 {
6101 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
6102 let mut visitor = EventFieldVisitor::default();
6103 event.record(&mut visitor);
6104 self.events.lock().unwrap().push(CapturedEvent {
6105 target: event.metadata().target().to_string(),
6106 level: *event.metadata().level(),
6107 fields: visitor.fields,
6108 });
6109 }
6110 }
6111
6112 #[derive(Default)]
6113 struct EventFieldVisitor {
6114 fields: BTreeMap<String, String>,
6115 }
6116
6117 impl Visit for EventFieldVisitor {
6118 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
6119 self.fields
6120 .insert(field.name().to_string(), format!("{value:?}"));
6121 }
6122 }
6123
6124 fn health_response(corr: u64, status: HealthStatus) -> Frame {
6125 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
6126 status,
6127 detail: Some("warming".to_string()),
6128 metrics: Some(json!({"queue_depth": 3})),
6129 })
6130 .unwrap();
6131 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
6132 }
6133
6134 fn route_bind_ack(corr: u64) -> Frame {
6135 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
6136 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
6137 }
6138
6139 fn unique_project_root(label: &str) -> TestTempDir {
6140 TestTempDir::new(label)
6141 }
6142
6143 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
6144 match parse_route_poll(frame) {
6145 ClientControlResponse::RoutePoll {
6146 status: None,
6147 live: Some(live),
6148 ..
6149 } => assert_eq!(live, expected_live),
6150 other => panic!("unexpected route.poll response: {other:?}"),
6151 }
6152 }
6153
6154 fn bind_liveness_route(
6155 registry: &Registry,
6156 forwarding: &ForwardingTable,
6157 module_id: &str,
6158 ) -> (RouteCtx, u16, u32) {
6159 let module_connection = ConnectionId::new(101);
6160 let client_connection = ConnectionId::new(202);
6161 let registration = registry
6162 .register_with_control_ops(
6163 manifest(module_id, PROTOCOL_VERSION),
6164 PROTOCOL_VERSION,
6165 module_connection,
6166 module_baseline_control_ops(),
6167 )
6168 .unwrap();
6169 let (module_tx, _module_rx) = mpsc::channel(8);
6170 let endpoint = forwarding
6171 .register_module_connection(
6172 module_connection,
6173 module_id.to_string(),
6174 PROTOCOL_VERSION,
6175 manifest_concurrency(®istration.manifest),
6176 FrameSink::new(module_tx),
6177 )
6178 .unwrap();
6179 let (client_ctx, _client_rx) = route_ctx(client_connection);
6180 let pending = forwarding
6181 .begin_route_bind_relay_for_test(
6182 client_connection,
6183 client_ctx.egress.clone(),
6184 1,
6185 module_id,
6186 )
6187 .unwrap();
6188 assert_eq!(pending.endpoint, endpoint);
6189 let route_channel = pending.client_channel;
6190 let route_epoch = pending.client_epoch;
6191 forwarding
6192 .complete_pending_relay(
6193 module_connection,
6194 pending.corr,
6195 RouteBindRelayOutcome::Accepted,
6196 )
6197 .unwrap();
6198 (client_ctx, route_channel, route_epoch)
6199 }
6200
6201 struct FakeProcessLiveness {
6202 live: Option<bool>,
6203 }
6204
6205 impl ModuleProcessLiveness for FakeProcessLiveness {
6206 fn process_live(&self, _module_id: &str) -> Option<bool> {
6207 self.live
6208 }
6209 }
6210
6211 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6212 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
6213 {
6214 let registry = Arc::new(Registry::default());
6215 let supervisor_handle = SupervisorHandle::new();
6216 let supervisor = Supervisor::new(
6217 Arc::clone(®istry),
6218 RestartPolicy::new(1, Duration::from_millis(10)),
6219 )
6220 .with_handle(supervisor_handle.clone());
6221 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
6222 let module = supervisor
6223 .spawn(ModuleSpec {
6224 module_id: "stderr-tail-wire".to_string(),
6225 program: fake_aft_stub_path(),
6226 args: Vec::new(),
6227 env: vec![
6228 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
6229 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
6230 ],
6231 reserved: false,
6232 reserved_prefixes: Vec::new(),
6233 protocol: ModuleProtocol::Subc,
6234 overlap: Default::default(),
6235 })
6236 .unwrap();
6237
6238 let deadline = Instant::now() + Duration::from_secs(5);
6239 loop {
6240 let tail = module.stderr_tail(None, None);
6241 if tail
6242 .entries
6243 .iter()
6244 .any(|entry| matches!(entry, TailEntry::ProcessStart))
6245 && tail.entries.iter().any(|entry| {
6246 matches!(
6247 entry,
6248 TailEntry::Line {
6249 truncated: true,
6250 ..
6251 }
6252 )
6253 })
6254 {
6255 break;
6256 }
6257 assert!(
6258 Instant::now() < deadline,
6259 "module did not produce a truncated line and restart boundary: {tail:?}"
6260 );
6261 sleep(Duration::from_millis(10)).await;
6262 }
6263
6264 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
6265 let request = ClientControlRequest::SupervisorStderrTail {
6266 module_id: "stderr-tail-wire".to_string(),
6267 max_lines: None,
6268 max_bytes: None,
6269 };
6270 let frame = Frame::build(
6271 FrameType::Request,
6272 control_flags(),
6273 0,
6274 0,
6275 1,
6276 serde_json::to_vec(&request).unwrap(),
6277 )
6278 .unwrap();
6279 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6280 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
6281 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
6282 serde_json::from_slice(&responses[0].body).unwrap()
6283 else {
6284 panic!("expected supervisor.stderr_tail response");
6285 };
6286
6287 assert!(
6288 tail.entries
6289 .iter()
6290 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
6291 "the control response lost the restart boundary"
6292 );
6293 let Some(StderrTailEntry::Line { text, truncated }) = tail.entries.iter().find(|entry| {
6294 matches!(
6295 entry,
6296 StderrTailEntry::Line {
6297 truncated: true,
6298 ..
6299 }
6300 )
6301 }) else {
6302 panic!("the control response lost the truncated line");
6303 };
6304 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
6305 assert!(*truncated);
6306 }
6307
6308 #[tokio::test(flavor = "current_thread")]
6313 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
6314 let dir = TestTempDir::new("terminals-off-worker");
6315 let journal_path = dir.join("terminals.jsonl");
6316 let registry = Arc::new(Registry::default());
6317 let supervisor_handle = SupervisorHandle::new();
6318 let supervisor =
6319 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
6320 .with_handle(supervisor_handle.clone())
6321 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
6322 let module = supervisor
6323 .spawn(ModuleSpec {
6324 module_id: "terminal-off-worker".to_string(),
6325 program: fake_aft_stub_path(),
6326 args: Vec::new(),
6327 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
6328 reserved: false,
6329 reserved_prefixes: Vec::new(),
6330 protocol: ModuleProtocol::Subc,
6331 overlap: Default::default(),
6332 })
6333 .unwrap();
6334 let deadline = Instant::now() + Duration::from_secs(5);
6335 while module.terminal_history().entries.len() != 2 {
6336 assert!(Instant::now() < deadline, "module did not record two exits");
6337 sleep(Duration::from_millis(10)).await;
6338 }
6339
6340 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
6341 let handler =
6342 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
6343 let frame = Frame::build(
6344 FrameType::Request,
6345 control_flags(),
6346 0,
6347 0,
6348 1,
6349 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
6350 module_id: "terminal-off-worker".to_string(),
6351 })
6352 .unwrap(),
6353 )
6354 .unwrap();
6355 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6356 let spawned_at = std::time::Instant::now();
6357 let read = tokio::spawn({
6358 let handler = Arc::clone(&handler);
6359 async move { handler.handle_control_frame(&ctx, frame).await }
6360 });
6361 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
6364 .await
6365 .unwrap()
6366 .expect("the history read reached its pause");
6367 let elapsed = spawned_at.elapsed();
6368 assert!(
6369 elapsed < Duration::from_secs(2) && !read.is_finished(),
6370 "this task could not run while the history read was paused \
6371 (resumed after {elapsed:?}, read finished: {})",
6372 read.is_finished()
6373 );
6374
6375 drop(release);
6376 let responses = read.await.unwrap().unwrap();
6377 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
6378 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
6379 panic!("expected supervisor.terminals response");
6380 };
6381 assert_eq!(terminals.entries.len(), 2);
6382 assert_eq!(terminals.journal_skipped_lines, 0);
6383 assert_eq!(terminals.journal_read_errors, 0);
6384 }
6385
6386 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6387 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
6388 let registry = Arc::new(Registry::default());
6389 let supervisor_handle = SupervisorHandle::new();
6390 let supervisor =
6391 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
6392 .with_handle(supervisor_handle.clone());
6393 let module = supervisor
6394 .spawn(ModuleSpec {
6395 module_id: "terminal-golden".to_string(),
6396 program: fake_aft_stub_path(),
6397 args: Vec::new(),
6398 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
6399 reserved: false,
6400 reserved_prefixes: Vec::new(),
6401 protocol: ModuleProtocol::Subc,
6402 overlap: Default::default(),
6403 })
6404 .unwrap();
6405
6406 let deadline = Instant::now() + Duration::from_secs(5);
6407 while module.terminal_history().entries.len() != 2 {
6408 assert!(
6409 Instant::now() < deadline,
6410 "module did not retain two terminal exits: {:?}",
6411 module.terminal_history()
6412 );
6413 sleep(Duration::from_millis(10)).await;
6414 }
6415
6416 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
6417 let request = ClientControlRequest::SupervisorTerminals {
6418 module_id: "terminal-golden".to_string(),
6419 };
6420 let frame = Frame::build(
6421 FrameType::Request,
6422 control_flags(),
6423 0,
6424 0,
6425 1,
6426 serde_json::to_vec(&request).unwrap(),
6427 )
6428 .unwrap();
6429 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6430 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
6431 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
6432 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
6433 panic!("expected supervisor.terminals response");
6434 };
6435 assert_eq!(terminals.entries.len(), 2);
6436 assert_eq!(terminals.dropped, 0);
6437
6438 let mut rendered = serde_json::to_value(response).unwrap();
6439 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
6442 for (index, entry) in rendered["entries"]
6443 .as_array_mut()
6444 .expect("terminal response entries array")
6445 .iter_mut()
6446 .enumerate()
6447 {
6448 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
6449 }
6450
6451 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
6452 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
6453 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
6454 if std::env::var_os("UPDATE_GOLDEN").is_some() {
6455 std::fs::write(&golden_path, &serialized).unwrap();
6456 }
6457 let expected: Value =
6458 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
6459 assert_eq!(rendered, expected);
6460 }
6461
6462 #[test]
6463 fn hello_registers_manifest_and_returns_ack() {
6464 let registry = Arc::new(Registry::default());
6465 let handler = ControlHandler::new(Arc::clone(®istry));
6466 let conn = ConnectionId::new(1);
6467
6468 let responses = handler
6469 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
6470 .unwrap();
6471
6472 assert_eq!(responses.len(), 1);
6473 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
6474 assert_eq!(responses[0].header.channel, 0);
6475 assert_eq!(responses[0].header.corr, 7);
6476 let ack = parse_ack(&responses[0]);
6477 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
6478 assert!(ack
6479 .subc_capabilities
6480 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
6481 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
6482 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
6483 assert!(ack
6484 .subc_ops
6485 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
6486 assert!(ack
6487 .subc_ops
6488 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
6489
6490 let registration = registry.get_module("aft").unwrap().unwrap();
6491 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
6492 assert_eq!(registration.state, ChannelState::Active);
6493 assert_eq!(registration.connection_id, conn);
6494 assert_eq!(registration.control_ops, module_baseline_control_ops());
6495 }
6496
6497 #[test]
6498 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
6499 let invalid_identifiers = [
6500 ("case_change", "credentials-Provider/v1"),
6501 ("leading_zero", "credentials-provider/v01"),
6502 ("trailing_hyphen", "credentials-provider-/v1"),
6503 ("consecutive_hyphens", "credentials--provider/v1"),
6504 ("uppercase", "Credentials-provider/v1"),
6505 ("missing_v", "credentials-provider/1"),
6506 ("whitespace", "credentials provider/v1"),
6507 ("zero_version", "credentials-provider/v0"),
6508 ("out_of_range_version", "credentials-provider/v4294967296"),
6509 (
6510 "overlength_name",
6511 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
6512 ),
6513 ];
6514 let mut cases = invalid_identifiers
6515 .into_iter()
6516 .map(|(name, identifier)| {
6517 (
6518 format!("identifier_{name}"),
6519 "capabilities.provides[0]".to_string(),
6520 identifier.to_string(),
6521 json!({ "provides": [identifier] }),
6522 None,
6523 )
6524 })
6525 .collect::<Vec<_>>();
6526 cases.extend([
6527 (
6528 "unknown_need".to_string(),
6529 "capabilities.requires[0].need".to_string(),
6530 "deferred".to_string(),
6531 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
6532 None,
6533 ),
6534 (
6535 "duplicate_provides".to_string(),
6536 "capabilities.provides[1]".to_string(),
6537 "credentials-provider/v1".to_string(),
6538 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
6539 None,
6540 ),
6541 (
6542 "duplicate_must_never_reach".to_string(),
6543 "capabilities.must_never_reach[1]".to_string(),
6544 "credentials-provider/v1".to_string(),
6545 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
6546 None,
6547 ),
6548 (
6549 "duplicate_requires_same_need".to_string(),
6550 "capabilities.requires[1]".to_string(),
6551 "credentials-provider/v1".to_string(),
6552 json!({ "requires": [
6553 { "capability": "credentials-provider/v1", "need": "required" },
6554 { "capability": "credentials-provider/v1", "need": "required" }
6555 ] }),
6556 None,
6557 ),
6558 (
6559 "duplicate_requires_conflicting_need".to_string(),
6560 "capabilities.requires[1]".to_string(),
6561 "credentials-provider/v1".to_string(),
6562 json!({ "requires": [
6563 { "capability": "credentials-provider/v1", "need": "required" },
6564 { "capability": "credentials-provider/v1", "need": "optional" }
6565 ] }),
6566 None,
6567 ),
6568 (
6569 "capabilities_root_pointer".to_string(),
6570 "runtime_computed[0]".to_string(),
6571 "/capabilities".to_string(),
6572 json!({}),
6573 Some(json!(["/capabilities"])),
6574 ),
6575 (
6576 "capabilities_descendant_pointer".to_string(),
6577 "runtime_computed[0]".to_string(),
6578 "/capabilities/provides".to_string(),
6579 json!({}),
6580 Some(json!(["/capabilities/provides"])),
6581 ),
6582 (
6583 "malformed_pointer_without_leading_slash".to_string(),
6584 "runtime_computed[0]".to_string(),
6585 "capabilities".to_string(),
6586 json!({}),
6587 Some(json!(["capabilities"])),
6588 ),
6589 (
6590 "malformed_pointer_escape".to_string(),
6591 "runtime_computed[0]".to_string(),
6592 "/roles/~2/tools".to_string(),
6593 json!({}),
6594 Some(json!(["/roles/~2/tools"])),
6595 ),
6596 (
6597 "unknown_capabilities_field".to_string(),
6598 "capabilities.future".to_string(),
6599 "<array>".to_string(),
6600 json!({ "future": [] }),
6601 None,
6602 ),
6603 ]);
6604
6605 for (index, (name, field, value, capabilities, runtime_computed)) in
6606 cases.into_iter().enumerate()
6607 {
6608 let registry = Arc::new(Registry::default());
6609 let handler = ControlHandler::new(Arc::clone(®istry));
6610 let response = handler
6611 .handle_control(
6612 ConnectionId::new((index + 1) as u64),
6613 capability_grammar_hello_frame(
6614 capabilities,
6615 runtime_computed,
6616 index as u64 + 1,
6617 ),
6618 )
6619 .expect("invalid HELLO returns a refusal");
6620
6621 assert_eq!(response.len(), 1, "{name} must emit one refusal");
6622 let error = parse_error(&response[0]);
6623 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
6624 let message = error["message"]
6625 .as_str()
6626 .expect("error message is a string");
6627 assert!(
6628 message.contains(&field),
6629 "{name}: field missing from {message}"
6630 );
6631 assert!(
6632 message.contains(&value),
6633 "{name}: value missing from {message}"
6634 );
6635 assert_eq!(
6636 registry
6637 .active_registration_count()
6638 .expect("registry reads"),
6639 0,
6640 "{name}: refused HELLO must not create a catalog entry"
6641 );
6642 }
6643 }
6644
6645 #[test]
6646 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
6647 let registry = Arc::new(Registry::default());
6648 let handler = ControlHandler::new(Arc::clone(®istry));
6649 let capabilities = json!({
6650 "provides": ["credentials-provider/v1"],
6651 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
6652 "must_never_reach": ["federation-transport/v1"]
6653 });
6654 let response = handler
6655 .handle_control(
6656 ConnectionId::new(99),
6657 capability_grammar_hello_frame(
6658 capabilities.clone(),
6659 Some(json!(["/roles/0/tools"])),
6660 99,
6661 ),
6662 )
6663 .expect("valid HELLO registers");
6664 assert_eq!(response[0].header.ty, FrameType::HelloAck);
6665
6666 let request = Frame::build(
6667 FrameType::Request,
6668 control_flags(),
6669 0,
6670 0,
6671 100,
6672 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
6673 .expect("catalog request serializes"),
6674 )
6675 .expect("catalog request frame builds");
6676 let response = handler
6677 .handle_catalog_list(request, None)
6678 .expect("catalog list succeeds");
6679 let ClientControlResponse::CatalogList { modules, .. } =
6680 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
6681 else {
6682 panic!("catalog request must return catalog.list");
6683 };
6684 assert_eq!(modules.len(), 1);
6685 assert_eq!(
6686 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
6687 capabilities
6688 );
6689 }
6690
6691 #[test]
6692 fn catalog_list_mirrors_management_operation_description() {
6693 let registry = Arc::new(Registry::default());
6694 let handler = ControlHandler::new(Arc::clone(®istry));
6695 let description = "List managed records and return their identifiers and metadata.";
6696 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
6697 manifest.provides = vec![ProviderRole::ManagementSurface {
6698 operations: vec![ManagementOperation {
6699 name: "records.list".to_string(),
6700 kind: ManagementOperationKind::Query,
6701 description: Some(description.to_string()),
6702 }],
6703 config_schema: json!({"type": "object"}),
6704 observability: vec![ObservabilitySurface {
6705 name: "records.stats".to_string(),
6706 kind: ObservabilityKind::Snapshot,
6707 }],
6708 identity_scope: vec![IdentityScope::Project],
6709 concurrency: Concurrency::ModuleManaged,
6710 }];
6711 registry
6712 .register_with_control_ops(
6713 manifest,
6714 PROTOCOL_VERSION,
6715 ConnectionId::new(99),
6716 Vec::new(),
6717 )
6718 .expect("described management manifest registers");
6719
6720 let request = Frame::build(
6721 FrameType::Request,
6722 control_flags(),
6723 0,
6724 0,
6725 100,
6726 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
6727 .expect("catalog request serializes"),
6728 )
6729 .expect("catalog request frame builds");
6730 let response = handler
6731 .handle_catalog_list(request, None)
6732 .expect("catalog list succeeds");
6733 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
6734 assert_eq!(
6735 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
6736 "catalog.list must preserve the declared operation description verbatim"
6737 );
6738 }
6739
6740 #[test]
6741 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
6742 let registry = Arc::new(Registry::default());
6743 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
6744 [("vault".to_string(), true), ("squatter".to_string(), true)],
6745 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
6746 );
6747 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
6748 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
6749 provides: vec!["credentials-provider/v1".to_string()],
6750 requires: Vec::new(),
6751 must_never_reach: Vec::new(),
6752 });
6753 let frame = Frame::build(
6754 FrameType::Hello,
6755 control_flags(),
6756 0,
6757 0,
6758 77,
6759 serde_json::to_vec(&ModuleHelloBody {
6760 manifest: squatter,
6761 protocol_ver: PROTOCOL_VERSION,
6762 control_ops: None,
6763 launch_nonce: None,
6764 })
6765 .expect("HELLO serializes"),
6766 )
6767 .expect("HELLO frame builds");
6768 let response = handler
6769 .handle_control(ConnectionId::new(77), frame)
6770 .expect("reserved claim receives a typed refusal");
6771 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
6772 assert_eq!(
6773 registry
6774 .active_registration_count()
6775 .expect("registry reads"),
6776 0,
6777 "a reserved capability refusal must not leave a catalog entry"
6778 );
6779 }
6780
6781 #[test]
6782 fn server_describe_surfaces_required_capability_verdict_fields() {
6783 let registry = Arc::new(Registry::default());
6784 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
6785 [
6786 ("consumer".to_string(), true),
6787 ("provider".to_string(), false),
6788 ],
6789 BTreeMap::new(),
6790 );
6791 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
6792 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
6793 provides: Vec::new(),
6794 requires: vec![subc_protocol::manifest::CapabilityRequirement {
6795 capability: "credentials-provider/v1".to_string(),
6796 need: subc_protocol::manifest::CapabilityNeed::Required,
6797 }],
6798 must_never_reach: Vec::new(),
6799 });
6800 let hello = Frame::build(
6801 FrameType::Hello,
6802 control_flags(),
6803 0,
6804 0,
6805 78,
6806 serde_json::to_vec(&ModuleHelloBody {
6807 manifest: consumer,
6808 protocol_ver: PROTOCOL_VERSION,
6809 control_ops: None,
6810 launch_nonce: None,
6811 })
6812 .expect("HELLO serializes"),
6813 )
6814 .expect("HELLO frame builds");
6815 handler
6816 .handle_control(ConnectionId::new(78), hello)
6817 .expect("consumer registers");
6818 let describe = Frame::build(
6819 FrameType::Request,
6820 control_flags(),
6821 0,
6822 0,
6823 79,
6824 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
6825 .expect("request serializes"),
6826 )
6827 .expect("describe frame builds");
6828 let response = handler
6829 .handle_server_describe(describe)
6830 .expect("server.describe succeeds");
6831 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
6832 let requirement = &rendered["capability_requirements"][0];
6833 assert_eq!(requirement["consumer"], "consumer");
6834 assert_eq!(requirement["verdict"], "never_provided");
6835 assert_eq!(requirement["episode_seq"], 1);
6836 assert_eq!(requirement["config_satisfiable"], false);
6837 assert_eq!(requirement["runtime_available"], false);
6838 assert!(requirement["detail"]
6839 .as_str()
6840 .expect("detail string")
6841 .contains("credentials-provider/v1"));
6842 }
6843
6844 #[test]
6845 fn catalog_list_omits_capabilities_for_legacy_manifest() {
6846 let registry = Arc::new(Registry::default());
6847 let handler = ControlHandler::new(Arc::clone(®istry));
6848 let hello = handler
6849 .handle_control(
6850 ConnectionId::new(101),
6851 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
6852 )
6853 .expect("legacy HELLO registers");
6854 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
6855
6856 let request = Frame::build(
6857 FrameType::Request,
6858 control_flags(),
6859 0,
6860 0,
6861 102,
6862 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
6863 .expect("catalog request serializes"),
6864 )
6865 .expect("catalog request frame builds");
6866 let response = handler
6867 .handle_catalog_list(request, None)
6868 .expect("catalog list succeeds");
6869 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
6870 assert!(
6871 body["modules"][0].get("capabilities").is_none(),
6872 "legacy manifest must retain an absent capabilities field on catalog.list"
6873 );
6874 }
6875
6876 #[test]
6877 fn hello_ack_omits_storage_when_no_storage_config() {
6878 let registry = Arc::new(Registry::default());
6879 let handler = ControlHandler::new(Arc::clone(®istry));
6880 let responses = handler
6881 .handle_control(
6882 ConnectionId::new(1),
6883 hello_frame("aft", PROTOCOL_VERSION, 7),
6884 )
6885 .unwrap();
6886 let ack = parse_ack(&responses[0]);
6887 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
6888 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
6889 }
6890
6891 #[tokio::test]
6892 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
6893 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
6894 let registry = Arc::new(Registry::default());
6895 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
6896 let responses = handler
6897 .handle_control(
6898 ConnectionId::new(1),
6899 hello_frame("aft", PROTOCOL_VERSION, 7),
6900 )
6901 .unwrap();
6902 let ack = parse_ack(&responses[0]);
6903 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
6904
6905 let described = handler
6906 .handle_control_frame(
6907 &route_ctx(ConnectionId::new(2)).0,
6908 Frame::build(
6909 FrameType::Request,
6910 control_flags(),
6911 0,
6912 0,
6913 9,
6914 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
6915 )
6916 .unwrap(),
6917 )
6918 .await
6919 .unwrap();
6920 let ClientControlResponse::ServerDescribe { machine_id, .. } =
6921 serde_json::from_slice(&described[0].body).unwrap()
6922 else {
6923 panic!("server.describe answered with another shape");
6924 };
6925 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
6926 }
6927
6928 #[test]
6929 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
6930 let registry = Arc::new(Registry::default());
6933 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
6934 crate::daemon_config::StorageConfig::Sqlite {
6935 data_home: std::path::PathBuf::from("/data"),
6936 },
6937 ));
6938
6939 let responses = handler
6940 .handle_control(
6941 ConnectionId::new(1),
6942 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
6943 )
6944 .unwrap();
6945 let ack = parse_ack(&responses[0]);
6946 assert_eq!(
6947 ack.storage,
6948 Some(serde_json::json!({
6949 "module_id": "alfonso-routing",
6950 "storage_namespace": "default",
6951 "isolation": { "kind": "module" },
6952 "backend": {
6953 "backend": "sqlite",
6954 "path": "/data/cortexkit/alfonso-routing/store.db"
6955 }
6956 })),
6957 "the delivered descriptor is the module's own sqlite store path"
6958 );
6959 }
6960
6961 #[test]
6962 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
6963 let registry = Arc::new(Registry::default());
6964 let handler = ControlHandler::new(Arc::clone(®istry));
6965 let conn = ConnectionId::new(1);
6966 let responses = handler
6967 .handle_control(
6968 conn,
6969 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
6970 )
6971 .unwrap();
6972 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
6973 let registration = registry.get_module("aft").unwrap().unwrap();
6974 assert_eq!(registration.control_ops, module_baseline_control_ops());
6975
6976 let frame =
6977 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
6978 assert!(handler
6979 .guard_module_control_op(&frame, "aft", "route.bind")
6980 .unwrap()
6981 .is_none());
6982 let error = handler
6983 .guard_module_control_op(&frame, "aft", "test.synthetic")
6984 .unwrap()
6985 .expect("synthetic ungranted op should be rejected");
6986 assert_eq!(error.header.ty, FrameType::Error);
6987 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
6988 }
6989
6990 #[test]
6991 fn hello_control_ops_some_adds_optional_grants() {
6992 let registry = Arc::new(Registry::default());
6993 let handler = ControlHandler::new(Arc::clone(®istry));
6994 handler
6995 .handle_control(
6996 ConnectionId::new(1),
6997 hello_frame_with_control_ops(
6998 "aft",
6999 PROTOCOL_VERSION,
7000 7,
7001 Some(vec![
7002 "future.synthetic".to_string(),
7003 "route.bind".to_string(),
7004 ]),
7005 ),
7006 )
7007 .unwrap();
7008 let registration = registry.get_module("aft").unwrap().unwrap();
7009 assert_eq!(
7010 registration.control_ops,
7011 vec![
7012 "route.bind".to_string(),
7013 "route.status".to_string(),
7014 "future.synthetic".to_string(),
7015 ]
7016 );
7017 let frame =
7018 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
7019 assert!(handler
7020 .guard_module_control_op(&frame, "aft", "future.synthetic")
7021 .unwrap()
7022 .is_none());
7023 }
7024
7025 #[tokio::test]
7026 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
7027 let registry = Arc::new(Registry::default());
7028 let forwarding = Arc::new(ForwardingTable::default());
7029 let handler =
7030 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
7031 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
7032 hello_via_sink(
7033 &handler,
7034 &module_ctx,
7035 &mut module_rx,
7036 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
7037 )
7038 .await;
7039
7040 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
7041 let responses = handler
7042 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
7043 .await
7044 .unwrap();
7045 assert_eq!(responses.len(), 1);
7046 assert_eq!(responses[0].header.ty, FrameType::Error);
7047 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
7048 assert!(module_rx.try_recv().is_err());
7049 }
7050
7051 #[tokio::test]
7052 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
7053 let registry = Arc::new(Registry::default());
7054 let forwarding = Arc::new(ForwardingTable::default());
7055 let handler =
7056 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
7057 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
7058 hello_via_sink(
7059 &handler,
7060 &module_ctx,
7061 &mut module_rx,
7062 hello_frame_with_control_ops(
7063 "aft",
7064 PROTOCOL_VERSION,
7065 7,
7066 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
7067 ),
7068 )
7069 .await;
7070
7071 let project_root = unique_project_root("demux");
7072 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
7073 let route_handler = handler.clone();
7074 let route_task = tokio::spawn(async move {
7075 route_handler
7076 .handle_control_frame(
7077 &route_client_ctx,
7078 route_open_frame(100, "aft", project_root),
7079 )
7080 .await
7081 .unwrap()
7082 });
7083 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7084 .await
7085 .unwrap()
7086 .unwrap();
7087 assert!(matches!(
7088 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
7089 ModuleControlRequest::RouteBind { .. }
7090 ));
7091
7092 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
7093 let health_handler = handler.clone();
7094 let health_task = tokio::spawn(async move {
7095 health_handler
7096 .handle_control_frame(
7097 &health_client_ctx,
7098 supervisor_health_probe_frame(101, "aft"),
7099 )
7100 .await
7101 .unwrap()
7102 });
7103 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7104 .await
7105 .unwrap()
7106 .unwrap();
7107 assert_eq!(
7108 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
7109 ModuleControlRequest::HealthCheck {}
7110 );
7111
7112 handler
7113 .handle_control_frame(
7114 &module_ctx,
7115 health_response(health_frame.header.corr, HealthStatus::Degraded),
7116 )
7117 .await
7118 .unwrap();
7119 let health_response = health_task.await.unwrap();
7120 assert_eq!(health_response.len(), 1);
7121 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
7122 ClientControlResponse::SupervisorHealthProbe {
7123 module_id,
7124 status,
7125 detail,
7126 metrics,
7127 } => {
7128 assert_eq!(module_id, "aft");
7129 assert_eq!(status, HealthStatus::Degraded);
7130 assert_eq!(detail.as_deref(), Some("warming"));
7131 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
7132 }
7133 other => panic!("unexpected health response: {other:?}"),
7134 }
7135
7136 handler
7137 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
7138 .await
7139 .unwrap();
7140 let route_response = route_task.await.unwrap();
7141 assert!(route_response.is_empty());
7142 let published = route_client_rx.recv().await.unwrap();
7143 assert!(matches!(
7144 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
7145 ClientControlResponse::RouteOpen { .. }
7146 ));
7147 }
7148
7149 async fn relay_route_open(
7154 handler: &ControlHandler,
7155 client_connection: ConnectionId,
7156 client_egress: &FrameSink,
7157 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
7158 corr: u64,
7159 module_id: &str,
7160 project_root_label: &str,
7161 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
7162 let ctx = RouteCtx {
7163 connection_id: client_connection,
7164 egress: client_egress.clone(),
7165 };
7166 let handler = handler.clone();
7167 let project_root = unique_project_root(project_root_label);
7168 let module_id = module_id.to_string();
7169 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
7170 let task = tokio::spawn(async move {
7171 let _guard = tracing::dispatcher::set_default(&dispatch);
7172 handler
7173 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
7174 .await
7175 .unwrap()
7176 });
7177 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
7178 .await
7179 .expect("module receives the relayed route.bind")
7180 .expect("module egress is open");
7181 (task, bind.frame)
7182 }
7183
7184 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
7185 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
7186 ModuleControlRequest::RouteBind {
7187 route_channel,
7188 epoch,
7189 ..
7190 } => (route_channel, epoch),
7191 other => panic!("expected a route.bind request, got {other:?}"),
7192 }
7193 }
7194
7195 fn published_route(frame: &Frame) -> (u16, u32) {
7196 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
7197 ClientControlResponse::RouteOpen {
7198 route_channel,
7199 route_epoch,
7200 } => (route_channel, route_epoch),
7201 other => panic!("expected a route.open response, got {other:?}"),
7202 }
7203 }
7204
7205 #[tokio::test]
7225 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
7226 let registry = Arc::new(Registry::default());
7227 let forwarding = Arc::new(ForwardingTable::default());
7228 let handler =
7229 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
7230
7231 let module_connection = ConnectionId::new(30);
7232 let (module_ctx, mut module_rx) = route_ctx(module_connection);
7233 hello_via_sink(
7234 &handler,
7235 &module_ctx,
7236 &mut module_rx,
7237 hello_frame("aft", PROTOCOL_VERSION, 7),
7238 )
7239 .await;
7240
7241 let dying_client = ConnectionId::new(31);
7242 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
7243
7244 let (first_task, first_bind) = relay_route_open(
7247 &handler,
7248 dying_client,
7249 &dying_ctx.egress,
7250 &mut module_rx,
7251 100,
7252 "aft",
7253 "closing-first",
7254 )
7255 .await;
7256 handler
7257 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
7258 .await
7259 .unwrap();
7260 assert!(first_task.await.unwrap().is_empty());
7261 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
7262
7263 let (second_task, second_bind) = relay_route_open(
7265 &handler,
7266 dying_client,
7267 &dying_ctx.egress,
7268 &mut module_rx,
7269 101,
7270 "aft",
7271 "closing-second",
7272 )
7273 .await;
7274 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
7275
7276 assert!(forwarding
7279 .escalate_client_delivery_failure(
7280 dying_client,
7281 first_channel,
7282 first_epoch,
7283 CloseReason::new(
7284 "module_to_client_delivery_failed",
7285 "client egress refused a module frame",
7286 ),
7287 crate::forwarding::UndeliveredFrame {
7288 module_id: None,
7289 sink: &dying_ctx.egress,
7290 },
7291 )
7292 .unwrap());
7293 assert!(!dying_ctx.egress.is_closed());
7294
7295 let ack = handler
7297 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
7298 .await;
7299 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
7300 if module_loop_error.is_some() {
7301 handler.cleanup_connection(module_connection).unwrap();
7305 }
7306 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7310 .await
7311 .ok()
7312 .flatten();
7313
7314 assert!(
7316 registry
7317 .get_module_by_connection(module_connection)
7318 .unwrap()
7319 .is_some(),
7320 "one client's closing connection ended the shared module connection: \
7321 {module_loop_error:?}"
7322 );
7323 let cotenant = ConnectionId::new(32);
7325 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
7326 let (cotenant_task, cotenant_bind) = relay_route_open(
7327 &handler,
7328 cotenant,
7329 &cotenant_ctx.egress,
7330 &mut module_rx,
7331 102,
7332 "aft",
7333 "closing-cotenant",
7334 )
7335 .await;
7336 handler
7337 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
7338 .await
7339 .unwrap();
7340 assert!(cotenant_task.await.unwrap().is_empty());
7341 let (cotenant_channel, cotenant_epoch) =
7342 published_route(&cotenant_rx.recv().await.unwrap());
7343 assert!(matches!(
7344 forwarding
7345 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
7346 .unwrap(),
7347 DataRoute::Client(DataRouteState::Bound(_))
7348 ));
7349
7350 let goodbye = post_ack_module_frame
7353 .expect("module receives a GOODBYE for the abandoned route channel");
7354 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
7355 assert_eq!(goodbye.header.channel, abandoned_channel);
7356 assert_eq!(goodbye.header.epoch, abandoned_epoch);
7357
7358 assert!(dying_rx.try_recv().is_err());
7362 let second_response = second_task.await.unwrap();
7363 assert_eq!(second_response.len(), 1);
7364 assert_eq!(
7365 parse_error(&second_response[0])["code"],
7366 "target_unavailable"
7367 );
7368 }
7369
7370 #[test]
7377 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
7378 let handler = ControlHandler::default();
7379 let module_connection = ConnectionId::new(30);
7380 let client_connection = ConnectionId::new(31);
7381
7382 handler
7383 .refuse_to_end_module_connection_for_a_client(
7384 module_connection,
7385 77,
7386 ForwardingError::ConnectionClosing {
7387 connection_id: client_connection,
7388 },
7389 )
7390 .expect("a closing client must never end the module connection");
7391
7392 assert!(matches!(
7393 handler.refuse_to_end_module_connection_for_a_client(
7394 module_connection,
7395 78,
7396 ForwardingError::ConnectionClosing {
7397 connection_id: module_connection,
7398 },
7399 ),
7400 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
7401 connection_id
7402 })) if connection_id == module_connection
7403 ));
7404 assert!(matches!(
7405 handler.refuse_to_end_module_connection_for_a_client(
7406 module_connection,
7407 79,
7408 ForwardingError::Poisoned,
7409 ),
7410 Err(RouterError::Forwarding(ForwardingError::Poisoned))
7411 ));
7412 assert!(matches!(
7413 handler.refuse_to_end_module_connection_for_a_client(
7414 module_connection,
7415 80,
7416 ForwardingError::StaleModuleEndpoint,
7417 ),
7418 Err(RouterError::Forwarding(
7419 ForwardingError::StaleModuleEndpoint
7420 ))
7421 ));
7422 }
7423
7424 #[tokio::test]
7437 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
7438 let registry = Arc::new(Registry::default());
7439 let forwarding = Arc::new(ForwardingTable::default());
7440 let supervisor = SupervisorHandle::new();
7441 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7442 let handler =
7443 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
7444 .with_supervisor(supervisor);
7445
7446 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
7447 hello_via_sink(
7448 &handler,
7449 &target_ctx,
7450 &mut target_rx,
7451 hello_frame("target", PROTOCOL_VERSION, 1),
7452 )
7453 .await;
7454
7455 let wrong_nonce = handler
7459 .handle_control_frame(
7460 &route_ctx(ConnectionId::new(91)).0,
7461 route_open_frame_with_admission_facts(
7462 20,
7463 "target",
7464 unique_project_root("admission-facts"),
7465 Some(subc_control::ConsumerIdentity {
7466 module_id: "fed".to_string(),
7467 launch_nonce: "not-the-real-nonce".to_string(),
7468 }),
7469 None,
7470 ),
7471 )
7472 .await
7473 .unwrap();
7474 assert_eq!(
7475 parse_error(&wrong_nonce[0])["code"],
7476 "bad_consumer_identity",
7477 "a mismatched launch nonce must be refused, not stamped Reserved"
7478 );
7479
7480 let never_spawned = handler
7484 .handle_control_frame(
7485 &route_ctx(ConnectionId::new(92)).0,
7486 route_open_frame_with_admission_facts(
7487 21,
7488 "target",
7489 unique_project_root("admission-facts"),
7490 Some(subc_control::ConsumerIdentity {
7491 module_id: "never-spawned".to_string(),
7492 launch_nonce: "any-nonce".to_string(),
7493 }),
7494 None,
7495 ),
7496 )
7497 .await
7498 .unwrap();
7499 assert_eq!(
7500 parse_error(&never_spawned[0])["code"],
7501 "bad_consumer_identity",
7502 "an unspawned module_id must be refused rather than accepted for lack of a record"
7503 );
7504 }
7505
7506 #[tokio::test]
7529 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
7530 let registry = Arc::new(Registry::default());
7531 let forwarding = Arc::new(ForwardingTable::default());
7532 let supervisor = SupervisorHandle::new();
7533 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7534 let handler =
7535 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
7536 .with_supervisor(supervisor);
7537
7538 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
7539 hello_via_sink(
7540 &handler,
7541 &target_ctx,
7542 &mut target_rx,
7543 hello_frame("target", PROTOCOL_VERSION, 1),
7544 )
7545 .await;
7546
7547 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
7548 let route_handler = handler.clone();
7549 let route_task = tokio::spawn(async move {
7550 route_handler
7551 .handle_control_frame(
7552 &client_ctx,
7553 route_open_frame_with_admission_facts(
7554 30,
7555 "target",
7556 unique_project_root("admission-facts"),
7557 Some(subc_control::ConsumerIdentity {
7558 module_id: "fed".to_string(),
7559 launch_nonce: "fed-nonce".to_string(),
7560 }),
7561 None,
7562 ),
7563 )
7564 .await
7565 .unwrap()
7566 });
7567
7568 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7575 .await
7576 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
7577 .expect("module control channel closed before route.bind");
7578 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
7579 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
7580 panic!("expected route.bind")
7581 };
7582 assert_eq!(
7583 principal,
7584 Some(Principal::Reserved {
7585 module_id: "fed".to_string()
7586 }),
7587 "a correctly attested consumer must be stamped Reserved for its own id"
7588 );
7589
7590 handler
7591 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
7592 .await
7593 .unwrap();
7594 assert!(route_task.await.unwrap().is_empty());
7595 assert!(
7596 matches!(
7597 serde_json::from_slice::<ClientControlResponse>(
7598 &client_rx.recv().await.unwrap().body
7599 )
7600 .unwrap(),
7601 ClientControlResponse::RouteOpen { .. }
7602 ),
7603 "the route must actually open, not merely avoid an error"
7604 );
7605 }
7606
7607 #[tokio::test(start_paused = true)]
7608 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
7609 let registry = Arc::new(Registry::default());
7610 let forwarding = Arc::new(ForwardingTable::default());
7611 let supervisor = SupervisorHandle::new();
7612 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7613 let handler =
7614 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
7615 .with_supervisor(supervisor);
7616
7617 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
7618 hello_via_sink(
7619 &handler,
7620 &target_ctx,
7621 &mut target_rx,
7622 hello_frame("target", PROTOCOL_VERSION, 1),
7623 )
7624 .await;
7625
7626 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
7627 let direct_handler = handler.clone();
7628 let direct_open = tokio::spawn(async move {
7629 direct_handler
7630 .handle_control_frame(
7631 &direct_ctx,
7632 route_open_frame(2, "target", unique_project_root("route-census-direct")),
7633 )
7634 .await
7635 .unwrap()
7636 });
7637 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7638 .await
7639 .expect("no direct route.bind within 5s")
7640 .expect("target control channel closed before direct route.bind");
7641 handler
7642 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
7643 .await
7644 .unwrap();
7645 assert!(direct_open.await.unwrap().is_empty());
7646 let _ = direct_rx.recv().await.unwrap();
7647
7648 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
7649 let reserved_handler = handler.clone();
7650 let reserved_open = tokio::spawn(async move {
7651 reserved_handler
7652 .handle_control_frame(
7653 &reserved_ctx,
7654 route_open_frame_with_admission_facts(
7655 3,
7656 "target",
7657 unique_project_root("admission-facts"),
7658 Some(ConsumerIdentity {
7659 module_id: "fed".to_string(),
7660 launch_nonce: "fed-nonce".to_string(),
7661 }),
7662 None,
7663 ),
7664 )
7665 .await
7666 .unwrap()
7667 });
7668 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7669 .await
7670 .expect("no reserved route.bind within 5s")
7671 .expect("target control channel closed before reserved route.bind");
7672 handler
7673 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
7674 .await
7675 .unwrap();
7676 assert!(reserved_open.await.unwrap().is_empty());
7677 let _ = reserved_rx.recv().await.unwrap();
7678
7679 forwarding
7680 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
7681 .unwrap();
7682 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
7683 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
7684 module_id: Some("target".to_string()),
7685 })
7686 .unwrap();
7687 let census_frame =
7688 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
7689 let response = handler
7690 .handle_control_frame(&census_ctx, census_frame)
7691 .await
7692 .unwrap()
7693 .pop()
7694 .unwrap();
7695 let actual: Value = serde_json::from_slice(&response.body).unwrap();
7696 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
7697 assert!(matches!(
7698 decoded,
7699 ClientControlResponse::SupervisorRoutes { .. }
7700 ));
7701 let routes = actual["modules"][0]["routes"].as_array().unwrap();
7702 assert_eq!(routes.len(), 2);
7703 assert!(routes.iter().all(|route| route["draining"] == true));
7704 assert!(
7707 routes.iter().all(|route| route["drain_reason"] == "reload"),
7708 "draining routes must name the drain's reason: {routes:?}"
7709 );
7710 assert!(routes.iter().any(|route| {
7711 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
7712 }));
7713 assert!(routes.iter().any(|route| {
7714 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
7715 }));
7716
7717 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
7718 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
7719 if std::env::var_os("UPDATE_GOLDEN").is_some() {
7720 std::fs::write(
7721 &golden_path,
7722 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
7723 )
7724 .unwrap();
7725 }
7726 let expected: Value =
7727 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
7728 assert_eq!(actual, expected);
7729 }
7730
7731 async fn query_live_roots(
7732 handler: &ControlHandler,
7733 module_ctx: &RouteCtx,
7734 ) -> ModuleControlResponseToModule {
7735 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
7736 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
7737 let response = handler
7738 .handle_control_frame(module_ctx, frame)
7739 .await
7740 .unwrap()
7741 .pop()
7742 .unwrap();
7743 serde_json::from_slice(&response.body).unwrap()
7744 }
7745
7746 #[tokio::test(start_paused = true)]
7747 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
7748 let registry = Arc::new(Registry::default());
7749 let forwarding = Arc::new(ForwardingTable::default());
7750 let handler = ControlHandler::with_forwarding(registry, forwarding);
7751 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
7752 hello_via_sink(
7753 &handler,
7754 &target_ctx,
7755 &mut target_rx,
7756 hello_frame("target", PROTOCOL_VERSION, 1),
7757 )
7758 .await;
7759 let root = unique_project_root("live-roots-known");
7760 let path = ProjectRootId::from_path_allowing_missing(root.path())
7761 .unwrap()
7762 .as_path()
7763 .to_path_buf();
7764 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
7765 let open_handler = handler.clone();
7766 let opened = tokio::spawn(async move {
7767 open_handler
7768 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
7769 .await
7770 .unwrap()
7771 });
7772 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7773 .await
7774 .unwrap()
7775 .unwrap();
7776 handler
7777 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
7778 .await
7779 .unwrap();
7780 assert!(opened.await.unwrap().is_empty());
7781 let _ = client_rx.recv().await.unwrap();
7782
7783 let root = unique_project_root("live-roots-pending");
7784 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
7785 .unwrap()
7786 .as_path()
7787 .to_path_buf();
7788 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
7789 let open_handler = handler.clone();
7790 let pending = tokio::spawn(async move {
7791 open_handler
7792 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
7793 .await
7794 .unwrap()
7795 });
7796 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7797 .await
7798 .unwrap()
7799 .unwrap();
7800 let actual = query_live_roots(&handler, &target_ctx).await;
7801 let ModuleControlResponseToModule::LiveRoots {
7802 roots,
7803 unknown_root_bindings,
7804 total_bindings,
7805 } = actual
7806 else {
7807 panic!("expected live roots")
7808 };
7809 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
7810 assert_eq!(unknown_root_bindings, 0);
7811 assert_eq!(
7812 roots.len(),
7813 2,
7814 "root-known arm must retain each canonical root"
7815 );
7816 assert_eq!(
7817 total_bindings,
7818 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
7819 );
7820 let counts = roots
7821 .iter()
7822 .map(|root| (root.project_root.clone(), root.bound, root.pending))
7823 .collect::<Vec<_>>();
7824 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
7825 expected.sort_by(|a, b| a.0.cmp(&b.0));
7826 assert_eq!(
7827 counts, expected,
7828 "roots must sort by path and count pending separately"
7829 );
7830 handler
7831 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
7832 .await
7833 .unwrap();
7834 assert!(pending.await.unwrap().is_empty());
7835 }
7836
7837 #[tokio::test(start_paused = true)]
7838 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
7839 let forwarding = Arc::new(ForwardingTable::default());
7840 let handler =
7841 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
7842 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
7843 hello_via_sink(
7844 &handler,
7845 &target_ctx,
7846 &mut target_rx,
7847 hello_frame("target", PROTOCOL_VERSION, 1),
7848 )
7849 .await;
7850 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
7851 let pending = forwarding
7852 .begin_route_bind_relay_for_test(
7853 client_ctx.connection_id,
7854 client_ctx.egress.clone(),
7855 2,
7856 "target",
7857 )
7858 .unwrap();
7859 forwarding
7860 .complete_pending_relay(
7861 target_ctx.connection_id,
7862 pending.corr,
7863 RouteBindRelayOutcome::Accepted,
7864 )
7865 .unwrap();
7866 let actual = query_live_roots(&handler, &target_ctx).await;
7867 let ModuleControlResponseToModule::LiveRoots {
7868 roots,
7869 unknown_root_bindings,
7870 total_bindings,
7871 } = actual
7872 else {
7873 panic!("expected live roots")
7874 };
7875 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
7876 assert_eq!(
7877 unknown_root_bindings, 1,
7878 "unknown-root arm must not read as no bindings"
7879 );
7880 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
7881 assert_eq!(
7882 total_bindings,
7883 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
7884 );
7885 }
7886
7887 #[tokio::test(start_paused = true)]
7895 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
7896 let forwarding = Arc::new(ForwardingTable::default());
7897 let handler =
7898 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
7899 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
7900 let replies = handler
7901 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
7902 .await
7903 .unwrap();
7904 let queued_by_hello = module_rx.len();
7905
7906 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
7907 let open_handler = handler.clone();
7908 let open = tokio::spawn(async move {
7909 open_handler
7910 .handle_control_frame(
7911 &client_ctx,
7912 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
7913 )
7914 .await
7915 .unwrap()
7916 });
7917 let mut spins = 0;
7919 while module_rx.len() == queued_by_hello {
7920 spins += 1;
7921 assert!(spins < 10_000, "route.open never queued a route.bind");
7922 tokio::task::yield_now().await;
7923 }
7924
7925 for reply in replies {
7927 module_ctx.egress.send(reply).await.unwrap();
7928 }
7929
7930 let first = module_rx.recv().await.unwrap().frame;
7931 assert_eq!(
7932 first.header.ty,
7933 FrameType::HelloAck,
7934 "the first frame a registering module reads must be its HELLO_ACK"
7935 );
7936 assert_eq!(first.header.corr, 7);
7937 let second = module_rx.recv().await.unwrap().frame;
7938 assert_eq!(second.header.ty, FrameType::Request);
7939 assert!(
7940 matches!(
7941 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
7942 ModuleControlRequest::RouteBind { .. }
7943 ),
7944 "the route.bind follows the ack"
7945 );
7946 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
7947
7948 handler
7949 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
7950 .await
7951 .unwrap();
7952 assert!(open.await.unwrap().is_empty());
7953 let _ = client_rx.recv().await.unwrap();
7954 }
7955
7956 #[tokio::test(start_paused = true)]
7957 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
7958 let handler = ControlHandler::with_forwarding(
7959 Arc::new(Registry::default()),
7960 Arc::new(ForwardingTable::default()),
7961 );
7962 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
7963 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
7964 hello_via_sink(
7965 &handler,
7966 &first_ctx,
7967 &mut first_rx,
7968 hello_frame("first", PROTOCOL_VERSION, 1),
7969 )
7970 .await;
7971 hello_via_sink(
7972 &handler,
7973 &second_ctx,
7974 &mut second_rx,
7975 hello_frame("second", PROTOCOL_VERSION, 2),
7976 )
7977 .await;
7978 let root = unique_project_root("second-only");
7979 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
7980 let cloned = handler.clone();
7981 let open = tokio::spawn(async move {
7982 cloned
7983 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
7984 .await
7985 .unwrap()
7986 });
7987 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
7988 .await
7989 .unwrap()
7990 .unwrap();
7991 let first = query_live_roots(&handler, &first_ctx).await;
7992 let second = query_live_roots(&handler, &second_ctx).await;
7993 assert!(
7994 matches!(
7995 first,
7996 ModuleControlResponseToModule::LiveRoots {
7997 total_bindings: 0,
7998 ..
7999 }
8000 ),
8001 "cross-module scope must not expose another module's roots"
8002 );
8003 assert!(
8004 matches!(
8005 second,
8006 ModuleControlResponseToModule::LiveRoots {
8007 total_bindings: 1,
8008 ..
8009 }
8010 ),
8011 "second module must see its pending route"
8012 );
8013 handler
8014 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
8015 .await
8016 .unwrap();
8017 assert!(open.await.unwrap().is_empty());
8018 }
8019
8020 #[tokio::test(start_paused = true)]
8021 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
8022 let handler = ControlHandler::with_forwarding(
8023 Arc::new(Registry::default()),
8024 Arc::new(ForwardingTable::default()),
8025 );
8026 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
8027 hello_via_sink(
8028 &handler,
8029 &target_ctx,
8030 &mut target_rx,
8031 hello_frame("target", PROTOCOL_VERSION, 1),
8032 )
8033 .await;
8034 let actual = query_live_roots(&handler, &target_ctx).await;
8035 let ModuleControlResponseToModule::LiveRoots {
8036 roots,
8037 unknown_root_bindings,
8038 total_bindings,
8039 } = actual
8040 else {
8041 panic!("expected live roots")
8042 };
8043 assert!(roots.is_empty());
8044 assert_eq!(unknown_root_bindings, 0);
8045 assert_eq!(total_bindings, 0);
8046 assert_eq!(
8047 total_bindings,
8048 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
8049 );
8050 }
8051
8052 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
8059 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8060 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
8061 let text = std::fs::read_to_string(&path)
8062 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
8063 let vectors: Vec<(String, Value)> = text
8064 .lines()
8065 .filter(|line| !line.trim().is_empty())
8066 .map(|line| {
8067 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
8068 let id = entry["corpus_id"]
8069 .as_str()
8070 .expect("every vector carries a corpus_id")
8071 .to_string();
8072 (id, entry["package"].clone())
8073 })
8074 .collect();
8075 assert_eq!(
8079 vectors.len(),
8080 3,
8081 "vendored fed corpus changed size; re-sync from subc-federation"
8082 );
8083
8084 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
8099 let richest = vectors
8100 .iter()
8101 .filter_map(|(_, package)| package.as_object())
8102 .map(|object| {
8103 object
8104 .keys()
8105 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
8106 .count()
8107 })
8108 .max()
8109 .unwrap_or(0);
8110 assert!(
8111 richest >= 2,
8112 "vendored corpus no longer carries a package with unmodelled fields, \
8113 so the relay test can no longer distinguish a verbatim relay from a lossy one"
8114 );
8115
8116 vectors
8117 }
8118
8119 #[tokio::test]
8127 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
8128 for (corpus_id, package) in fed_admission_facts_vectors() {
8129 let registry = Arc::new(Registry::default());
8130 let forwarding = Arc::new(ForwardingTable::default());
8131 let supervisor = SupervisorHandle::new();
8132 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8133 let handler =
8134 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8135 .with_supervisor(supervisor)
8136 .with_admission_facts_config(
8137 Some("fed".to_string()),
8138 Some(vec!["target".to_string()]),
8139 );
8140
8141 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
8142 hello_via_sink(
8143 &handler,
8144 &target_ctx,
8145 &mut target_rx,
8146 hello_frame("target", PROTOCOL_VERSION, 1),
8147 )
8148 .await;
8149
8150 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
8151 let route_handler = handler.clone();
8152 let expected = package.clone();
8153 let route_task = tokio::spawn(async move {
8154 route_handler
8155 .handle_control_frame(
8156 &client_ctx,
8157 route_open_frame_with_admission_facts(
8158 20,
8159 "target",
8160 unique_project_root("admission-facts"),
8161 Some(subc_control::ConsumerIdentity {
8162 module_id: "fed".to_string(),
8163 launch_nonce: "fed-nonce".to_string(),
8164 }),
8165 Some(package),
8166 ),
8167 )
8168 .await
8169 .unwrap()
8170 });
8171
8172 let bind_frame = target_rx.recv().await.unwrap();
8173 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8174 let ModuleControlRequest::RouteBind {
8175 admission_facts, ..
8176 } = bind
8177 else {
8178 panic!("{corpus_id}: expected route.bind")
8179 };
8180 assert_eq!(
8181 admission_facts,
8182 Some(expected),
8183 "{corpus_id}: relay must not add, drop or reshape any field"
8184 );
8185
8186 handler
8187 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8188 .await
8189 .unwrap();
8190 route_task.await.unwrap();
8191 }
8192 }
8193
8194 #[tokio::test]
8195 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
8196 let registry = Arc::new(Registry::default());
8197 let forwarding = Arc::new(ForwardingTable::default());
8198 let supervisor = SupervisorHandle::new();
8199 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8200 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
8201 let handler =
8202 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8203 .with_supervisor(supervisor)
8204 .with_admission_facts_config(
8205 Some("fed".to_string()),
8206 Some(vec!["target".to_string()]),
8207 );
8208
8209 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
8210 hello_via_sink(
8211 &handler,
8212 &target_ctx,
8213 &mut target_rx,
8214 hello_frame("target", PROTOCOL_VERSION, 1),
8215 )
8216 .await;
8217 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
8218 hello_via_sink(
8219 &handler,
8220 &other_ctx,
8221 &mut other_rx,
8222 hello_frame("other", PROTOCOL_VERSION, 2),
8223 )
8224 .await;
8225
8226 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
8227 let expected_facts = facts.clone();
8228 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
8229 let route_handler = handler.clone();
8230 let route_task = tokio::spawn(async move {
8231 route_handler
8232 .handle_control_frame(
8233 &client_ctx,
8234 route_open_frame_with_admission_facts(
8235 10,
8236 "target",
8237 unique_project_root("admission-facts"),
8238 Some(subc_control::ConsumerIdentity {
8239 module_id: "fed".to_string(),
8240 launch_nonce: "fed-nonce".to_string(),
8241 }),
8242 Some(facts.clone()),
8243 ),
8244 )
8245 .await
8246 .unwrap()
8247 });
8248 let bind_frame = target_rx.recv().await.unwrap();
8249 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8250 let ModuleControlRequest::RouteBind {
8251 admission_facts, ..
8252 } = bind
8253 else {
8254 panic!("expected route.bind")
8255 };
8256 assert_eq!(admission_facts, Some(expected_facts));
8257 handler
8258 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8259 .await
8260 .unwrap();
8261 assert!(route_task.await.unwrap().is_empty());
8262 assert!(matches!(
8263 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
8264 .unwrap(),
8265 ClientControlResponse::RouteOpen { .. }
8266 ));
8267
8268 let direct = handler
8269 .handle_control_frame(
8270 &route_ctx(ConnectionId::new(73)).0,
8271 route_open_frame_with_admission_facts(
8272 11,
8273 "target",
8274 unique_project_root("admission-facts"),
8275 None,
8276 Some(json!({"x": 1})),
8277 ),
8278 )
8279 .await
8280 .unwrap();
8281 assert_eq!(
8282 parse_error(&direct[0])["code"],
8283 "admission_facts_not_permitted"
8284 );
8285
8286 let different_reserved = handler
8287 .handle_control_frame(
8288 &route_ctx(ConnectionId::new(77)).0,
8289 route_open_frame_with_admission_facts(
8290 15,
8291 "target",
8292 unique_project_root("admission-facts"),
8293 Some(subc_control::ConsumerIdentity {
8294 module_id: "other".to_string(),
8295 launch_nonce: "other-nonce".to_string(),
8296 }),
8297 Some(json!({"x": 1})),
8298 ),
8299 )
8300 .await
8301 .unwrap();
8302 assert_eq!(
8303 parse_error(&different_reserved[0])["code"],
8304 "admission_facts_not_permitted"
8305 );
8306
8307 let other_target = handler
8308 .handle_control_frame(
8309 &route_ctx(ConnectionId::new(74)).0,
8310 route_open_frame_with_admission_facts(
8311 12,
8312 "other",
8313 unique_project_root("admission-facts"),
8314 Some(subc_control::ConsumerIdentity {
8315 module_id: "fed".to_string(),
8316 launch_nonce: "fed-nonce".to_string(),
8317 }),
8318 Some(json!({"x": 1})),
8319 ),
8320 )
8321 .await
8322 .unwrap();
8323 assert_eq!(
8324 parse_error(&other_target[0])["code"],
8325 "admission_facts_target_not_allowed"
8326 );
8327
8328 let nonexistent = handler
8329 .handle_control_frame(
8330 &route_ctx(ConnectionId::new(75)).0,
8331 route_open_frame_with_admission_facts(
8332 13,
8333 "missing",
8334 unique_project_root("admission-facts"),
8335 None,
8336 Some(json!({"x": 1})),
8337 ),
8338 )
8339 .await
8340 .unwrap();
8341 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
8342
8343 let described = handler
8344 .handle_control_frame(
8345 &route_ctx(ConnectionId::new(76)).0,
8346 Frame::build(
8347 FrameType::Request,
8348 control_flags(),
8349 0,
8350 0,
8351 14,
8352 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
8353 )
8354 .unwrap(),
8355 )
8356 .await
8357 .unwrap();
8358 let ClientControlResponse::ServerDescribe { capabilities, .. } =
8359 serde_json::from_slice(&described[0].body).unwrap()
8360 else {
8361 panic!("expected server.describe response")
8362 };
8363 assert!(capabilities
8364 .iter()
8365 .any(|cap| cap == "admission_facts_relay_v1"));
8366 }
8367
8368 #[tokio::test]
8369 async fn admission_facts_without_configured_carrier_are_rejected() {
8370 let registry = Arc::new(Registry::default());
8371 let forwarding = Arc::new(ForwardingTable::default());
8372 let handler = ControlHandler::with_forwarding(registry, forwarding);
8373 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
8374 hello_via_sink(
8375 &handler,
8376 &target_ctx,
8377 &mut target_rx,
8378 hello_frame("target", PROTOCOL_VERSION, 1),
8379 )
8380 .await;
8381
8382 let responses = handler
8383 .handle_control_frame(
8384 &route_ctx(ConnectionId::new(79)).0,
8385 route_open_frame_with_admission_facts(
8386 16,
8387 "target",
8388 unique_project_root("admission-facts"),
8389 None,
8390 Some(json!({"x": 1})),
8391 ),
8392 )
8393 .await
8394 .unwrap();
8395 assert_eq!(
8396 parse_error(&responses[0])["code"],
8397 "admission_facts_not_permitted"
8398 );
8399 }
8400
8401 #[tokio::test]
8402 async fn route_open_relays_consumer_capabilities_verbatim() {
8403 let registry = Arc::new(Registry::default());
8404 let forwarding = Arc::new(ForwardingTable::default());
8405 let handler =
8406 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8407 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
8408 hello_via_sink(
8409 &handler,
8410 &module_ctx,
8411 &mut module_rx,
8412 hello_frame("aft", PROTOCOL_VERSION, 7),
8413 )
8414 .await;
8415
8416 let expected = vec!["elicitation".to_string(), "roots".to_string()];
8417 let expected_for_request = expected.clone();
8418 let project_root = unique_project_root("consumer-capabilities-present");
8419 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
8420 let route_handler = handler.clone();
8421 let route_task = tokio::spawn(async move {
8422 route_handler
8423 .handle_control_frame(
8424 &client_ctx,
8425 route_open_frame_with_consumer_capabilities(
8426 401,
8427 "aft",
8428 project_root,
8429 Some(expected_for_request),
8430 ),
8431 )
8432 .await
8433 .unwrap()
8434 });
8435 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8436 .await
8437 .unwrap()
8438 .unwrap();
8439 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8440 let ModuleControlRequest::RouteBind {
8441 consumer_capabilities,
8442 ..
8443 } = bind
8444 else {
8445 panic!("expected route.bind request, got {bind:?}");
8446 };
8447 assert_eq!(consumer_capabilities, Some(expected.clone()));
8448
8449 handler
8450 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8451 .await
8452 .unwrap();
8453 let route_response = route_task.await.unwrap();
8454 assert!(route_response.is_empty());
8455 let published = client_rx.recv().await.unwrap();
8456 assert!(matches!(
8457 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8458 ClientControlResponse::RouteOpen { .. }
8459 ));
8460 }
8461
8462 #[tokio::test]
8463 async fn route_open_without_consumer_capabilities_relays_none() {
8464 let registry = Arc::new(Registry::default());
8465 let forwarding = Arc::new(ForwardingTable::default());
8466 let handler =
8467 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8468 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
8469 hello_via_sink(
8470 &handler,
8471 &module_ctx,
8472 &mut module_rx,
8473 hello_frame("aft", PROTOCOL_VERSION, 7),
8474 )
8475 .await;
8476
8477 let project_root = unique_project_root("consumer-capabilities-absent");
8478 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
8479 let route_handler = handler.clone();
8480 let route_task = tokio::spawn(async move {
8481 route_handler
8482 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
8483 .await
8484 .unwrap()
8485 });
8486 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8487 .await
8488 .unwrap()
8489 .unwrap();
8490 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8491 let ModuleControlRequest::RouteBind {
8492 consumer_capabilities,
8493 ..
8494 } = bind
8495 else {
8496 panic!("expected route.bind request, got {bind:?}");
8497 };
8498 assert_eq!(consumer_capabilities, None);
8499
8500 handler
8501 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8502 .await
8503 .unwrap();
8504 let route_response = route_task.await.unwrap();
8505 assert!(route_response.is_empty());
8506 let published = client_rx.recv().await.unwrap();
8507 assert!(matches!(
8508 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8509 ClientControlResponse::RouteOpen { .. }
8510 ));
8511 }
8512
8513 #[tokio::test]
8514 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
8515 let registry = Arc::new(Registry::default());
8516 let forwarding = Arc::new(ForwardingTable::default());
8517 let handler =
8518 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8519 .with_health_probe_timeout(Duration::from_secs(5));
8520 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
8521 hello_via_sink(
8522 &handler,
8523 &module_ctx,
8524 &mut module_rx,
8525 non_routable_hello_frame_with_control_ops(
8526 "mcp",
8527 300,
8528 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
8529 ),
8530 )
8531 .await;
8532 assert!(registry
8533 .get_module("mcp")
8534 .unwrap()
8535 .unwrap()
8536 .manifest
8537 .provides
8538 .is_empty());
8539
8540 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
8541 let route_response = handler
8542 .handle_control_frame(
8543 &route_client_ctx,
8544 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
8545 )
8546 .await
8547 .unwrap();
8548 assert_eq!(route_response[0].header.ty, FrameType::Error);
8549 assert_eq!(
8550 parse_error(&route_response[0])["code"],
8551 "target_unavailable"
8552 );
8553 assert!(parse_error(&route_response[0])["message"]
8554 .as_str()
8555 .unwrap()
8556 .contains("does not provide the requested target"));
8557 assert!(module_rx.try_recv().is_err());
8558
8559 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
8560 let health_handler = handler.clone();
8561 let health_task = tokio::spawn(async move {
8562 health_handler
8563 .handle_control_frame(
8564 &health_client_ctx,
8565 supervisor_health_probe_frame(302, "mcp"),
8566 )
8567 .await
8568 .unwrap()
8569 });
8570 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8571 .await
8572 .unwrap()
8573 .unwrap();
8574 assert_eq!(
8575 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
8576 ModuleControlRequest::HealthCheck {}
8577 );
8578 handler
8579 .handle_control_frame(
8580 &module_ctx,
8581 health_response(health_frame.header.corr, HealthStatus::Ok),
8582 )
8583 .await
8584 .unwrap();
8585 let health_response = health_task.await.unwrap();
8586 assert_eq!(health_response[0].header.ty, FrameType::Response);
8587 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
8588 ClientControlResponse::SupervisorHealthProbe {
8589 module_id, status, ..
8590 } => {
8591 assert_eq!(module_id, "mcp");
8592 assert_eq!(status, HealthStatus::Ok);
8593 }
8594 other => panic!("unexpected health response: {other:?}"),
8595 }
8596
8597 forwarding
8602 .cleanup_connection(module_ctx.connection_id)
8603 .unwrap();
8604 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
8605 let cleanup_response = tokio::time::timeout(
8606 Duration::from_millis(200),
8607 handler.handle_control_frame(
8608 &cleanup_probe_ctx,
8609 supervisor_health_probe_frame(303, "mcp"),
8610 ),
8611 )
8612 .await
8613 .expect("probe should fail immediately when the control lane is gone")
8614 .unwrap();
8615 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
8616 assert_eq!(
8617 parse_error(&cleanup_response[0])["code"],
8618 "target_unavailable"
8619 );
8620 assert!(parse_error(&cleanup_response[0])["message"]
8621 .as_str()
8622 .unwrap()
8623 .contains("no module connection"));
8624
8625 handler
8626 .cleanup_connection(module_ctx.connection_id)
8627 .unwrap();
8628 }
8629
8630 #[tokio::test]
8631 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
8632 let registry = Arc::new(Registry::default());
8633 let supervisor_handle = SupervisorHandle::new();
8634 let supervisor =
8635 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
8636 .with_handle(supervisor_handle.clone())
8637 .with_connection_file_path(
8638 std::env::temp_dir()
8639 .join(format!("subc-route-open-warming-{}", std::process::id())),
8640 );
8641 let module = supervisor
8642 .supervise_configured(
8643 ModuleSpec {
8644 module_id: "warming".to_string(),
8645 program: fake_aft_stub_path(),
8646 args: Vec::new(),
8647 env: Vec::new(),
8648 reserved: false,
8649 reserved_prefixes: Vec::new(),
8650 protocol: ModuleProtocol::Subc,
8651 overlap: Default::default(),
8652 },
8653 true,
8654 )
8655 .unwrap();
8656 assert_eq!(module.state().unwrap(), ModuleState::Running);
8657
8658 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8659 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
8660 let response = handler
8661 .handle_control_frame(
8662 &ctx,
8663 route_open_frame(304, "warming", unique_project_root("warming")),
8664 )
8665 .await
8666 .unwrap();
8667 module.stop().await.unwrap();
8668
8669 assert_eq!(response[0].header.ty, FrameType::Error);
8670 let error = parse_error(&response[0]);
8671 assert_eq!(error["code"], "module_warming");
8672 assert!(error["message"]
8673 .as_str()
8674 .unwrap()
8675 .contains("state=running, enabled=true, live=false"));
8676 }
8677
8678 #[test]
8679 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
8680 let handler = ControlHandler::new(Arc::new(Registry::default()));
8681 let capture = EventCapture::default();
8682 let _subscriber =
8683 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8684 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
8685 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
8686 let pending = (0..limit).collect::<Vec<_>>();
8687 let response = handler
8688 .route_open_capacity_refusal(
8689 &ctx,
8690 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
8691 "busy",
8692 pending.len(),
8693 limit,
8694 )
8695 .unwrap();
8696 assert_eq!(parse_error(&response)["code"], "target_unavailable");
8697 let event = capture
8698 .events()
8699 .into_iter()
8700 .find(|event| {
8701 event.target == "control"
8702 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
8703 })
8704 .expect("connection admission refusal event");
8705 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
8706 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
8707 }
8708
8709 #[test]
8710 fn route_open_target_cap_logs_admission_reason_and_capacity() {
8711 let handler = ControlHandler::new(Arc::new(Registry::default()));
8712 let capture = EventCapture::default();
8713 let _subscriber =
8714 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8715 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
8716 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
8717 let guards = (0..limit)
8718 .map(|_| {
8719 handler
8720 .route_bind_concurrency
8721 .try_admit("busy", limit)
8722 .unwrap()
8723 })
8724 .collect::<Vec<_>>();
8725 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
8726 Err(in_flight) => in_flight,
8727 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
8728 };
8729 let response = handler
8730 .route_open_target_capacity_refusal(
8731 &ctx,
8732 &route_open_frame(397, "busy", unique_project_root("target-cap")),
8733 "busy",
8734 in_flight,
8735 )
8736 .unwrap();
8737 assert_eq!(parse_error(&response)["code"], "target_unavailable");
8738 let event = capture
8739 .events()
8740 .into_iter()
8741 .find(|event| {
8742 event.target == "control"
8743 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
8744 })
8745 .expect("target admission refusal event");
8746 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
8747 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
8748 drop(guards);
8749 }
8750
8751 #[tokio::test]
8756 async fn route_open_refusal_names_the_check_that_refused() {
8757 let handler = ControlHandler::new(Arc::new(Registry::default()));
8758 let capture = EventCapture::default();
8759 let _subscriber =
8760 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8761 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
8762 let response = handler
8763 .handle_control_frame(
8764 &ctx,
8765 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
8766 )
8767 .await
8768 .unwrap();
8769
8770 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
8771 let event = capture
8772 .events()
8773 .into_iter()
8774 .find(|event| {
8775 event.target == "control"
8776 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
8777 })
8778 .expect("route.open refusal event");
8779 assert_eq!(
8780 event.fields.get("reason"),
8781 Some(&"\"not_registered\"".to_string())
8782 );
8783 }
8784
8785 #[tokio::test]
8786 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
8787 let registry = Arc::new(Registry::default());
8788 let supervisor_handle = SupervisorHandle::new();
8789 let supervisor =
8790 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
8791 .with_handle(supervisor_handle.clone())
8792 .with_connection_file_path(std::env::temp_dir().join(format!(
8793 "subc-route-open-refusal-info-{}",
8794 std::process::id()
8795 )));
8796 let module = supervisor
8797 .supervise_configured(
8798 ModuleSpec {
8799 module_id: "warming".to_string(),
8800 program: fake_aft_stub_path(),
8801 args: Vec::new(),
8802 env: Vec::new(),
8803 reserved: false,
8804 reserved_prefixes: Vec::new(),
8805 protocol: ModuleProtocol::Subc,
8806 overlap: Default::default(),
8807 },
8808 true,
8809 )
8810 .unwrap();
8811 assert_eq!(module.state().unwrap(), ModuleState::Running);
8812
8813 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8814 assert!(handler
8815 .counters()
8816 .snapshot()
8817 .get("route_open_refused_by_code")
8818 .is_none());
8819 let capture = EventCapture::default();
8820 let _subscriber =
8821 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8822 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
8823 let response = handler
8824 .handle_control_frame(
8825 &ctx,
8826 route_open_frame(394, "warming", unique_project_root("refusal-info")),
8827 )
8828 .await
8829 .unwrap();
8830 module.stop().await.unwrap();
8831
8832 assert_eq!(parse_error(&response[0])["code"], "module_warming");
8833 let event = capture
8834 .events()
8835 .into_iter()
8836 .find(|event| {
8837 event.target == "control"
8838 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
8839 })
8840 .expect("route.open refusal event");
8841 assert_eq!(
8842 event.fields.get("module_id"),
8843 Some(&"\"warming\"".to_string())
8844 );
8845 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
8846 assert_eq!(
8847 event.fields.get("reason"),
8848 Some(&"\"supervised_not_registered\"".to_string())
8849 );
8850 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
8851 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
8852 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
8853 assert_eq!(
8854 handler.counters().snapshot()["route_open_refused_by_code"],
8855 json!({ "module_warming": 1 })
8856 );
8857 }
8858
8859 const OUTAGE_START: &str = "route.open refusing module: not serving";
8860 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
8861
8862 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
8863 capture
8864 .events()
8865 .into_iter()
8866 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
8867 .collect()
8868 }
8869
8870 fn supervise_stub(
8871 registry: &Arc<Registry>,
8872 module_id: &str,
8873 enabled: bool,
8874 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
8875 let supervisor_handle = SupervisorHandle::new();
8876 let supervisor =
8877 Supervisor::new(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
8878 .with_handle(supervisor_handle.clone())
8879 .with_connection_file_path(std::env::temp_dir().join(format!(
8880 "subc-route-outage-{module_id}-{}",
8881 std::process::id()
8882 )));
8883 let module = supervisor
8884 .supervise_configured(
8885 ModuleSpec {
8886 module_id: module_id.to_string(),
8887 program: fake_aft_stub_path(),
8888 args: Vec::new(),
8889 env: Vec::new(),
8890 reserved: false,
8891 reserved_prefixes: Vec::new(),
8892 protocol: ModuleProtocol::Subc,
8893 overlap: Default::default(),
8894 },
8895 enabled,
8896 )
8897 .unwrap();
8898 (supervisor_handle, module)
8899 }
8900
8901 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
8902 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
8903 module_id: module_id.to_string(),
8904 drain_timeout_ms: Some(50),
8905 })
8906 .unwrap();
8907 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8908 }
8909
8910 #[test]
8914 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
8915 let registry = Arc::new(Registry::default());
8916 let forwarding = Arc::new(ForwardingTable::default());
8917 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8918 let second = ControlHandler::with_forwarding(registry, forwarding);
8919 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
8920 }
8921
8922 #[tokio::test(flavor = "current_thread")]
8926 async fn route_open_unknown_module_refusals_add_no_outage_state() {
8927 let handler = ControlHandler::new(Arc::new(Registry::default()));
8928 let capture = EventCapture::default();
8929 let _subscriber =
8930 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8931 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
8932 for corr in 0..8 {
8933 let response = handler
8934 .handle_control_frame(
8935 &ctx,
8936 route_open_frame(
8937 380 + corr,
8938 &format!("nobody-{corr}"),
8939 unique_project_root("outage-unknown"),
8940 ),
8941 )
8942 .await
8943 .unwrap();
8944 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
8945 }
8946
8947 assert_eq!(handler.route_outages.tracked_module_count(), 0);
8948 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
8949 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
8950 }
8951
8952 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8957 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
8958 let registry = Arc::new(Registry::default());
8959 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
8960 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8961 let capture = EventCapture::default();
8962 let _subscriber =
8963 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8964 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
8965 handler.route_outages.record_accepted("outage-restart");
8968
8969 let response = handler
8970 .handle_control_frame(
8971 &ctx,
8972 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
8973 )
8974 .await
8975 .unwrap();
8976 assert_eq!(response[0].header.ty, FrameType::Error);
8977 let starts = outage_lines(&capture, OUTAGE_START);
8978 assert_eq!(starts.len(), 1, "{starts:?}");
8979 assert_eq!(starts[0].level, tracing::Level::WARN);
8980 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
8981 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
8982 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
8983 handler.route_outages.record_accepted("outage-restart");
8984 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
8985
8986 let restart = handler
8987 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
8988 .await
8989 .unwrap();
8990 assert_eq!(
8991 restart[0].header.ty,
8992 FrameType::Response,
8993 "{:?}",
8994 parse_error(&restart[0])
8995 );
8996 handler
8997 .handle_control_frame(
8998 &ctx,
8999 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
9000 )
9001 .await
9002 .unwrap();
9003 module.stop().await.unwrap();
9004
9005 let starts = outage_lines(&capture, OUTAGE_START);
9006 assert_eq!(starts.len(), 2, "{starts:?}");
9007 assert_eq!(starts[1].level, tracing::Level::INFO);
9008 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
9009 }
9010
9011 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9015 async fn failed_operator_restart_leaves_no_operator_mark() {
9016 let registry = Arc::new(Registry::default());
9017 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
9018 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9019 let capture = EventCapture::default();
9020 let _subscriber =
9021 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9022 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
9023 handler.route_outages.record_accepted("outage-disabled");
9024
9025 let restart = handler
9026 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
9027 .await
9028 .unwrap();
9029 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
9030 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
9031
9032 handler
9033 .handle_control_frame(
9034 &ctx,
9035 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
9036 )
9037 .await
9038 .unwrap();
9039 let starts = outage_lines(&capture, OUTAGE_START);
9040 assert_eq!(starts.len(), 1, "{starts:?}");
9041 assert_eq!(starts[0].level, tracing::Level::WARN);
9042 }
9043
9044 #[tokio::test(flavor = "current_thread")]
9045 async fn route_open_unknown_module_escapes_target_module_id() {
9046 let handler = ControlHandler::new(Arc::new(Registry::default()));
9047 let capture = EventCapture::default();
9048 let _subscriber =
9049 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9050 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
9051 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
9052 let response = handler
9053 .handle_control_frame(
9054 &ctx,
9055 route_open_frame(
9056 395,
9057 hostile_module_id,
9058 unique_project_root("hostile-target-module-id"),
9059 ),
9060 )
9061 .await
9062 .unwrap();
9063
9064 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
9065 let event = capture
9066 .events()
9067 .into_iter()
9068 .find(|event| {
9069 event.target == "control"
9070 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
9071 })
9072 .expect("route.open unknown-module refusal event");
9073 let logged = event.fields.get("module_id").expect("module_id field");
9074 assert!(!logged.bytes().any(|byte| byte < 0x20));
9075 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
9076 }
9077
9078 #[tokio::test(flavor = "current_thread")]
9079 async fn route_open_module_rejection_uses_daemon_counter_key() {
9080 let registry = Arc::new(Registry::default());
9081 let forwarding = Arc::new(ForwardingTable::default());
9082 let handler =
9083 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9084 let module_connection = ConnectionId::new(95);
9085 let (module_ctx, mut module_rx) = route_ctx(module_connection);
9086 hello_via_sink(
9087 &handler,
9088 &module_ctx,
9089 &mut module_rx,
9090 hello_frame("aft", PROTOCOL_VERSION, 395),
9091 )
9092 .await;
9093
9094 let client_connection = ConnectionId::new(96);
9095 let (client_ctx, _client_rx) = route_ctx(client_connection);
9096 let capture = EventCapture::default();
9097 let _subscriber =
9098 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9099 let (route_task, bind) = relay_route_open(
9100 &handler,
9101 client_connection,
9102 &client_ctx.egress,
9103 &mut module_rx,
9104 396,
9105 "aft",
9106 "hostile-module-code",
9107 )
9108 .await;
9109 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
9110 let rejection = Frame::build(
9111 FrameType::Error,
9112 control_flags(),
9113 0,
9114 0,
9115 bind.header.corr,
9116 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
9117 )
9118 .unwrap();
9119 handler
9120 .handle_control_frame(&module_ctx, rejection)
9121 .await
9122 .unwrap();
9123
9124 let response = route_task.await.unwrap();
9125 assert_eq!(parse_error(&response[0])["code"], hostile_code);
9126 let counters = handler.counters().snapshot();
9127 assert_eq!(
9128 counters["route_open_refused_by_code"],
9129 json!({ "module_rejected": 1 })
9130 );
9131 assert!(counters["route_open_refused_by_code"]
9132 .get(hostile_code)
9133 .is_none());
9134
9135 let event = capture
9136 .events()
9137 .into_iter()
9138 .find(|event| {
9139 event.target == "control"
9140 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
9141 })
9142 .expect("route.open module-rejection refusal event");
9143 let logged = event.fields.get("module_code").expect("module_code field");
9144 assert!(!logged.bytes().any(|byte| byte < 0x20));
9145 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
9146 }
9147
9148 #[tokio::test]
9149 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
9150 let registry = Arc::new(Registry::default());
9151 let supervisor_handle = SupervisorHandle::new();
9152 let missing_program = std::env::temp_dir().join(format!(
9153 "subc-route-open-missing-program-{}",
9154 std::process::id()
9155 ));
9156 let supervisor =
9157 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
9158 .with_handle(supervisor_handle.clone());
9159 let module = supervisor
9160 .supervise_configured(
9161 ModuleSpec {
9162 module_id: "failed".to_string(),
9163 program: missing_program,
9164 args: Vec::new(),
9165 env: Vec::new(),
9166 reserved: false,
9167 reserved_prefixes: Vec::new(),
9168 protocol: ModuleProtocol::Subc,
9169 overlap: Default::default(),
9170 },
9171 true,
9172 )
9173 .unwrap();
9174 assert_eq!(module.state().unwrap(), ModuleState::Failed);
9175
9176 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9177 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
9178 let response = handler
9179 .handle_control_frame(
9180 &ctx,
9181 route_open_frame(305, "failed", unique_project_root("failed")),
9182 )
9183 .await
9184 .unwrap();
9185
9186 assert_eq!(response[0].header.ty, FrameType::Error);
9187 let error = parse_error(&response[0]);
9188 assert_eq!(error["code"], "target_unavailable");
9189 assert!(error["message"]
9190 .as_str()
9191 .unwrap()
9192 .contains("state=failed, enabled=true, live=false"));
9193 }
9194
9195 #[tokio::test]
9196 async fn route_open_role_mismatch_remains_target_unavailable() {
9197 let registry = Arc::new(Registry::default());
9198 let handler = ControlHandler::new(Arc::clone(®istry));
9199 handler
9200 .handle_control(
9201 ConnectionId::new(41),
9202 non_routable_hello_frame_with_control_ops("health-only", 306, None),
9203 )
9204 .unwrap();
9205
9206 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
9207 let response = handler
9208 .handle_control_frame(
9209 &ctx,
9210 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
9211 )
9212 .await
9213 .unwrap();
9214
9215 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
9216 assert!(parse_error(&response[0])["message"]
9217 .as_str()
9218 .unwrap()
9219 .contains("does not provide the requested target"));
9220 }
9221
9222 #[tokio::test]
9223 async fn route_open_inactive_registration_remains_target_unavailable() {
9224 let registry = Arc::new(Registry::default());
9225 let handler = ControlHandler::new(Arc::clone(®istry));
9226 handler
9227 .handle_control(
9228 ConnectionId::new(43),
9229 hello_frame("inactive", PROTOCOL_VERSION, 308),
9230 )
9231 .unwrap();
9232 assert!(registry
9233 .set_module_state_for_test("inactive", ChannelState::Closed)
9234 .unwrap());
9235
9236 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
9237 let response = handler
9238 .handle_control_frame(
9239 &ctx,
9240 route_open_frame(309, "inactive", unique_project_root("inactive")),
9241 )
9242 .await
9243 .unwrap();
9244
9245 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
9246 assert!(parse_error(&response[0])["message"]
9247 .as_str()
9248 .unwrap()
9249 .contains("is not active"));
9250 }
9251
9252 #[tokio::test]
9253 async fn late_health_reply_is_recorded_through_the_module_response_path() {
9254 let registry = Arc::new(Registry::default());
9255 let forwarding = Arc::new(ForwardingTable::default());
9256 let supervisor_handle = SupervisorHandle::new();
9257 let supervisor = Supervisor::new(Arc::clone(®istry), crate::RestartPolicy::default())
9258 .with_forwarding(Arc::clone(&forwarding))
9259 .with_handle(supervisor_handle.clone());
9260 let module = supervisor
9261 .supervise_configured(
9262 crate::ModuleSpec {
9263 module_id: "late-health-response".to_string(),
9264 program: PathBuf::from("disabled-module"),
9265 args: Vec::new(),
9266 env: Vec::new(),
9267 reserved: false,
9268 reserved_prefixes: Vec::new(),
9269 protocol: ModuleProtocol::Subc,
9270 overlap: Default::default(),
9271 },
9272 false,
9273 )
9274 .unwrap();
9275 let handler =
9276 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9277 .with_supervisor(supervisor_handle);
9278 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
9279 handler
9280 .handle_control_frame(
9281 &module_ctx,
9282 hello_frame_with_control_ops(
9283 "late-health-response",
9284 PROTOCOL_VERSION,
9285 7,
9286 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9287 ),
9288 )
9289 .await
9290 .unwrap();
9291 let probe_started_at = Instant::now() - Duration::from_millis(80);
9292 let pending = forwarding
9293 .begin_health_probe_rpc_for(
9294 "late-health-response",
9295 MODULE_CONTROL_OP_HEALTH_CHECK,
9296 probe_started_at,
9297 Instant::now() - Duration::from_millis(1),
9298 )
9299 .unwrap();
9300 assert!(forwarding
9301 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
9302 .unwrap());
9303
9304 let responses = handler
9305 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
9306 .await
9307 .unwrap();
9308
9309 assert!(responses.is_empty());
9310 let health = module.status().unwrap().health;
9311 assert_eq!(health.late_answer_count, 1);
9312 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
9313 }
9314
9315 #[tokio::test]
9316 async fn health_probe_timeout_and_module_death_are_typed() {
9317 let registry = Arc::new(Registry::default());
9318 let forwarding = Arc::new(ForwardingTable::default());
9319 let handler =
9320 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9321 .with_health_probe_timeout(Duration::from_millis(50));
9322 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
9323 hello_via_sink(
9324 &handler,
9325 &module_ctx,
9326 &mut module_rx,
9327 hello_frame_with_control_ops(
9328 "aft",
9329 PROTOCOL_VERSION,
9330 7,
9331 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9332 ),
9333 )
9334 .await;
9335
9336 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
9337 let responses = handler
9338 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
9339 .await
9340 .unwrap();
9341 assert_eq!(responses[0].header.ty, FrameType::Error);
9342 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
9343 let _ = module_rx.try_recv();
9344
9345 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
9346 let health_handler = handler.clone();
9347 let death_task = tokio::spawn(async move {
9348 health_handler
9349 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
9350 .await
9351 .unwrap()
9352 });
9353 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9354 .await
9355 .unwrap()
9356 .unwrap();
9357 handler
9358 .cleanup_connection(module_ctx.connection_id)
9359 .unwrap();
9360 let responses = death_task.await.unwrap();
9361 assert_eq!(responses[0].header.ty, FrameType::Error);
9362 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
9363 }
9364
9365 #[test]
9366 fn hello_requires_exact_protocol_version() {
9367 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
9368 let registry = Arc::new(Registry::default());
9369 let handler = ControlHandler::new(Arc::clone(®istry));
9370 let responses = handler
9371 .handle_control(
9372 ConnectionId::new(connection),
9373 hello_frame("aft", offered, 9),
9374 )
9375 .unwrap();
9376
9377 assert_eq!(responses.len(), 1);
9378 assert_eq!(responses[0].header.ty, FrameType::Error);
9379 let error = parse_error(&responses[0]);
9380 assert_eq!(error["code"], "version_unsupported");
9381 assert!(registry.get_module("aft").unwrap().is_none());
9382 assert_eq!(registry.active_registration_count().unwrap(), 0);
9383 }
9384 }
9385
9386 #[test]
9387 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
9388 let registry = Arc::new(Registry::default());
9389 let forwarding = Arc::new(ForwardingTable::default());
9390 let handler =
9391 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9392 let module_connection = ConnectionId::new(301);
9393 let registration = registry
9394 .register_with_control_ops(
9395 manifest("aft-push", PROTOCOL_VERSION),
9396 PROTOCOL_VERSION,
9397 module_connection,
9398 module_baseline_control_ops(),
9399 )
9400 .unwrap();
9401 let (module_tx, _module_rx) = mpsc::channel(8);
9402 let endpoint = forwarding
9403 .register_module_connection(
9404 module_connection,
9405 "aft-push".to_string(),
9406 PROTOCOL_VERSION,
9407 manifest_concurrency(®istration.manifest),
9408 FrameSink::new(module_tx),
9409 )
9410 .unwrap();
9411
9412 let unknown = Frame::build(
9414 FrameType::Push,
9415 control_flags(),
9416 0,
9417 0,
9418 5,
9419 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
9420 )
9421 .unwrap();
9422 let out = handler.handle_status_update(endpoint, unknown).unwrap();
9423 assert!(
9424 out.is_empty(),
9425 "unknown push op must be ignored, got {out:?}"
9426 );
9427
9428 let malformed = Frame::build(
9430 FrameType::Push,
9431 control_flags(),
9432 0,
9433 0,
9434 6,
9435 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
9436 )
9437 .unwrap();
9438 let out = handler.handle_status_update(endpoint, malformed).unwrap();
9439 assert_eq!(out.len(), 1);
9440 assert_eq!(out[0].header.ty, FrameType::Error);
9441 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
9442 }
9443
9444 #[test]
9445 fn hello_rejected_when_connection_already_owns_client_routes() {
9446 let registry = Arc::new(Registry::default());
9447 let forwarding = Arc::new(ForwardingTable::default());
9448 let handler =
9449 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9450 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
9452 let client_connection = ConnectionId::new(202);
9453
9454 let responses = handler
9457 .handle_control(
9458 client_connection,
9459 hello_frame("aft-second", PROTOCOL_VERSION, 9),
9460 )
9461 .unwrap();
9462 assert_eq!(responses[0].header.ty, FrameType::Error);
9463 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
9464 assert!(registry.get_module("aft-second").unwrap().is_none());
9465 }
9466
9467 #[test]
9468 fn reserved_module_hello_requires_matching_launch_nonce() {
9469 let registry = Arc::new(Registry::default());
9470 let supervisor = SupervisorHandle::new();
9471 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
9474 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9475
9476 let no_nonce = handler
9478 .handle_control(
9479 ConnectionId::new(1),
9480 hello_frame("vault", PROTOCOL_VERSION, 1),
9481 )
9482 .unwrap();
9483 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
9484 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
9485 assert!(registry.get_module("vault").unwrap().is_none());
9486
9487 let wrong = handler
9489 .handle_control(
9490 ConnectionId::new(2),
9491 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
9492 )
9493 .unwrap();
9494 assert_eq!(wrong[0].header.ty, FrameType::Error);
9495 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
9496 assert!(registry.get_module("vault").unwrap().is_none());
9497
9498 let ok = handler
9500 .handle_control(
9501 ConnectionId::new(3),
9502 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
9503 )
9504 .unwrap();
9505 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
9506 assert!(registry.get_module("vault").unwrap().is_some());
9507 }
9508
9509 #[test]
9510 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
9511 let registry = Arc::new(Registry::default());
9512 let supervisor = SupervisorHandle::new();
9513 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
9514 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
9515 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9516
9517 let squat = handler
9518 .handle_control(
9519 ConnectionId::new(1),
9520 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
9521 )
9522 .unwrap();
9523 assert_eq!(squat[0].header.ty, FrameType::Error);
9524 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
9525 assert!(parse_error(&squat[0])["message"]
9526 .as_str()
9527 .unwrap()
9528 .contains("fed:"));
9529
9530 let accepted_peer = handler
9531 .handle_control(
9532 ConnectionId::new(2),
9533 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
9534 )
9535 .unwrap();
9536 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
9537
9538 let accepted_short = handler
9539 .handle_control(
9540 ConnectionId::new(3),
9541 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
9542 )
9543 .unwrap();
9544 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
9545
9546 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
9547 let response = handler
9548 .handle_control(
9549 ConnectionId::new(conn),
9550 hello_frame(module_id, PROTOCOL_VERSION, conn),
9551 )
9552 .unwrap();
9553 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
9554 }
9555 }
9556
9557 #[test]
9558 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
9559 let registry = Arc::new(Registry::default());
9560 let supervisor = SupervisorHandle::new();
9561 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
9562 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
9563 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
9564 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9565
9566 let owner_nonce = handler
9567 .handle_control(
9568 ConnectionId::new(1),
9569 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
9570 )
9571 .unwrap();
9572 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
9573 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
9574 assert!(registry.get_module("fed:special").unwrap().is_none());
9575
9576 let exact_nonce = handler
9577 .handle_control(
9578 ConnectionId::new(2),
9579 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
9580 )
9581 .unwrap();
9582 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
9583 assert!(registry.get_module("fed:special").unwrap().is_some());
9584 }
9585
9586 #[test]
9587 fn non_reserved_module_ignores_launch_nonce() {
9588 let registry = Arc::new(Registry::default());
9589 let handler = ControlHandler::new(Arc::clone(®istry));
9592 let no_nonce = handler
9593 .handle_control(
9594 ConnectionId::new(1),
9595 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
9596 )
9597 .unwrap();
9598 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
9599 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
9600
9601 let echoed_nonce = handler
9602 .handle_control(
9603 ConnectionId::new(2),
9604 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
9605 )
9606 .unwrap();
9607 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
9608 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
9609 }
9610
9611 #[test]
9612 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
9613 let handler = ControlHandler::default();
9614 let conn = ConnectionId::new(1);
9615 let malformed = Frame::build(
9616 FrameType::Hello,
9617 control_flags(),
9618 0,
9619 0,
9620 3,
9621 b"{not json".to_vec(),
9622 )
9623 .unwrap();
9624
9625 let error = handler.handle_control(conn, malformed).unwrap();
9626 assert_eq!(error[0].header.ty, FrameType::Error);
9627 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
9628
9629 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
9630 let pong = handler.handle_control(conn, ping).unwrap();
9631 assert_eq!(pong[0].header.ty, FrameType::Pong);
9632 assert_eq!(pong[0].header.corr, 4);
9633 }
9634
9635 #[test]
9636 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
9637 let registry = Arc::new(Registry::default());
9638 let handler = ControlHandler::new(Arc::clone(®istry));
9639
9640 handler
9641 .handle_control(
9642 ConnectionId::new(1),
9643 hello_frame("aft", PROTOCOL_VERSION, 1),
9644 )
9645 .unwrap();
9646 let duplicate = handler
9647 .handle_control(
9648 ConnectionId::new(2),
9649 hello_frame("aft", PROTOCOL_VERSION, 2),
9650 )
9651 .unwrap();
9652
9653 assert_eq!(duplicate[0].header.ty, FrameType::Error);
9654 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
9655 let registration = registry.get_module("aft").unwrap().unwrap();
9656 assert_eq!(registration.connection_id, ConnectionId::new(1));
9657 }
9658
9659 #[test]
9660 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
9661 let registry = Arc::new(Registry::default());
9662 let forwarding = Arc::new(ForwardingTable::default());
9663 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
9664 let handler =
9665 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9666 .with_process_liveness(process_liveness);
9667 let (ctx, route_channel, route_epoch) =
9668 bind_liveness_route(®istry, &forwarding, "aft-dead");
9669 let responses = handler
9670 .handle_route_poll(
9671 &ctx,
9672 route_poll_frame(41, PollKind::Liveness, route_channel),
9673 route_channel,
9674 route_epoch,
9675 PollKind::Liveness,
9676 )
9677 .unwrap();
9678
9679 assert_eq!(responses.len(), 1);
9680 assert_eq!(responses[0].header.ty, FrameType::Response);
9681 assert_route_poll_liveness(&responses[0], false);
9682 }
9683
9684 #[test]
9685 fn liveness_poll_without_process_source_uses_bound_route() {
9686 let registry = Arc::new(Registry::default());
9687 let forwarding = Arc::new(ForwardingTable::default());
9688 let handler =
9689 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9690 let (ctx, route_channel, route_epoch) =
9691 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
9692 let responses = handler
9693 .handle_route_poll(
9694 &ctx,
9695 route_poll_frame(42, PollKind::Liveness, route_channel),
9696 route_channel,
9697 route_epoch,
9698 PollKind::Liveness,
9699 )
9700 .unwrap();
9701
9702 assert_route_poll_liveness(&responses[0], true);
9703 }
9704
9705 #[test]
9706 fn liveness_poll_untracked_process_source_uses_bound_route() {
9707 let registry = Arc::new(Registry::default());
9708 let forwarding = Arc::new(ForwardingTable::default());
9709 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
9710 let handler =
9711 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9712 .with_process_liveness(process_liveness);
9713 let (ctx, route_channel, route_epoch) =
9714 bind_liveness_route(®istry, &forwarding, "aft-untracked");
9715 let responses = handler
9716 .handle_route_poll(
9717 &ctx,
9718 route_poll_frame(43, PollKind::Liveness, route_channel),
9719 route_channel,
9720 route_epoch,
9721 PollKind::Liveness,
9722 )
9723 .unwrap();
9724
9725 assert_route_poll_liveness(&responses[0], true);
9726 }
9727
9728 #[tokio::test]
9729 async fn unknown_op_returns_unknown_control_op() {
9730 let handler = ControlHandler::default();
9731 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
9732 let request = Frame::build(
9733 FrameType::Request,
9734 control_flags(),
9735 0,
9736 0,
9737 55,
9738 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
9739 )
9740 .unwrap();
9741
9742 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
9743
9744 assert_eq!(response.len(), 1);
9745 assert_eq!(response[0].header.ty, FrameType::Error);
9746 assert_eq!(response[0].header.corr, 55);
9747 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
9748 }
9749
9750 #[tokio::test]
9751 async fn supervisor_provenance_rejects_unknown_exact_module() {
9752 let handler = ControlHandler::default();
9753 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
9754 let request = Frame::build(
9755 FrameType::Request,
9756 control_flags(),
9757 0,
9758 0,
9759 57,
9760 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
9761 )
9762 .unwrap();
9763
9764 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
9765
9766 assert_eq!(response.len(), 1);
9767 assert_eq!(response[0].header.ty, FrameType::Error);
9768 assert_eq!(response[0].header.corr, 57);
9769 let error = parse_error(&response[0]);
9770 assert_eq!(error["code"], "unknown_module");
9771 assert_eq!(error["message"], "module_id 'missing' is not supervised");
9772 }
9773
9774 #[test]
9775 fn provenance_probe_override_keeps_handler_tests_deterministic() {
9776 let expected = subc_control::RunningImageAgreement::Unavailable {
9777 reason: subc_control::RunningImageUnavailableReason::HashFailed,
9778 };
9779 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
9780 assert_eq!(handler.provenance_probe_override, Some(expected));
9781 }
9782
9783 #[test]
9784 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
9785 let verdict = reload_verdict(
9786 std::path::Path::new("/bin/new"),
9787 Some(std::path::Path::new("/bin/old")),
9788 subc_control::RunningImageAgreement::Unavailable {
9789 reason: subc_control::RunningImageUnavailableReason::HashFailed,
9790 },
9791 );
9792 assert!(matches!(
9793 verdict.path,
9794 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
9795 if configured == std::path::Path::new("/bin/new")
9796 && spawned_from == std::path::Path::new("/bin/old")
9797 ));
9798 }
9799
9800 #[test]
9801 fn reload_verdict_detects_replaced_image_at_same_path() {
9802 let image = subc_control::RunningImageAgreement::Mismatch {
9803 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
9804 digest: "old".into(),
9805 },
9806 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
9807 digest: "new".into(),
9808 },
9809 };
9810 let verdict = reload_verdict(
9811 std::path::Path::new("/bin/same"),
9812 Some(std::path::Path::new("/bin/same")),
9813 image.clone(),
9814 );
9815 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
9816 assert_eq!(verdict.image, image);
9817 }
9818
9819 #[test]
9820 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
9821 let image = subc_control::RunningImageAgreement::Unavailable {
9822 reason: subc_control::RunningImageUnavailableReason::NotRunning,
9823 };
9824 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
9825 assert_eq!(
9826 verdict.path,
9827 subc_control::ReloadPathAgreement::Unavailable {
9828 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
9829 }
9830 );
9831 assert_eq!(verdict.image, image);
9832
9833 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
9834 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
9835 };
9836 let verdict = reload_verdict(
9837 std::path::Path::new("/bin/same"),
9838 Some(std::path::Path::new("/bin/same")),
9839 unconfirmed.clone(),
9840 );
9841 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
9842 assert_eq!(verdict.image, unconfirmed);
9843 }
9844
9845 #[test]
9846 fn reload_verdict_preserves_each_image_unavailability_reason() {
9847 use subc_control::RunningImageUnavailableReason as Reason;
9848
9849 for reason in [
9850 Reason::NotRunning,
9851 Reason::UnsupportedPlatform,
9852 Reason::RunningExecutableUnreadable,
9853 Reason::SpawnedPathUnreadable,
9854 Reason::HashFailed,
9855 Reason::ProcessIdentityUnconfirmed,
9856 Reason::Unknown("future_probe_reason".to_string()),
9857 ] {
9858 let image = subc_control::RunningImageAgreement::Unavailable {
9859 reason: reason.clone(),
9860 };
9861 let verdict = reload_verdict(
9862 std::path::Path::new("/bin/same"),
9863 Some(std::path::Path::new("/bin/same")),
9864 image.clone(),
9865 );
9866 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
9867 assert_eq!(verdict.image, image, "{reason:?}");
9868 }
9869 }
9870
9871 #[tokio::test]
9872 async fn malformed_control_bodies_return_invalid_control_body() {
9873 let handler = ControlHandler::default();
9874 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
9875
9876 for (corr, body) in [
9877 (56, br#"{"route_channel":1}"#.as_slice()),
9878 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
9879 (
9880 58,
9881 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
9882 ),
9883 ] {
9884 let request = Frame::build(
9885 FrameType::Request,
9886 control_flags(),
9887 0,
9888 0,
9889 corr,
9890 body.to_vec(),
9891 )
9892 .unwrap();
9893 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
9894
9895 assert_eq!(response.len(), 1);
9896 assert_eq!(response[0].header.ty, FrameType::Error);
9897 assert_eq!(response[0].header.corr, corr);
9898 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
9899 }
9900 }
9901
9902 #[tokio::test]
9903 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
9904 let registry = Arc::new(Registry::default());
9905 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
9906 let router = Router::with_control_handler(Arc::clone(&control));
9907 let connection = router.begin_connection();
9908 let (ctx, mut rx) = route_ctx(connection.id());
9909
9910 router
9911 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
9912 .await
9913 .unwrap();
9914 let response = rx.recv().await.unwrap();
9915 let ack = parse_ack(&response);
9916 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9917 let channel = 1;
9918
9919 let goodbye =
9920 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
9921 router.route_for_connection(&ctx, goodbye).await.unwrap();
9922 assert!(rx.try_recv().is_err());
9923 assert!(registry.get_module("aft").unwrap().is_none());
9924
9925 router
9926 .route_for_connection(&ctx, channel_request(channel, 13))
9927 .await
9928 .unwrap();
9929 let error_frame = rx.recv().await.unwrap();
9930 assert_eq!(error_frame.header.ty, FrameType::Error);
9931 assert_eq!(error_frame.header.channel, channel);
9932 }
9933
9934 #[tokio::test]
9935 async fn dropping_router_connection_releases_registration() {
9936 let registry = Arc::new(Registry::default());
9937 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
9938 let router = Router::with_control_handler(control);
9939 let connection = router.begin_connection();
9940 let (ctx, mut rx) = route_ctx(connection.id());
9941
9942 router
9943 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
9944 .await
9945 .unwrap();
9946 let response = rx.recv().await.unwrap();
9947 let ack = parse_ack(&response);
9948 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9949 assert!(registry.get_module("aft").unwrap().is_some());
9950
9951 drop(connection);
9952
9953 assert!(registry.get_module("aft").unwrap().is_none());
9954 assert_eq!(registry.active_registration_count().unwrap(), 0);
9955 }
9956
9957 fn capability_manifest(
9958 module_id: &str,
9959 provides: &[&str],
9960 must_never_reach: &[&str],
9961 ) -> ModuleManifest {
9962 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
9963 manifest.capabilities = Some(CapabilityDeclarations {
9964 provides: provides
9965 .iter()
9966 .map(|capability| (*capability).to_string())
9967 .collect(),
9968 requires: Vec::new(),
9969 must_never_reach: must_never_reach
9970 .iter()
9971 .map(|capability| (*capability).to_string())
9972 .collect(),
9973 });
9974 manifest
9975 }
9976
9977 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
9978 Frame::build(
9979 FrameType::Hello,
9980 control_flags(),
9981 0,
9982 0,
9983 corr,
9984 serde_json::to_vec(&ModuleHelloBody {
9985 protocol_ver: manifest.protocol_ver,
9986 manifest,
9987 control_ops: None,
9988 launch_nonce: None,
9989 })
9990 .expect("capability test HELLO serializes"),
9991 )
9992 .expect("capability test HELLO frame builds")
9993 }
9994
9995 fn catalog_update_with_capabilities_frame(
9996 corr: u64,
9997 capabilities: CapabilityDeclarations,
9998 ) -> Frame {
9999 Frame::build(
10000 FrameType::Request,
10001 control_flags(),
10002 0,
10003 0,
10004 corr,
10005 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
10006 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
10007 capabilities: Some(capabilities),
10008 ready: None,
10009 })
10010 .expect("capability catalog.update serializes"),
10011 )
10012 .expect("capability catalog.update frame builds")
10013 }
10014
10015 async fn register_capability_manifest(
10016 handler: &ControlHandler,
10017 ctx: &RouteCtx,
10018 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
10019 manifest: ModuleManifest,
10020 corr: u64,
10021 ) {
10022 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
10023 }
10024
10025 async fn open_route_for_capability_test(
10026 handler: &ControlHandler,
10027 target_ctx: &RouteCtx,
10028 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
10029 client_connection_id: u64,
10030 corr: u64,
10031 target_module_id: &str,
10032 consumer_identity: Option<ConsumerIdentity>,
10033 ) -> (
10034 mpsc::Receiver<crate::router::OutboundFrame>,
10035 ModuleControlRequest,
10036 ) {
10037 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
10038 let route_handler = handler.clone();
10039 let target_module_id = target_module_id.to_string();
10040 let route_task = tokio::spawn(async move {
10041 route_handler
10042 .handle_control_frame(
10043 &client_ctx,
10044 route_open_frame_with_admission_facts(
10045 corr,
10046 &target_module_id,
10047 unique_project_root("admission-facts"),
10048 consumer_identity,
10049 None,
10050 ),
10051 )
10052 .await
10053 .expect("capability test route.open succeeds")
10054 });
10055 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
10056 .await
10057 .expect("capability test route.open must reach route.bind")
10058 .expect("target control receiver stays open");
10059 let bind_request: ModuleControlRequest =
10060 serde_json::from_slice(&bind.body).expect("route.bind decodes");
10061 handler
10062 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
10063 .await
10064 .expect("capability test route.bind ACK succeeds");
10065 assert!(route_task.await.expect("route.open task joins").is_empty());
10066 let opened = client_rx
10067 .recv()
10068 .await
10069 .expect("successful route.open publishes a response");
10070 assert!(matches!(
10071 serde_json::from_slice::<ClientControlResponse>(&opened.body),
10072 Ok(ClientControlResponse::RouteOpen { .. })
10073 ));
10074 (client_rx, bind_request)
10075 }
10076
10077 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
10078 assert_eq!(frame.header.ty, FrameType::Push);
10079 assert_eq!(frame.header.channel, 0);
10080 assert_eq!(
10081 serde_json::from_slice::<ClientControlPush>(&frame.body)
10082 .expect("route.closed control push decodes"),
10083 ClientControlPush::RouteClosed {
10084 module_id: target_module_id.to_string(),
10085 reason: RouteCloseReason::CapabilityDenied,
10086 drained: false,
10087 abandoned: 0,
10088 excluded_subscriptions: 0,
10089 terminal: Some(false),
10090 }
10091 );
10092 }
10093
10094 #[tokio::test]
10095 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
10096 let registry = Arc::new(Registry::default());
10097 let forwarding = Arc::new(ForwardingTable::default());
10098 let supervisor = SupervisorHandle::new();
10099 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
10100 let handler =
10101 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10102 .with_supervisor(supervisor);
10103 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
10104 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
10105 register_capability_manifest(
10106 &handler,
10107 &target_ctx,
10108 &mut target_rx,
10109 capability_manifest("target", &["credentials-provider/v1"], &[]),
10110 1,
10111 )
10112 .await;
10113 register_capability_manifest(
10114 &handler,
10115 &opener_ctx,
10116 &mut opener_rx,
10117 capability_manifest("opener", &[], &["credentials-provider/v1"]),
10118 2,
10119 )
10120 .await;
10121
10122 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
10123 let replies = handler
10124 .handle_control_frame(
10125 &client_ctx,
10126 route_open_frame_with_admission_facts(
10127 3,
10128 "target",
10129 unique_project_root("admission-facts"),
10130 Some(ConsumerIdentity {
10131 module_id: "opener".to_string(),
10132 launch_nonce: "opener-nonce".to_string(),
10133 }),
10134 None,
10135 ),
10136 )
10137 .await
10138 .expect("denied route.open returns a typed frame");
10139 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
10140 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
10141 assert!(
10142 target_rx.try_recv().is_err(),
10143 "forbidden route.open must not relay route.bind"
10144 );
10145 }
10146
10147 #[tokio::test]
10148 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
10149 let registry = Arc::new(Registry::default());
10150 let forwarding = Arc::new(ForwardingTable::default());
10151 let supervisor = SupervisorHandle::new();
10152 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
10153 let handler =
10154 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10155 .with_supervisor(supervisor);
10156 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
10157 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
10158 register_capability_manifest(
10159 &handler,
10160 &target_ctx,
10161 &mut target_rx,
10162 capability_manifest("target", &["credentials-provider/v1"], &[]),
10163 1,
10164 )
10165 .await;
10166 register_capability_manifest(
10167 &handler,
10168 &old_opener_ctx,
10169 &mut old_opener_rx,
10170 capability_manifest("opener", &[], &[]),
10171 2,
10172 )
10173 .await;
10174 let (mut client_rx, _) = open_route_for_capability_test(
10175 &handler,
10176 &target_ctx,
10177 &mut target_rx,
10178 712,
10179 3,
10180 "target",
10181 Some(ConsumerIdentity {
10182 module_id: "opener".to_string(),
10183 launch_nonce: "opener-nonce".to_string(),
10184 }),
10185 )
10186 .await;
10187 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
10188
10189 handler
10190 .cleanup_connection(old_opener_ctx.connection_id)
10191 .expect("old opener registration cleans up");
10192 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
10193 register_capability_manifest(
10194 &handler,
10195 &new_opener_ctx,
10196 &mut new_opener_rx,
10197 capability_manifest("opener", &[], &["credentials-provider/v1"]),
10198 4,
10199 )
10200 .await;
10201
10202 assert_capability_denied_push(
10203 client_rx
10204 .try_recv()
10205 .expect("HELLO deny addition must emit route.closed")
10206 .frame,
10207 "target",
10208 );
10209 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
10210 assert!(matches!(
10211 target_rx.try_recv(),
10212 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
10213 ));
10214 }
10215
10216 #[tokio::test]
10217 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
10218 let registry = Arc::new(Registry::default());
10219 let forwarding = Arc::new(ForwardingTable::default());
10220 let supervisor = SupervisorHandle::new();
10221 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
10222 let handler =
10223 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10224 .with_supervisor(supervisor);
10225 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
10226 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
10227 register_capability_manifest(
10228 &handler,
10229 &target_ctx,
10230 &mut target_rx,
10231 capability_manifest("target", &[], &[]),
10232 1,
10233 )
10234 .await;
10235 register_capability_manifest(
10236 &handler,
10237 &opener_ctx,
10238 &mut opener_rx,
10239 capability_manifest("opener", &[], &["credentials-provider/v1"]),
10240 2,
10241 )
10242 .await;
10243 let (mut client_rx, _) = open_route_for_capability_test(
10244 &handler,
10245 &target_ctx,
10246 &mut target_rx,
10247 722,
10248 3,
10249 "target",
10250 Some(ConsumerIdentity {
10251 module_id: "opener".to_string(),
10252 launch_nonce: "opener-nonce".to_string(),
10253 }),
10254 )
10255 .await;
10256 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
10257
10258 let replies = handler
10259 .handle_control_frame(
10260 &target_ctx,
10261 catalog_update_with_capabilities_frame(
10262 4,
10263 CapabilityDeclarations {
10264 provides: vec!["credentials-provider/v1".to_string()],
10265 requires: Vec::new(),
10266 must_never_reach: Vec::new(),
10267 },
10268 ),
10269 )
10270 .await
10271 .expect("claim catalog.update succeeds");
10272 assert!(matches!(
10273 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
10274 Ok(ModuleControlResponseToModule::CatalogUpdate {})
10275 ));
10276 assert_capability_denied_push(
10277 client_rx
10278 .try_recv()
10279 .expect("claim addition must emit route.closed")
10280 .frame,
10281 "target",
10282 );
10283 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
10284 assert!(matches!(
10285 target_rx.try_recv(),
10286 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
10287 ));
10288 }
10289
10290 #[tokio::test]
10291 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
10292 let registry = Arc::new(Registry::default());
10293 let forwarding = Arc::new(ForwardingTable::default());
10294 let supervisor = SupervisorHandle::new();
10295 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
10296 let handler =
10297 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10298 .with_supervisor(supervisor);
10299 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
10300 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
10301 register_capability_manifest(
10302 &handler,
10303 &target_ctx,
10304 &mut target_rx,
10305 capability_manifest("target", &["credentials-provider/v1"], &[]),
10306 1,
10307 )
10308 .await;
10309 register_capability_manifest(
10310 &handler,
10311 &opener_ctx,
10312 &mut opener_rx,
10313 capability_manifest("opener", &[], &[]),
10314 2,
10315 )
10316 .await;
10317 let (mut client_rx, _) = open_route_for_capability_test(
10318 &handler,
10319 &target_ctx,
10320 &mut target_rx,
10321 732,
10322 3,
10323 "target",
10324 Some(ConsumerIdentity {
10325 module_id: "opener".to_string(),
10326 launch_nonce: "opener-nonce".to_string(),
10327 }),
10328 )
10329 .await;
10330 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
10331
10332 handler
10333 .handle_control_frame(
10334 &target_ctx,
10335 catalog_update_with_capabilities_frame(
10336 4,
10337 CapabilityDeclarations {
10338 provides: Vec::new(),
10339 requires: Vec::new(),
10340 must_never_reach: Vec::new(),
10341 },
10342 ),
10343 )
10344 .await
10345 .expect("claim removal catalog.update succeeds");
10346 assert_eq!(
10347 forwarding.active_binding_count().unwrap(),
10348 1,
10349 "removing an attested target claim must leave the route census unchanged"
10350 );
10351 assert!(
10352 client_rx.try_recv().is_err(),
10353 "claim removal must not emit route.closed capability_denied"
10354 );
10355 assert!(
10356 target_rx.try_recv().is_err(),
10357 "claim removal must not send the target a route GOODBYE"
10358 );
10359 }
10360
10361 #[tokio::test]
10364 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
10365 let registry = Arc::new(Registry::default());
10366 let forwarding = Arc::new(ForwardingTable::default());
10367 let supervisor = SupervisorHandle::new();
10368 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
10369 let handler =
10370 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10371 .with_supervisor(supervisor);
10372 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
10373 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
10374 register_capability_manifest(
10375 &handler,
10376 &target_ctx,
10377 &mut target_rx,
10378 capability_manifest("target", &["credentials-provider/v1"], &[]),
10379 1,
10380 )
10381 .await;
10382 register_capability_manifest(
10383 &handler,
10384 &opener_ctx,
10385 &mut opener_rx,
10386 capability_manifest("opener", &[], &["credentials-provider/v1"]),
10387 2,
10388 )
10389 .await;
10390
10391 let (_client_rx, bind) = open_route_for_capability_test(
10392 &handler,
10393 &target_ctx,
10394 &mut target_rx,
10395 742,
10396 3,
10397 "target",
10398 None,
10399 )
10400 .await;
10401 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
10402 panic!("direct scope-honesty route must bind");
10403 };
10404 assert_eq!(principal, Some(Principal::Direct));
10405 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
10406 }
10407
10408 #[tokio::test]
10411 async fn must_never_reach_self_route_is_capability_forbidden() {
10412 let registry = Arc::new(Registry::default());
10413 let forwarding = Arc::new(ForwardingTable::default());
10414 let supervisor = SupervisorHandle::new();
10415 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
10416 let handler =
10417 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10418 .with_supervisor(supervisor);
10419 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
10420 register_capability_manifest(
10421 &handler,
10422 &self_ctx,
10423 &mut self_rx,
10424 capability_manifest(
10425 "self-provider",
10426 &["credentials-provider/v1"],
10427 &["credentials-provider/v1"],
10428 ),
10429 1,
10430 )
10431 .await;
10432
10433 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
10434 let replies = handler
10435 .handle_control_frame(
10436 &client_ctx,
10437 route_open_frame_with_admission_facts(
10438 2,
10439 "self-provider",
10440 unique_project_root("admission-facts"),
10441 Some(ConsumerIdentity {
10442 module_id: "self-provider".to_string(),
10443 launch_nonce: "self-nonce".to_string(),
10444 }),
10445 None,
10446 ),
10447 )
10448 .await
10449 .expect("self-route refusal returns a typed frame");
10450 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
10451 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
10452 assert!(
10453 self_rx.try_recv().is_err(),
10454 "self denial must not relay route.bind"
10455 );
10456 }
10457
10458 #[test]
10459 fn unsupported_channel_zero_frame_returns_error() {
10460 let handler = ControlHandler::default();
10461 let request = Frame::build(
10462 FrameType::Request,
10463 control_flags(),
10464 0,
10465 0,
10466 21,
10467 b"opaque".to_vec(),
10468 )
10469 .unwrap();
10470
10471 let response = handler
10472 .handle_control(ConnectionId::new(1), request)
10473 .unwrap();
10474
10475 assert_eq!(response[0].header.ty, FrameType::Error);
10476 assert_eq!(
10477 parse_error(&response[0])["code"],
10478 "unsupported_control_frame"
10479 );
10480 }
10481
10482 mod swap {
10487 use super::*;
10488
10489 const INCUMBENT: ConnectionId = ConnectionId::new(30);
10490 const CANDIDATE: ConnectionId = ConnectionId::new(40);
10491
10492 struct Swap {
10493 registry: Arc<Registry>,
10494 forwarding: Arc<ForwardingTable>,
10495 handler: ControlHandler,
10496 incumbent_ctx: RouteCtx,
10497 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
10498 candidate_ctx: RouteCtx,
10499 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
10500 }
10501
10502 async fn swap_with_incumbent() -> Swap {
10503 let registry = Arc::new(Registry::default());
10504 let forwarding = Arc::new(ForwardingTable::default());
10505 let handler =
10506 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10507 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
10508 hello_via_sink(
10509 &handler,
10510 &incumbent_ctx,
10511 &mut incumbent_rx,
10512 hello_frame("aft", PROTOCOL_VERSION, 7),
10513 )
10514 .await;
10515 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
10516 Swap {
10517 registry,
10518 forwarding,
10519 handler,
10520 incumbent_ctx,
10521 incumbent_rx,
10522 candidate_ctx,
10523 candidate_rx,
10524 }
10525 }
10526
10527 fn register_candidate(swap: &Swap, ready: Option<bool>) {
10528 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
10529 candidate_manifest.ready = ready;
10530 let registration = swap
10531 .registry
10532 .register_candidate_with_control_ops(
10533 candidate_manifest,
10534 PROTOCOL_VERSION,
10535 CANDIDATE,
10536 module_baseline_control_ops(),
10537 )
10538 .unwrap();
10539 swap.forwarding
10540 .register_candidate_module_connection(
10541 CANDIDATE,
10542 "aft".to_string(),
10543 PROTOCOL_VERSION,
10544 manifest_concurrency(®istration.manifest),
10545 swap.candidate_ctx.egress.clone(),
10546 )
10547 .unwrap();
10548 }
10549
10550 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
10551 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
10552 swap.registry.promote_candidate("aft").unwrap().unwrap();
10553 cutover.incumbent.unwrap()
10554 }
10555
10556 fn keyed_total(counters: &Value, key: &str) -> u64 {
10557 counters[key]
10558 .as_object()
10559 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
10560 .unwrap_or(0)
10561 }
10562
10563 #[tokio::test]
10569 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
10570 let mut swap = swap_with_incumbent().await;
10571 let handler = swap.handler.clone();
10572
10573 let cotenant = ConnectionId::new(31);
10575 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
10576 let (cotenant_task, cotenant_bind) = relay_route_open(
10577 &handler,
10578 cotenant,
10579 &cotenant_ctx.egress,
10580 &mut swap.incumbent_rx,
10581 100,
10582 "aft",
10583 "swap-cotenant",
10584 )
10585 .await;
10586 handler
10587 .handle_control_frame(
10588 &swap.incumbent_ctx,
10589 route_bind_ack(cotenant_bind.header.corr),
10590 )
10591 .await
10592 .unwrap();
10593 assert!(cotenant_task.await.unwrap().is_empty());
10594 let (cotenant_channel, cotenant_epoch) =
10595 published_route(&cotenant_rx.recv().await.unwrap());
10596
10597 let caller = ConnectionId::new(32);
10599 let (caller_ctx, mut caller_rx) = route_ctx(caller);
10600 let (caller_task, caller_bind) = relay_route_open(
10601 &handler,
10602 caller,
10603 &caller_ctx.egress,
10604 &mut swap.incumbent_rx,
10605 101,
10606 "aft",
10607 "swap-caller",
10608 )
10609 .await;
10610 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
10611
10612 register_candidate(&swap, None);
10613 cutover(&swap);
10614
10615 let ack = handler
10617 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
10618 .await;
10619 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
10620 if module_loop_error.is_some() {
10621 handler.cleanup_connection(INCUMBENT).unwrap();
10624 }
10625
10626 assert!(
10628 cotenant_rx.try_recv().is_err(),
10629 "the co-tenant route on the incumbent was torn down by one late ack: \
10630 {module_loop_error:?}"
10631 );
10632 assert!(matches!(
10633 swap.forwarding
10634 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
10635 .unwrap(),
10636 DataRoute::Client(DataRouteState::Bound(_))
10637 ));
10638 assert_eq!(module_loop_error, None);
10639 assert!(swap
10640 .registry
10641 .get_module_by_connection(INCUMBENT)
10642 .unwrap()
10643 .is_some());
10644
10645 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
10647 .await
10648 .expect("the incumbent is told to drop the abandoned binding")
10649 .unwrap()
10650 .frame;
10651 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10652 assert_eq!(goodbye.header.channel, abandoned_channel);
10653 assert_eq!(goodbye.header.epoch, abandoned_epoch);
10654 assert!(swap.incumbent_rx.try_recv().is_err());
10655
10656 let response = caller_task.await.unwrap();
10658 assert_eq!(response.len(), 1);
10659 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
10660 assert!(caller_rx.try_recv().is_err());
10661
10662 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
10666 let counters = handler.counters().snapshot();
10667 assert_eq!(
10668 keyed_total(&counters, "route_open_accepted_by_principal"),
10669 1
10670 );
10671 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
10672 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
10673 }
10674
10675 #[tokio::test]
10679 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
10680 let mut swap = swap_with_incumbent().await;
10681 register_candidate(&swap, None);
10682 let incumbent = cutover(&swap);
10683 swap.forwarding
10684 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
10685 .unwrap()
10686 .expect("the incumbent is still registered");
10687
10688 let client = ConnectionId::new(33);
10689 let (client_ctx, mut client_rx) = route_ctx(client);
10690 let route_handler = swap.handler.clone();
10691 let open_ctx = RouteCtx {
10692 connection_id: client,
10693 egress: client_ctx.egress.clone(),
10694 };
10695 let mut route_task = tokio::spawn(async move {
10696 route_handler
10697 .handle_control_frame(
10698 &open_ctx,
10699 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
10700 )
10701 .await
10702 .unwrap()
10703 });
10704 let bind = tokio::select! {
10705 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
10706 response = &mut route_task => {
10707 let response = response.unwrap();
10708 panic!(
10709 "post-cutover route.open was refused instead of relayed to the candidate: {}",
10710 parse_error(&response[0])["code"]
10711 );
10712 }
10713 };
10714 swap.handler
10715 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
10716 .await
10717 .unwrap();
10718 assert!(route_task.await.unwrap().is_empty());
10719 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
10720 match swap
10721 .forwarding
10722 .lookup_data_route(client, channel, epoch)
10723 .unwrap()
10724 {
10725 DataRoute::Client(DataRouteState::Bound(route)) => {
10726 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
10727 }
10728 other => panic!("expected a bound route on the candidate, got {other:?}"),
10729 }
10730 assert!(swap.incumbent_rx.try_recv().is_err());
10731 }
10732
10733 #[tokio::test]
10738 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
10739 let swap = swap_with_incumbent().await;
10740 register_candidate(&swap, Some(false));
10741 let update = Frame::build(
10742 FrameType::Request,
10743 control_flags(),
10744 0,
10745 0,
10746 55,
10747 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
10748 provides: manifest("aft", PROTOCOL_VERSION).provides,
10749 capabilities: None,
10750 ready: Some(true),
10751 })
10752 .unwrap(),
10753 )
10754 .unwrap();
10755
10756 let replies = swap
10757 .handler
10758 .handle_control_frame(&swap.candidate_ctx, update)
10759 .await
10760 .unwrap();
10761
10762 assert_eq!(replies.len(), 1);
10763 assert_eq!(
10764 replies[0].header.ty,
10765 FrameType::Response,
10766 "candidate catalog.update was refused: {:?}",
10767 serde_json::from_slice::<Value>(&replies[0].body).ok()
10768 );
10769 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
10770 assert_eq!(
10771 swap.registry
10772 .get_module("aft")
10773 .unwrap()
10774 .unwrap()
10775 .connection_id,
10776 INCUMBENT
10777 );
10778 }
10779 }
10780
10781 mod swap_admission {
10785 use super::*;
10786
10787 const INCUMBENT_NONCE: &str = "incumbent-nonce";
10788 const CANDIDATE_NONCE: &str = "candidate-nonce";
10789
10790 fn handler_with_incumbent(
10791 module_id: &str,
10792 reserved: bool,
10793 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
10794 let registry = Arc::new(Registry::default());
10795 let supervisor = SupervisorHandle::new();
10796 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
10797 if reserved {
10798 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
10799 }
10800 let handler =
10801 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
10802 let incumbent = handler
10803 .handle_control(
10804 ConnectionId::new(1),
10805 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
10806 )
10807 .unwrap();
10808 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
10809 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
10810 (registry, supervisor, handler)
10811 }
10812
10813 #[test]
10819 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
10820 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
10821
10822 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
10823 let replies = handler
10824 .handle_control(
10825 ConnectionId::new(connection),
10826 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
10827 )
10828 .unwrap();
10829 assert_eq!(replies[0].header.ty, FrameType::Error);
10830 assert_eq!(
10831 parse_error(&replies[0])["code"],
10832 "swap_token_invalid",
10833 "nonce {nonce:?}"
10834 );
10835 }
10836 assert!(registry.get_candidate("aft").unwrap().is_none());
10837 assert_eq!(
10838 registry.get_module("aft").unwrap().unwrap().connection_id,
10839 ConnectionId::new(1)
10840 );
10841
10842 let admitted = handler
10845 .handle_control(
10846 ConnectionId::new(4),
10847 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
10848 )
10849 .unwrap();
10850 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
10851 assert_eq!(
10852 registry
10853 .get_candidate("aft")
10854 .unwrap()
10855 .unwrap()
10856 .connection_id,
10857 ConnectionId::new(4)
10858 );
10859 assert_eq!(
10860 registry.get_module("aft").unwrap().unwrap().connection_id,
10861 ConnectionId::new(1),
10862 "the candidate must not take the active slot"
10863 );
10864 let replayed = handler
10865 .handle_control(
10866 ConnectionId::new(5),
10867 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
10868 )
10869 .unwrap();
10870 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
10871
10872 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
10876 let squatter = handler
10877 .handle_control(
10878 ConnectionId::new(6),
10879 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
10880 )
10881 .unwrap();
10882 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
10883 assert!(
10884 registry.get_module("aft").unwrap().is_none(),
10885 "a squatter took the active slot of an id being swapped"
10886 );
10887 }
10888
10889 #[test]
10894 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
10895 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
10896
10897 let replies = handler
10898 .handle_control(
10899 ConnectionId::new(2),
10900 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
10901 )
10902 .unwrap();
10903
10904 assert_eq!(
10905 replies[0].header.ty,
10906 FrameType::HelloAck,
10907 "reserved candidate refused: {:?}",
10908 serde_json::from_slice::<Value>(&replies[0].body).ok()
10909 );
10910 assert_eq!(
10911 registry
10912 .get_candidate("vault")
10913 .unwrap()
10914 .unwrap()
10915 .connection_id,
10916 ConnectionId::new(2)
10917 );
10918 }
10919
10920 #[test]
10923 fn without_an_open_swap_the_ordinary_gates_decide() {
10924 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
10925 supervisor.close_swap("vault");
10926
10927 let candidate = handler
10928 .handle_control(
10929 ConnectionId::new(2),
10930 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
10931 )
10932 .unwrap();
10933 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
10934 let duplicate = handler
10935 .handle_control(
10936 ConnectionId::new(3),
10937 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
10938 )
10939 .unwrap();
10940 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
10941 assert!(registry.get_candidate("vault").unwrap().is_none());
10942 }
10943 }
10944}
10945
10946#[cfg(test)]
10947mod concurrency_default_exposure_tests {
10948 use super::*;
10949
10950 fn hello_body(role_json: &str) -> Vec<u8> {
10951 format!(
10952 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
10953 )
10954 .into_bytes()
10955 }
10956
10957 fn manifest_from(body: &[u8]) -> ModuleManifest {
10958 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
10959 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
10960 .expect("manifest parses")
10961 }
10962
10963 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
10964
10965 #[test]
10966 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
10967 let body = hello_body(&format!(
10968 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
10969 ));
10970 let manifest = manifest_from(&body);
10971 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
10974 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
10975 }
10976
10977 #[test]
10978 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
10979 let body = hello_body(&format!(
10980 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
10981 ));
10982 let manifest = manifest_from(&body);
10983 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
10984 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
10985 }
10986
10987 #[test]
10988 fn non_management_roles_are_never_reported() {
10989 let body = hello_body(
10990 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
10991 );
10992 let manifest = manifest_from(&body);
10993 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
10994 }
10995}